mbox series

[0/6] PCI: Fix deadlocks when enabling ASPM

Message ID 20231114135553.32301-1-johan+linaro@kernel.org
Headers show
Series PCI: Fix deadlocks when enabling ASPM | expand

Message

Johan Hovold Nov. 14, 2023, 1:55 p.m. UTC
The pci_enable_link_state() helper is currently only called from
pci_walk_bus(), something which can lead to a deadlock as both helpers
take a pci_bus_sem read lock.

Add a new locked helper which can be called with the read lock held and
fix up the two current users (the second is new in 6.7-rc1).

Note that there are no users left of the original unlocked variant after
this series, but I decided to leave it in place for now (e.g. to mirror
the corresponding helpers to disable link states).

Included are also a couple of related cleanups.

Johan


Johan Hovold (6):
  PCI/ASPM: Add locked helper for enabling link state
  PCI: vmd: Fix deadlock when enabling ASPM
  PCI: qcom: Fix deadlock when enabling ASPM
  PCI: qcom: Clean up ASPM comment
  PCI/ASPM: Clean up disable link state parameter
  PCI/ASPM: Add lockdep assert to link state helper

 drivers/pci/controller/dwc/pcie-qcom.c |  7 ++-
 drivers/pci/controller/vmd.c           |  2 +-
 drivers/pci/pcie/aspm.c                | 65 +++++++++++++++++++-------
 include/linux/pci.h                    |  3 ++
 4 files changed, 56 insertions(+), 21 deletions(-)

Comments

Manivannan Sadhasivam Nov. 17, 2023, 10:28 a.m. UTC | #1
On Tue, Nov 14, 2023 at 02:55:48PM +0100, Johan Hovold wrote:
> Add a helper for enabling link states that can be used in contexts where
> a pci_bus_sem read lock is already held (e.g. from pci_walk_bus()).
> 
> This helper will be used to fix a couple of potential deadlocks where
> the current helper is called with the lock already held, hence the CC
> stable tag.
> 
> Fixes: f492edb40b54 ("PCI: vmd: Add quirk to configure PCIe ASPM and LTR")
> Cc: stable@vger.kernel.org	# 6.3
> Cc: Michael Bottini <michael.a.bottini@linux.intel.com>
> Cc: David E. Box <david.e.box@linux.intel.com>
> Signed-off-by: Johan Hovold <johan+linaro@kernel.org>

With the Kdoc comment fixed,

Reviewed-by: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>

- Mani

> ---
>  drivers/pci/pcie/aspm.c | 53 +++++++++++++++++++++++++++++++----------
>  include/linux/pci.h     |  3 +++
>  2 files changed, 43 insertions(+), 13 deletions(-)
> 
> diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c
> index 50b04ae5c394..8cf8cc2d6bba 100644
> --- a/drivers/pci/pcie/aspm.c
> +++ b/drivers/pci/pcie/aspm.c
> @@ -1109,17 +1109,7 @@ int pci_disable_link_state(struct pci_dev *pdev, int state)
>  }
>  EXPORT_SYMBOL(pci_disable_link_state);
>  
> -/**
> - * pci_enable_link_state - Clear and set the default device link state so that
> - * the link may be allowed to enter the specified states. Note that if the
> - * BIOS didn't grant ASPM control to the OS, this does nothing because we can't
> - * touch the LNKCTL register. Also note that this does not enable states
> - * disabled by pci_disable_link_state(). Return 0 or a negative errno.
> - *
> - * @pdev: PCI device
> - * @state: Mask of ASPM link states to enable
> - */
> -int pci_enable_link_state(struct pci_dev *pdev, int state)
> +static int __pci_enable_link_state(struct pci_dev *pdev, int state, bool locked)
>  {
>  	struct pcie_link_state *link = pcie_aspm_get_link(pdev);
>  
> @@ -1136,7 +1126,8 @@ int pci_enable_link_state(struct pci_dev *pdev, int state)
>  		return -EPERM;
>  	}
>  
> -	down_read(&pci_bus_sem);
> +	if (!locked)
> +		down_read(&pci_bus_sem);
>  	mutex_lock(&aspm_lock);
>  	link->aspm_default = 0;
>  	if (state & PCIE_LINK_STATE_L0S)
> @@ -1157,12 +1148,48 @@ int pci_enable_link_state(struct pci_dev *pdev, int state)
>  	link->clkpm_default = (state & PCIE_LINK_STATE_CLKPM) ? 1 : 0;
>  	pcie_set_clkpm(link, policy_to_clkpm_state(link));
>  	mutex_unlock(&aspm_lock);
> -	up_read(&pci_bus_sem);
> +	if (!locked)
> +		up_read(&pci_bus_sem);
>  
>  	return 0;
>  }
> +
> +/**
> + * pci_enable_link_state - Clear and set the default device link state so that
> + * the link may be allowed to enter the specified states. Note that if the
> + * BIOS didn't grant ASPM control to the OS, this does nothing because we can't
> + * touch the LNKCTL register. Also note that this does not enable states
> + * disabled by pci_disable_link_state(). Return 0 or a negative errno.
> + *
> + * @pdev: PCI device
> + * @state: Mask of ASPM link states to enable
> + */
> +int pci_enable_link_state(struct pci_dev *pdev, int state)
> +{
> +	return __pci_enable_link_state(pdev, state, false);
> +}
>  EXPORT_SYMBOL(pci_enable_link_state);
>  
> +/**
> + * pci_enable_link_state - Clear and set the default device link state so that
> + * the link may be allowed to enter the specified states. Note that if the
> + * BIOS didn't grant ASPM control to the OS, this does nothing because we can't
> + * touch the LNKCTL register. Also note that this does not enable states
> + * disabled by pci_disable_link_state(). Return 0 or a negative errno.
> + *
> + * @pdev: PCI device
> + * @state: Mask of ASPM link states to enable
> + *
> + * Context: Caller holds pci_bus_sem read lock.
> + */
> +int pci_enable_link_state_locked(struct pci_dev *pdev, int state)
> +{
> +	lockdep_assert_held_read(&pci_bus_sem);
> +
> +	return __pci_enable_link_state(pdev, state, true);
> +}
> +EXPORT_SYMBOL(pci_enable_link_state_locked);
> +
>  static int pcie_aspm_set_policy(const char *val,
>  				const struct kernel_param *kp)
>  {
> diff --git a/include/linux/pci.h b/include/linux/pci.h
> index 60ca768bc867..dea043bc1e38 100644
> --- a/include/linux/pci.h
> +++ b/include/linux/pci.h
> @@ -1829,6 +1829,7 @@ extern bool pcie_ports_native;
>  int pci_disable_link_state(struct pci_dev *pdev, int state);
>  int pci_disable_link_state_locked(struct pci_dev *pdev, int state);
>  int pci_enable_link_state(struct pci_dev *pdev, int state);
> +int pci_enable_link_state_locked(struct pci_dev *pdev, int state);
>  void pcie_no_aspm(void);
>  bool pcie_aspm_support_enabled(void);
>  bool pcie_aspm_enabled(struct pci_dev *pdev);
> @@ -1839,6 +1840,8 @@ static inline int pci_disable_link_state_locked(struct pci_dev *pdev, int state)
>  { return 0; }
>  static inline int pci_enable_link_state(struct pci_dev *pdev, int state)
>  { return 0; }
> +static inline int pci_enable_link_state_locked(struct pci_dev *pdev, int state)
> +{ return 0; }
>  static inline void pcie_no_aspm(void) { }
>  static inline bool pcie_aspm_support_enabled(void) { return false; }
>  static inline bool pcie_aspm_enabled(struct pci_dev *pdev) { return false; }
> -- 
> 2.41.0
>
Manivannan Sadhasivam Nov. 17, 2023, 10:28 a.m. UTC | #2
On Tue, Nov 14, 2023 at 02:55:50PM +0100, Johan Hovold wrote:
> The qcom_pcie_enable_aspm() helper is called from pci_walk_bus() during
> host init to enable ASPM.
> 
> Since pci_walk_bus() already holds a pci_bus_sem read lock, use the new
> locked helper to enable link states in order to avoid a potential
> deadlock (e.g. in case someone takes a write lock before reacquiring
> the read lock).
> 
> This issue was reported by lockdep:
> 
>    ============================================
>    WARNING: possible recursive locking detected
>    6.7.0-rc1 #4 Not tainted
>    --------------------------------------------
>    kworker/u16:6/147 is trying to acquire lock:
>    ffffbf3ff9d2cfa0 (pci_bus_sem){++++}-{3:3}, at: pci_enable_link_state+0x74/0x1e8
> 
>    but task is already holding lock:
>    ffffbf3ff9d2cfa0 (pci_bus_sem){++++}-{3:3}, at: pci_walk_bus+0x34/0xbc
> 
>    other info that might help us debug this:
>     Possible unsafe locking scenario:
> 
>           CPU0
>           ----
>      lock(pci_bus_sem);
>      lock(pci_bus_sem);
> 
>     *** DEADLOCK ***
> 
> Fixes: 9f4f3dfad8cf ("PCI: qcom: Enable ASPM for platforms supporting 1.9.0 ops")
> Cc: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
> Signed-off-by: Johan Hovold <johan+linaro@kernel.org>

Reviewed-by: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>

- Mani

> ---
>  drivers/pci/controller/dwc/pcie-qcom.c | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c
> index ce3ece28fed2..21523115f6a4 100644
> --- a/drivers/pci/controller/dwc/pcie-qcom.c
> +++ b/drivers/pci/controller/dwc/pcie-qcom.c
> @@ -971,7 +971,7 @@ static int qcom_pcie_enable_aspm(struct pci_dev *pdev, void *userdata)
>  {
>  	/* Downstream devices need to be in D0 state before enabling PCI PM substates */
>  	pci_set_power_state(pdev, PCI_D0);
> -	pci_enable_link_state(pdev, PCIE_LINK_STATE_ALL);
> +	pci_enable_link_state_locked(pdev, PCIE_LINK_STATE_ALL);
>  
>  	return 0;
>  }
> -- 
> 2.41.0
>
Manivannan Sadhasivam Nov. 17, 2023, 10:32 a.m. UTC | #3
On Tue, Nov 14, 2023 at 02:55:51PM +0100, Johan Hovold wrote:
> Break up the newly added ASPM comment so that it fits within the soft 80
> character limit and becomes more readable.
> 
> Signed-off-by: Johan Hovold <johan+linaro@kernel.org>

I think we discussed (80column soft limit for comments) in the past, but I don't
think breaking here makes the comment more readable.

- Mani

> ---
>  drivers/pci/controller/dwc/pcie-qcom.c | 5 ++++-
>  1 file changed, 4 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c
> index 21523115f6a4..a6f08acff3d4 100644
> --- a/drivers/pci/controller/dwc/pcie-qcom.c
> +++ b/drivers/pci/controller/dwc/pcie-qcom.c
> @@ -969,7 +969,10 @@ static int qcom_pcie_post_init_2_7_0(struct qcom_pcie *pcie)
>  
>  static int qcom_pcie_enable_aspm(struct pci_dev *pdev, void *userdata)
>  {
> -	/* Downstream devices need to be in D0 state before enabling PCI PM substates */
> +	/*
> +	 * Downstream devices need to be in D0 state before enabling PCI PM
> +	 * substates.
> +	 */
>  	pci_set_power_state(pdev, PCI_D0);
>  	pci_enable_link_state_locked(pdev, PCIE_LINK_STATE_ALL);
>  
> -- 
> 2.41.0
>
Manivannan Sadhasivam Nov. 17, 2023, 10:35 a.m. UTC | #4
On Tue, Nov 14, 2023 at 02:55:52PM +0100, Johan Hovold wrote:
> Replace the current 'sem' parameter to the __pci_disable_link_state()
> helper with a more descriptive 'locked' parameter, which indicates
> whether a pci_bus_sem read lock is already held.
> 
> Signed-off-by: Johan Hovold <johan+linaro@kernel.org>
> ---

Reviewed-by: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>

- Mani

>  drivers/pci/pcie/aspm.c | 10 +++++-----
>  1 file changed, 5 insertions(+), 5 deletions(-)
> 
> diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c
> index 8cf8cc2d6bba..19b7256d2dc9 100644
> --- a/drivers/pci/pcie/aspm.c
> +++ b/drivers/pci/pcie/aspm.c
> @@ -1041,7 +1041,7 @@ static struct pcie_link_state *pcie_aspm_get_link(struct pci_dev *pdev)
>  	return bridge->link_state;
>  }
>  
> -static int __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem)
> +static int __pci_disable_link_state(struct pci_dev *pdev, int state, bool locked)
>  {
>  	struct pcie_link_state *link = pcie_aspm_get_link(pdev);
>  
> @@ -1060,7 +1060,7 @@ static int __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem)
>  		return -EPERM;
>  	}
>  
> -	if (sem)
> +	if (!locked)
>  		down_read(&pci_bus_sem);
>  	mutex_lock(&aspm_lock);
>  	if (state & PCIE_LINK_STATE_L0S)
> @@ -1082,7 +1082,7 @@ static int __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem)
>  		link->clkpm_disable = 1;
>  	pcie_set_clkpm(link, policy_to_clkpm_state(link));
>  	mutex_unlock(&aspm_lock);
> -	if (sem)
> +	if (!locked)
>  		up_read(&pci_bus_sem);
>  
>  	return 0;
> @@ -1090,7 +1090,7 @@ static int __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem)
>  
>  int pci_disable_link_state_locked(struct pci_dev *pdev, int state)
>  {
> -	return __pci_disable_link_state(pdev, state, false);
> +	return __pci_disable_link_state(pdev, state, true);
>  }
>  EXPORT_SYMBOL(pci_disable_link_state_locked);
>  
> @@ -1105,7 +1105,7 @@ EXPORT_SYMBOL(pci_disable_link_state_locked);
>   */
>  int pci_disable_link_state(struct pci_dev *pdev, int state)
>  {
> -	return __pci_disable_link_state(pdev, state, true);
> +	return __pci_disable_link_state(pdev, state, false);
>  }
>  EXPORT_SYMBOL(pci_disable_link_state);
>  
> -- 
> 2.41.0
>
Johan Hovold Nov. 17, 2023, 10:48 a.m. UTC | #5
On Fri, Nov 17, 2023 at 04:02:27PM +0530, Manivannan Sadhasivam wrote:
> On Tue, Nov 14, 2023 at 02:55:51PM +0100, Johan Hovold wrote:
> > Break up the newly added ASPM comment so that it fits within the soft 80
> > character limit and becomes more readable.
> > 
> > Signed-off-by: Johan Hovold <johan+linaro@kernel.org>
> 
> I think we discussed (80column soft limit for comments) in the past, but I don't
> think breaking here makes the comment more readable.

The coding style clearly states:

	The preferred limit on the length of a single line is 80 columns.

	Statements longer than 80 columns should be broken into sensible chunks,
	unless exceeding 80 columns significantly increases readability and does
	not hide information.

Going beyond 80 chars may sometimes be warranted for code, but the
exception is not intended for comments.

Johan
Manivannan Sadhasivam Nov. 17, 2023, 10:54 a.m. UTC | #6
On Fri, Nov 17, 2023 at 11:48:10AM +0100, Johan Hovold wrote:
> On Fri, Nov 17, 2023 at 04:02:27PM +0530, Manivannan Sadhasivam wrote:
> > On Tue, Nov 14, 2023 at 02:55:51PM +0100, Johan Hovold wrote:
> > > Break up the newly added ASPM comment so that it fits within the soft 80
> > > character limit and becomes more readable.
> > > 
> > > Signed-off-by: Johan Hovold <johan+linaro@kernel.org>
> > 
> > I think we discussed (80column soft limit for comments) in the past, but I don't
> > think breaking here makes the comment more readable.
> 
> The coding style clearly states:
> 
> 	The preferred limit on the length of a single line is 80 columns.
> 
> 	Statements longer than 80 columns should be broken into sensible chunks,
> 	unless exceeding 80 columns significantly increases readability and does
> 	not hide information.
> 
> Going beyond 80 chars may sometimes be warranted for code, but the
> exception is not intended for comments.
> 

Breaking the comment here is indeed making it hard to read. It's just one word
that needs to be broken if we go by 80 column limit and I won't prefer that,
sorry!

- Mani

> Johan
Johan Hovold Nov. 17, 2023, 11 a.m. UTC | #7
On Fri, Nov 17, 2023 at 04:24:04PM +0530, Manivannan Sadhasivam wrote:
> On Fri, Nov 17, 2023 at 11:48:10AM +0100, Johan Hovold wrote:
> > On Fri, Nov 17, 2023 at 04:02:27PM +0530, Manivannan Sadhasivam wrote:
> > > On Tue, Nov 14, 2023 at 02:55:51PM +0100, Johan Hovold wrote:
> > > > Break up the newly added ASPM comment so that it fits within the soft 80
> > > > character limit and becomes more readable.
> > > > 
> > > > Signed-off-by: Johan Hovold <johan+linaro@kernel.org>
> > > 
> > > I think we discussed (80column soft limit for comments) in the past, but I don't
> > > think breaking here makes the comment more readable.
> > 
> > The coding style clearly states:
> > 
> > 	The preferred limit on the length of a single line is 80 columns.
> > 
> > 	Statements longer than 80 columns should be broken into sensible chunks,
> > 	unless exceeding 80 columns significantly increases readability and does
> > 	not hide information.
> > 
> > Going beyond 80 chars may sometimes be warranted for code, but the
> > exception is not intended for comments.
> 
> Breaking the comment here is indeed making it hard to read. It's just one word
> that needs to be broken if we go by 80 column limit and I won't prefer that,
> sorry!

Please read the above quote again, it is as clear as it gets. 80 chars
is the preferred limit unless (for code) exceeding it *significantly*
increases readability, which clearly isn't the case here (even if this
exception applied to comments).

I really don't understand why you keep insisting on this. Just fix your
editor.

Johan
Manivannan Sadhasivam Nov. 17, 2023, 11:23 a.m. UTC | #8
On Fri, Nov 17, 2023 at 12:00:44PM +0100, Johan Hovold wrote:
> On Fri, Nov 17, 2023 at 04:24:04PM +0530, Manivannan Sadhasivam wrote:
> > On Fri, Nov 17, 2023 at 11:48:10AM +0100, Johan Hovold wrote:
> > > On Fri, Nov 17, 2023 at 04:02:27PM +0530, Manivannan Sadhasivam wrote:
> > > > On Tue, Nov 14, 2023 at 02:55:51PM +0100, Johan Hovold wrote:
> > > > > Break up the newly added ASPM comment so that it fits within the soft 80
> > > > > character limit and becomes more readable.
> > > > > 
> > > > > Signed-off-by: Johan Hovold <johan+linaro@kernel.org>
> > > > 
> > > > I think we discussed (80column soft limit for comments) in the past, but I don't
> > > > think breaking here makes the comment more readable.
> > > 
> > > The coding style clearly states:
> > > 
> > > 	The preferred limit on the length of a single line is 80 columns.
> > > 
> > > 	Statements longer than 80 columns should be broken into sensible chunks,
> > > 	unless exceeding 80 columns significantly increases readability and does
> > > 	not hide information.
> > > 
> > > Going beyond 80 chars may sometimes be warranted for code, but the
> > > exception is not intended for comments.
> > 
> > Breaking the comment here is indeed making it hard to read. It's just one word
> > that needs to be broken if we go by 80 column limit and I won't prefer that,
> > sorry!
> 
> Please read the above quote again, it is as clear as it gets. 80 chars
> is the preferred limit unless (for code) exceeding it *significantly*

Where does it say "code" in the Documentation? As I read it, the doc weighs both
code and comment as "statement".

And how on the world that breaking a single word to the next line improves
readability? I fail to get it :/

> increases readability, which clearly isn't the case here (even if this
> exception applied to comments).
> 
> I really don't understand why you keep insisting on this. Just fix your
> editor.
> 

May you should fix yours to extend the limit to 100?

But I do not want to get into a spat here. Checkpatch, the tool supposed to
check for the kernel coding style is not complaining and I do not want a patch
that _fixes_ a coding style that is not an issue.

And I do not want to argue more on this. If the PCI maintainers are comfortable
with this patch, they can apply it, but I'm not.

- Mani

> Johan
Johan Hovold Nov. 17, 2023, 2:03 p.m. UTC | #9
On Fri, Nov 17, 2023 at 04:53:52PM +0530, Manivannan Sadhasivam wrote:
> On Fri, Nov 17, 2023 at 12:00:44PM +0100, Johan Hovold wrote:
> > On Fri, Nov 17, 2023 at 04:24:04PM +0530, Manivannan Sadhasivam wrote:
> > > On Fri, Nov 17, 2023 at 11:48:10AM +0100, Johan Hovold wrote:

> > > > The coding style clearly states:
> > > > 
> > > > 	The preferred limit on the length of a single line is 80 columns.
> > > > 
> > > > 	Statements longer than 80 columns should be broken into sensible chunks,
> > > > 	unless exceeding 80 columns significantly increases readability and does
> > > > 	not hide information.
> > > > 
> > > > Going beyond 80 chars may sometimes be warranted for code, but the
> > > > exception is not intended for comments.
> > > 
> > > Breaking the comment here is indeed making it hard to read. It's just one word
> > > that needs to be broken if we go by 80 column limit and I won't prefer that,
> > > sorry!
> > 
> > Please read the above quote again, it is as clear as it gets. 80 chars
> > is the preferred limit unless (for code) exceeding it *significantly*
> 
> Where does it say "code" in the Documentation? As I read it, the doc weighs both
> code and comment as "statement".

No, comments are not statements (in C).

You'd also never even consider interpreting it that way if you knew
where that exception comes from (namely that people break long
*statements* just to fit under 80 chars, thereby sometimes making the
*code* unnecessarily hard to read).

> And how on the world that breaking a single word to the next line improves
> readability? I fail to get it :/

You got it backwards; you should only go *beyond* 80 chars if it
"significantly increases readability".

But again, this does NOT apply to comments in the first place.

> > increases readability, which clearly isn't the case here (even if this
> > exception applied to comments).
> > 
> > I really don't understand why you keep insisting on this. Just fix your
> > editor.

> But I do not want to get into a spat here. Checkpatch, the tool supposed to
> check for the kernel coding style is not complaining and I do not want a patch
> that _fixes_ a coding style that is not an issue.

Checkpatch is just a tool, not the standard, and knowing when it is ok
to break the 80 column rule for code requires human judgement.

Johan
David E. Box Nov. 18, 2023, midnight UTC | #10
On Tue, 2023-11-14 at 14:55 +0100, Johan Hovold wrote:
> Add a helper for enabling link states that can be used in contexts where
> a pci_bus_sem read lock is already held (e.g. from pci_walk_bus()).
> 
> This helper will be used to fix a couple of potential deadlocks where
> the current helper is called with the lock already held, hence the CC
> stable tag.

Thanks for sending a patch to address this. It had fallen off my radar.

> 
> Fixes: f492edb40b54 ("PCI: vmd: Add quirk to configure PCIe ASPM and LTR")
> Cc: stable@vger.kernel.org   # 6.3
> Cc: Michael Bottini <michael.a.bottini@linux.intel.com>
> Cc: David E. Box <david.e.box@linux.intel.com>
> Signed-off-by: Johan Hovold <johan+linaro@kernel.org>
> ---
>  drivers/pci/pcie/aspm.c | 53 +++++++++++++++++++++++++++++++----------
>  include/linux/pci.h     |  3 +++
>  2 files changed, 43 insertions(+), 13 deletions(-)
> 
> diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c
> index 50b04ae5c394..8cf8cc2d6bba 100644
> --- a/drivers/pci/pcie/aspm.c
> +++ b/drivers/pci/pcie/aspm.c
> @@ -1109,17 +1109,7 @@ int pci_disable_link_state(struct pci_dev *pdev, int
> state)
>  }
>  EXPORT_SYMBOL(pci_disable_link_state);
>  
> -/**
> - * pci_enable_link_state - Clear and set the default device link state so
> that
> - * the link may be allowed to enter the specified states. Note that if the
> - * BIOS didn't grant ASPM control to the OS, this does nothing because we
> can't
> - * touch the LNKCTL register. Also note that this does not enable states
> - * disabled by pci_disable_link_state(). Return 0 or a negative errno.
> - *
> - * @pdev: PCI device
> - * @state: Mask of ASPM link states to enable
> - */
> -int pci_enable_link_state(struct pci_dev *pdev, int state)
> +static int __pci_enable_link_state(struct pci_dev *pdev, int state, bool
> locked)
>  {
>         struct pcie_link_state *link = pcie_aspm_get_link(pdev);
>  
> @@ -1136,7 +1126,8 @@ int pci_enable_link_state(struct pci_dev *pdev, int
> state)
>                 return -EPERM;
>         }
>  
> -       down_read(&pci_bus_sem);
> +       if (!locked)
> +               down_read(&pci_bus_sem);

This solution is similar to the original one I proposed [1]. It just creates a
separate locked function rather than using a flag. While the API is consistent
with pci_disable_link_state_locked(), its usage is not. The vmd driver calls
pci_enable_link_state() from pci_walk_bus() which was problematic for Bjorn [2].
I owed an attempt to implement this as a fixup. However, now
qcom_pcie_enabled_aspm() is also using it from pci_walk_bus(). The fixup could
make sense for VMD since there it's fixing missing BIOS settings, but I'm not
sure about qcom. Bjorn, do you still see issues with the use in pci_bus_walk()?

[1]
https://lore.kernel.org/lkml/20230321233849.3408339-1-david.e.box@linux.intel.com/

[2] https://lore.kernel.org/lkml/20230322205702.GA2493123@bhelgaas/

David

>         mutex_lock(&aspm_lock);
>         link->aspm_default = 0;
>         if (state & PCIE_LINK_STATE_L0S)
> @@ -1157,12 +1148,48 @@ int pci_enable_link_state(struct pci_dev *pdev, int
> state)
>         link->clkpm_default = (state & PCIE_LINK_STATE_CLKPM) ? 1 : 0;
>         pcie_set_clkpm(link, policy_to_clkpm_state(link));
>         mutex_unlock(&aspm_lock);
> -       up_read(&pci_bus_sem);
> +       if (!locked)
> +               up_read(&pci_bus_sem);
>  
>         return 0;
>  }
> +
> +/**
> + * pci_enable_link_state - Clear and set the default device link state so
> that
> + * the link may be allowed to enter the specified states. Note that if the
> + * BIOS didn't grant ASPM control to the OS, this does nothing because we
> can't
> + * touch the LNKCTL register. Also note that this does not enable states
> + * disabled by pci_disable_link_state(). Return 0 or a negative errno.
> + *
> + * @pdev: PCI device
> + * @state: Mask of ASPM link states to enable
> + */
> +int pci_enable_link_state(struct pci_dev *pdev, int state)
> +{
> +       return __pci_enable_link_state(pdev, state, false);
> +}
>  EXPORT_SYMBOL(pci_enable_link_state);
>  
> +/**
> + * pci_enable_link_state - Clear and set the default device link state so
> that
> + * the link may be allowed to enter the specified states. Note that if the
> + * BIOS didn't grant ASPM control to the OS, this does nothing because we
> can't
> + * touch the LNKCTL register. Also note that this does not enable states
> + * disabled by pci_disable_link_state(). Return 0 or a negative errno.
> + *
> + * @pdev: PCI device
> + * @state: Mask of ASPM link states to enable
> + *
> + * Context: Caller holds pci_bus_sem read lock.
> + */
> +int pci_enable_link_state_locked(struct pci_dev *pdev, int state)
> +{
> +       lockdep_assert_held_read(&pci_bus_sem);
> +
> +       return __pci_enable_link_state(pdev, state, true);
> +}
> +EXPORT_SYMBOL(pci_enable_link_state_locked);
> +
>  static int pcie_aspm_set_policy(const char *val,
>                                 const struct kernel_param *kp)
>  {
> diff --git a/include/linux/pci.h b/include/linux/pci.h
> index 60ca768bc867..dea043bc1e38 100644
> --- a/include/linux/pci.h
> +++ b/include/linux/pci.h
> @@ -1829,6 +1829,7 @@ extern bool pcie_ports_native;
>  int pci_disable_link_state(struct pci_dev *pdev, int state);
>  int pci_disable_link_state_locked(struct pci_dev *pdev, int state);
>  int pci_enable_link_state(struct pci_dev *pdev, int state);
> +int pci_enable_link_state_locked(struct pci_dev *pdev, int state);
>  void pcie_no_aspm(void);
>  bool pcie_aspm_support_enabled(void);
>  bool pcie_aspm_enabled(struct pci_dev *pdev);
> @@ -1839,6 +1840,8 @@ static inline int pci_disable_link_state_locked(struct
> pci_dev *pdev, int state)
>  { return 0; }
>  static inline int pci_enable_link_state(struct pci_dev *pdev, int state)
>  { return 0; }
> +static inline int pci_enable_link_state_locked(struct pci_dev *pdev, int
> state)
> +{ return 0; }
>  static inline void pcie_no_aspm(void) { }
>  static inline bool pcie_aspm_support_enabled(void) { return false; }
>  static inline bool pcie_aspm_enabled(struct pci_dev *pdev) { return false; }