LKML Archive on lore.kernel.org
 help / color / Atom feed
From: "Alastair D'Silva" <alastair@d-silva.org>
To: alastair@d-silva.org
Cc: "Aneesh Kumar K . V" <aneesh.kumar@linux.ibm.com>,
	"Oliver O\'Halloran" <oohall@gmail.com>,
	"Benjamin Herrenschmidt" <benh@kernel.crashing.org>,
	"Paul Mackerras" <paulus@samba.org>,
	"Michael Ellerman" <mpe@ellerman.id.au>,
	"Frederic Barrat" <fbarrat@linux.ibm.com>,
	"Andrew Donnellan" <ajd@linux.ibm.com>,
	"Arnd Bergmann" <arnd@arndb.de>,
	"Greg Kroah-Hartman" <gregkh@linuxfoundation.org>,
	"Dan Williams" <dan.j.williams@intel.com>,
	"Vishal Verma" <vishal.l.verma@intel.com>,
	"Dave Jiang" <dave.jiang@intel.com>,
	"Ira Weiny" <ira.weiny@intel.com>,
	"Andrew Morton" <akpm@linux-foundation.org>,
	"Mauro Carvalho Chehab" <mchehab+samsung@kernel.org>,
	"David S. Miller" <davem@davemloft.net>,
	"Rob Herring" <robh@kernel.org>,
	"Anton Blanchard" <anton@ozlabs.org>,
	"Krzysztof Kozlowski" <krzk@kernel.org>,
	"Mahesh Salgaonkar" <mahesh@linux.vnet.ibm.com>,
	"Madhavan Srinivasan" <maddy@linux.vnet.ibm.com>,
	"Cédric Le Goater" <clg@kaod.org>,
	"Anju T Sudhakar" <anju@linux.vnet.ibm.com>,
	"Hari Bathini" <hbathini@linux.ibm.com>,
	"Thomas Gleixner" <tglx@linutronix.de>,
	"Greg Kurz" <groug@kaod.org>,
	"Nicholas Piggin" <npiggin@gmail.com>,
	"Masahiro Yamada" <yamada.masahiro@socionext.com>,
	"Alexey Kardashevskiy" <aik@ozlabs.ru>,
	linux-kernel@vger.kernel.org, linuxppc-dev@lists.ozlabs.org,
	linux-nvdimm@lists.01.org, linux-mm@kvack.org
Subject: [PATCH v4 06/25] ocxl: Tally up the LPC memory on a link & allow it to be mapped
Date: Fri, 27 Mar 2020 18:11:43 +1100
Message-ID: <20200327071202.2159885-7-alastair@d-silva.org> (raw)
In-Reply-To: <20200327071202.2159885-1-alastair@d-silva.org>

OpenCAPI LPC memory is allocated per link, but each link supports
multiple AFUs, and each AFU can have LPC memory assigned to it.

This patch tallys the memory for all AFUs on a link, allowing it
to be mapped in a single operation after the AFUs have been
enumerated.

Signed-off-by: Alastair D'Silva <alastair@d-silva.org>
---
 drivers/misc/ocxl/core.c          | 10 ++++++
 drivers/misc/ocxl/link.c          | 60 +++++++++++++++++++++++++++++++
 drivers/misc/ocxl/ocxl_internal.h | 33 +++++++++++++++++
 3 files changed, 103 insertions(+)

diff --git a/drivers/misc/ocxl/core.c b/drivers/misc/ocxl/core.c
index b7a09b21ab36..2531c6cf19a0 100644
--- a/drivers/misc/ocxl/core.c
+++ b/drivers/misc/ocxl/core.c
@@ -230,8 +230,18 @@ static int configure_afu(struct ocxl_afu *afu, u8 afu_idx, struct pci_dev *dev)
 	if (rc)
 		goto err_free_pasid;
 
+	if (afu->config.lpc_mem_size || afu->config.special_purpose_mem_size) {
+		rc = ocxl_link_add_lpc_mem(afu->fn->link, afu->config.lpc_mem_offset,
+					   afu->config.lpc_mem_size +
+					   afu->config.special_purpose_mem_size);
+		if (rc)
+			goto err_free_mmio;
+	}
+
 	return 0;
 
+err_free_mmio:
+	unmap_mmio_areas(afu);
 err_free_pasid:
 	reclaim_afu_pasid(afu);
 err_free_actag:
diff --git a/drivers/misc/ocxl/link.c b/drivers/misc/ocxl/link.c
index 58d111afd9f6..af119d3ef79a 100644
--- a/drivers/misc/ocxl/link.c
+++ b/drivers/misc/ocxl/link.c
@@ -84,6 +84,11 @@ struct ocxl_link {
 	int dev;
 	atomic_t irq_available;
 	struct spa *spa;
+	struct mutex lpc_mem_lock; /* protects lpc_mem & lpc_mem_sz */
+	u64 lpc_mem_sz; /* Total amount of LPC memory presented on the link */
+	u64 lpc_mem;
+	int lpc_consumers;
+
 	void *platform_data;
 };
 static struct list_head links_list = LIST_HEAD_INIT(links_list);
@@ -396,6 +401,8 @@ static int alloc_link(struct pci_dev *dev, int PE_mask, struct ocxl_link **out_l
 	if (rc)
 		goto err_spa;
 
+	mutex_init(&link->lpc_mem_lock);
+
 	/* platform specific hook */
 	rc = pnv_ocxl_spa_setup(dev, link->spa->spa_mem, PE_mask,
 				&link->platform_data);
@@ -711,3 +718,56 @@ void ocxl_link_free_irq(void *link_handle, int hw_irq)
 	atomic_inc(&link->irq_available);
 }
 EXPORT_SYMBOL_GPL(ocxl_link_free_irq);
+
+int ocxl_link_add_lpc_mem(void *link_handle, u64 offset, u64 size)
+{
+	struct ocxl_link *link = (struct ocxl_link *)link_handle;
+
+	// Check for overflow
+	if (offset > (offset + size))
+		return -EINVAL;
+
+	mutex_lock(&link->lpc_mem_lock);
+	link->lpc_mem_sz = max(link->lpc_mem_sz, offset + size);
+
+	mutex_unlock(&link->lpc_mem_lock);
+
+	return 0;
+}
+
+u64 ocxl_link_lpc_map(void *link_handle, struct pci_dev *pdev)
+{
+	struct ocxl_link *link = (struct ocxl_link *)link_handle;
+
+	mutex_lock(&link->lpc_mem_lock);
+
+	if (!link->lpc_mem)
+		link->lpc_mem = pnv_ocxl_platform_lpc_setup(pdev, link->lpc_mem_sz);
+
+	if (link->lpc_mem)
+		link->lpc_consumers++;
+	mutex_unlock(&link->lpc_mem_lock);
+
+	return link->lpc_mem;
+}
+
+void ocxl_link_lpc_release(void *link_handle, struct pci_dev *pdev)
+{
+	struct ocxl_link *link = (struct ocxl_link *)link_handle;
+
+	mutex_lock(&link->lpc_mem_lock);
+
+	if (!link->lpc_mem) {
+		mutex_unlock(&link->lpc_mem_lock);
+		return;
+	}
+
+	WARN_ON(--link->lpc_consumers < 0);
+
+	if (link->lpc_consumers == 0) {
+		pnv_ocxl_platform_lpc_release(pdev);
+		link->lpc_mem = 0;
+	}
+
+	mutex_unlock(&link->lpc_mem_lock);
+}
diff --git a/drivers/misc/ocxl/ocxl_internal.h b/drivers/misc/ocxl/ocxl_internal.h
index 198e4e4bc51d..2d7575225bd7 100644
--- a/drivers/misc/ocxl/ocxl_internal.h
+++ b/drivers/misc/ocxl/ocxl_internal.h
@@ -142,4 +142,37 @@ int ocxl_irq_offset_to_id(struct ocxl_context *ctx, u64 offset);
 u64 ocxl_irq_id_to_offset(struct ocxl_context *ctx, int irq_id);
 void ocxl_afu_irq_free_all(struct ocxl_context *ctx);
 
+/**
+ * ocxl_link_add_lpc_mem() - Increment the amount of memory required by an OpenCAPI link
+ *
+ * @link_handle: The OpenCAPI link handle
+ * @offset: The offset of the memory to add
+ * @size: The number of bytes to increment memory on the link by
+ *
+ * Returns 0 on success, -EINVAL on overflow
+ */
+int ocxl_link_add_lpc_mem(void *link_handle, u64 offset, u64 size);
+
+/**
+ * ocxl_link_lpc_map() - Map the LPC memory for an OpenCAPI device
+ * Since LPC memory belongs to a link, the whole LPC memory available
+ * on the link must be mapped in order to make it accessible to a device.
+ * @link_handle: The OpenCAPI link handle
+ * @pdev: A device that is on the link
+ *
+ * Returns the address of the mapped LPC memory, or 0 on error
+ */
+u64 ocxl_link_lpc_map(void *link_handle, struct pci_dev *pdev);
+
+/**
+ * ocxl_link_lpc_release() - Release the LPC memory device for an OpenCAPI device
+ *
+ * Offlines LPC memory on an OpenCAPI link for a device. If this is the
+ * last device on the link to release the memory, unmap it from the link.
+ *
+ * @link_handle: The OpenCAPI link handle
+ * @pdev: A device that is on the link
+ */
+void ocxl_link_lpc_release(void *link_handle, struct pci_dev *pdev);
+
 #endif /* _OCXL_INTERNAL_H_ */
-- 
2.24.1


  parent reply index

Thread overview: 60+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-03-27  7:11 [PATCH v4 00/25] Add support for OpenCAPI Persistent Memory devices Alastair D'Silva
2020-03-27  7:11 ` [PATCH v4 01/25] powerpc/powernv: Add OPAL calls for LPC memory alloc/release Alastair D'Silva
2020-04-01  8:48   ` Dan Williams
2020-04-01 22:51     ` Alastair D'Silva
2020-03-27  7:11 ` [PATCH v4 02/25] mm/memory_hotplug: Allow check_hotplug_memory_addressable to be called from drivers Alastair D'Silva
2020-04-01  8:48   ` Dan Williams
2020-04-02  4:33     ` Alastair D'Silva
2020-03-27  7:11 ` [PATCH v4 03/25] powerpc/powernv: Map & release OpenCAPI LPC memory Alastair D'Silva
2020-04-01  8:48   ` Dan Williams
2020-04-02  4:36     ` Alastair D'Silva
2020-04-02 10:41     ` Benjamin Herrenschmidt
2020-04-03  4:27       ` Michael Ellerman
2020-03-27  7:11 ` [PATCH v4 04/25] ocxl: Remove unnecessary externs Alastair D'Silva
2020-04-01  8:48   ` Dan Williams
2020-03-27  7:11 ` [PATCH v4 05/25] ocxl: Address kernel doc errors & warnings Alastair D'Silva
2020-04-01  8:49   ` Dan Williams
2020-03-27  7:11 ` Alastair D'Silva [this message]
2020-04-01  8:48   ` [PATCH v4 06/25] ocxl: Tally up the LPC memory on a link & allow it to be mapped Dan Williams
2020-04-02  6:21     ` Andrew Donnellan
2020-03-27  7:11 ` [PATCH v4 07/25] ocxl: Add functions to map/unmap LPC memory Alastair D'Silva
2020-04-01  8:49   ` Dan Williams
2020-04-03  3:50     ` Alastair D'Silva
2020-03-27  7:11 ` [PATCH v4 08/25] ocxl: Emit a log message showing how much LPC memory was detected Alastair D'Silva
2020-04-01  8:49   ` Dan Williams
2020-04-02  1:29     ` Joe Perches
2020-04-03  3:52     ` Alastair D'Silva
2020-03-27  7:11 ` [PATCH v4 09/25] ocxl: Save the device serial number in ocxl_fn Alastair D'Silva
2020-03-27  7:11 ` [PATCH v4 10/25] nvdimm: Add driver for OpenCAPI Persistent Memory Alastair D'Silva
2020-03-29  2:56   ` Matthew Wilcox
2020-03-29  2:59     ` Matthew Wilcox
2020-04-01  8:49   ` Dan Williams
2020-04-01 19:35     ` Dan Williams
2020-03-27  7:11 ` [PATCH v4 11/25] powerpc: Enable the OpenCAPI Persistent Memory driver for powernv_defconfig Alastair D'Silva
2020-04-01 20:26   ` Dan Williams
2020-03-27  7:11 ` [PATCH v4 12/25] nvdimm/ocxl: Add register addresses & status values to the header Alastair D'Silva
2020-04-01 20:27   ` Dan Williams
2020-03-27  7:11 ` [PATCH v4 13/25] nvdimm/ocxl: Read the capability registers & wait for device ready Alastair D'Silva
2020-04-02  0:20   ` Dan Williams
2020-03-27  7:11 ` [PATCH v4 14/25] nvdimm/ocxl: Add support for Admin commands Alastair D'Silva
2020-04-02  6:41   ` Dan Williams
2020-03-27  7:11 ` [PATCH v4 15/25] nvdimm/ocxl: Register a character device for userspace to interact with Alastair D'Silva
2020-04-02  0:27   ` Dan Williams
2020-03-27  7:11 ` [PATCH v4 16/25] nvdimm/ocxl: Implement the Read Error Log command Alastair D'Silva
2020-04-03  0:54   ` Dan Williams
2020-03-27  7:11 ` [PATCH v4 17/25] nvdimm/ocxl: Add controller dump IOCTLs Alastair D'Silva
2020-03-27  7:11 ` [PATCH v4 18/25] nvdimm/ocxl: Add an IOCTL to report controller statistics Alastair D'Silva
2020-03-27  7:11 ` [PATCH v4 19/25] nvdimm/ocxl: Forward events to userspace Alastair D'Silva
2020-04-02  2:08   ` Dan Williams
2020-03-27  7:11 ` [PATCH v4 20/25] nvdimm/ocxl: Add an IOCTL to request controller health & perf data Alastair D'Silva
2020-03-27  7:11 ` [PATCH v4 21/25] nvdimm/ocxl: Implement the heartbeat command Alastair D'Silva
2020-03-27  7:11 ` [PATCH v4 22/25] nvdimm/ocxl: Add debug IOCTLs Alastair D'Silva
2020-03-27  7:12 ` [PATCH v4 23/25] nvdimm/ocxl: Expose SMART data via ndctl Alastair D'Silva
2020-03-27  7:12 ` [PATCH v4 24/25] nvdimm/ocxl: Expose the serial number & firmware version in sysfs Alastair D'Silva
2020-03-27  7:12 ` [PATCH v4 25/25] MAINTAINERS: Add myself & nvdimm/ocxl to ocxl Alastair D'Silva
2020-04-01  8:47 ` [PATCH v4 00/25] Add support for OpenCAPI Persistent Memory devices Dan Williams
2020-04-01 22:44   ` Alastair D'Silva
2020-04-02  3:42     ` Michael Ellerman
2020-04-02  3:50       ` Oliver O'Halloran
2020-04-02 10:06         ` Michael Ellerman
2020-04-02 11:10           ` Greg Kurz

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20200327071202.2159885-7-alastair@d-silva.org \
    --to=alastair@d-silva.org \
    --cc=aik@ozlabs.ru \
    --cc=ajd@linux.ibm.com \
    --cc=akpm@linux-foundation.org \
    --cc=aneesh.kumar@linux.ibm.com \
    --cc=anju@linux.vnet.ibm.com \
    --cc=anton@ozlabs.org \
    --cc=arnd@arndb.de \
    --cc=benh@kernel.crashing.org \
    --cc=clg@kaod.org \
    --cc=dan.j.williams@intel.com \
    --cc=dave.jiang@intel.com \
    --cc=davem@davemloft.net \
    --cc=fbarrat@linux.ibm.com \
    --cc=gregkh@linuxfoundation.org \
    --cc=groug@kaod.org \
    --cc=hbathini@linux.ibm.com \
    --cc=ira.weiny@intel.com \
    --cc=krzk@kernel.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-mm@kvack.org \
    --cc=linux-nvdimm@lists.01.org \
    --cc=linuxppc-dev@lists.ozlabs.org \
    --cc=maddy@linux.vnet.ibm.com \
    --cc=mahesh@linux.vnet.ibm.com \
    --cc=mchehab+samsung@kernel.org \
    --cc=mpe@ellerman.id.au \
    --cc=npiggin@gmail.com \
    --cc=oohall@gmail.com \
    --cc=paulus@samba.org \
    --cc=robh@kernel.org \
    --cc=tglx@linutronix.de \
    --cc=vishal.l.verma@intel.com \
    --cc=yamada.masahiro@socionext.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

LKML Archive on lore.kernel.org

Archives are clonable:
	git clone --mirror https://lkml.kernel.org/lkml/0 lkml/git/0.git
	git clone --mirror https://lkml.kernel.org/lkml/1 lkml/git/1.git
	git clone --mirror https://lkml.kernel.org/lkml/2 lkml/git/2.git
	git clone --mirror https://lkml.kernel.org/lkml/3 lkml/git/3.git
	git clone --mirror https://lkml.kernel.org/lkml/4 lkml/git/4.git
	git clone --mirror https://lkml.kernel.org/lkml/5 lkml/git/5.git
	git clone --mirror https://lkml.kernel.org/lkml/6 lkml/git/6.git
	git clone --mirror https://lkml.kernel.org/lkml/7 lkml/git/7.git
	git clone --mirror https://lkml.kernel.org/lkml/8 lkml/git/8.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 lkml lkml/ https://lkml.kernel.org/lkml \
		linux-kernel@vger.kernel.org
	public-inbox-index lkml

Example config snippet for mirrors

Newsgroup available over NNTP:
	nntp://nntp.lore.kernel.org/org.kernel.vger.linux-kernel


AGPL code for this site: git clone https://public-inbox.org/public-inbox.git