LKML Archive on lore.kernel.org
help / color / mirror / Atom feed
* [PATCH 4/4] IB: introducing MTHCA_MR_DMABARRIER
@ 2008-01-30  5:54 akepner
  2008-01-31  3:00 ` Roland Dreier
  0 siblings, 1 reply; 3+ messages in thread
From: akepner @ 2008-01-30  5:54 UTC (permalink / raw)
  To: Tony Luck, Grant Grundler, Jesse Barnes, Jes Sorensen,
	Randy Dunlap, Roland Dreier, James Bottomley, David Miller,
	Muli Ben-Yehuda
  Cc: linux-kernel


Add MTHCA_MR_DMABARRIER to mthca's API, increment ABI version, 
and make use of MTHCA_MR_DMABARRIER when mapping a user-allocated 
memory region with ib_umem_get().


Signed-off-by: Arthur Kepner <akepner@sgi.com>

--

 drivers/infiniband/core/umem.c               |   15 +++++++++---
 drivers/infiniband/hw/mthca/mthca_provider.c |    7 ++++-
 drivers/infiniband/hw/mthca/mthca_user.h     |   10 +++++++-
 include/rdma/ib_verbs.h                      |   33 +++++++++++++++++++++++++++
 4 files changed, 59 insertions(+), 6 deletions(-)

diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c
index 5b00408..57b5ce9 100644
--- a/drivers/infiniband/core/umem.c
+++ b/drivers/infiniband/core/umem.c
@@ -38,6 +38,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/sched.h>
 #include <linux/hugetlb.h>
+#include <linux/dma-attrs.h>
 
 #include "uverbs.h"
 
@@ -72,6 +73,8 @@ static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int d
  * @addr: userspace virtual address to start at
  * @size: length of region to pin
  * @access: IB_ACCESS_xxx flags for memory being pinned
+ * @dmabarrier: set a "dma barrier" so that in-flight DMA is 
+ *  flushed when the memory region is written to
  */
 struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
 			    size_t size, int access, int dmabarrier)
@@ -87,6 +90,9 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
 	int ret;
 	int off;
 	int i;
+	struct dma_attrs attrs;
+
+	dma_set_attr(&attrs, dmabarrier ? DMA_ATTR_BARRIER : 0);
 
 	if (!can_do_mlock())
 		return ERR_PTR(-EPERM);
@@ -174,10 +180,11 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
 				sg_set_page(&chunk->page_list[i], page_list[i + off], PAGE_SIZE, 0);
 			}
 
-			chunk->nmap = ib_dma_map_sg(context->device,
-						    &chunk->page_list[0],
-						    chunk->nents,
-						    DMA_BIDIRECTIONAL);
+			chunk->nmap = ib_dma_map_sg_attrs(context->device,
+							  &chunk->page_list[0],
+							  chunk->nents,
+							  DMA_BIDIRECTIONAL, 
+							  &attrs);
 			if (chunk->nmap <= 0) {
 				for (i = 0; i < chunk->nents; ++i)
 					put_page(sg_page(&chunk->page_list[i]));
diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c
index 704d8ef..e837cc9 100644
--- a/drivers/infiniband/hw/mthca/mthca_provider.c
+++ b/drivers/infiniband/hw/mthca/mthca_provider.c
@@ -1017,17 +1017,22 @@ static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 	struct mthca_dev *dev = to_mdev(pd->device);
 	struct ib_umem_chunk *chunk;
 	struct mthca_mr *mr;
+	struct mthca_reg_mr ucmd;
 	u64 *pages;
 	int shift, n, len;
 	int i, j, k;
 	int err = 0;
 	int write_mtt_size;
 
+	if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd))
+		return ERR_PTR(-EFAULT);
+
 	mr = kmalloc(sizeof *mr, GFP_KERNEL);
 	if (!mr)
 		return ERR_PTR(-ENOMEM);
 
-	mr->umem = ib_umem_get(pd->uobject->context, start, length, acc, 0);
+	mr->umem = ib_umem_get(pd->uobject->context, start, length, acc, 
+			       ucmd.mr_attrs & MTHCA_MR_DMABARRIER);
 
 	if (IS_ERR(mr->umem)) {
 		err = PTR_ERR(mr->umem);
diff --git a/drivers/infiniband/hw/mthca/mthca_user.h b/drivers/infiniband/hw/mthca/mthca_user.h
index 02cc0a7..701a430 100644
--- a/drivers/infiniband/hw/mthca/mthca_user.h
+++ b/drivers/infiniband/hw/mthca/mthca_user.h
@@ -41,7 +41,7 @@
  * Increment this value if any changes that break userspace ABI
  * compatibility are made.
  */
-#define MTHCA_UVERBS_ABI_VERSION	1
+#define MTHCA_UVERBS_ABI_VERSION	2
 
 /*
  * Make sure that all structs defined in this file remain laid out so
@@ -61,6 +61,14 @@ struct mthca_alloc_pd_resp {
 	__u32 reserved;
 };
 
+struct mthca_reg_mr {
+	__u32 mr_attrs;
+#define MTHCA_MR_DMABARRIER 0x1  /* set a dma barrier in order to flush 
+				  * in-flight DMA on a write to memory 
+				  * region */
+	__u32 reserved;
+};
+
 struct mthca_create_cq {
 	__u32 lkey;
 	__u32 pdn;
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 11f3960..ac869e2 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -1507,6 +1507,24 @@ static inline void ib_dma_unmap_single(struct ib_device *dev,
 		dma_unmap_single(dev->dma_device, addr, size, direction);
 }
 
+static inline u64 ib_dma_map_single_attrs(struct ib_device *dev,
+					  void *cpu_addr, size_t size,
+					  enum dma_data_direction direction,
+					  struct dma_attrs *attrs)
+{
+	return dma_map_single_attrs(dev->dma_device, cpu_addr, size, 
+				    direction, attrs);
+}
+
+static inline void ib_dma_unmap_single_attrs(struct ib_device *dev,
+					     u64 addr, size_t size,
+					     enum dma_data_direction direction, 
+					     struct dma_attrs *attrs)
+{
+	return dma_unmap_single_attrs(dev->dma_device, addr, size, 
+				      direction, attrs);
+}
+
 /**
  * ib_dma_map_page - Map a physical page to DMA address
  * @dev: The device for which the dma_addr is to be created
@@ -1576,6 +1594,21 @@ static inline void ib_dma_unmap_sg(struct ib_device *dev,
 		dma_unmap_sg(dev->dma_device, sg, nents, direction);
 }
 
+static inline int ib_dma_map_sg_attrs(struct ib_device *dev,
+				      struct scatterlist *sg, int nents,
+				      enum dma_data_direction direction,
+				      struct dma_attrs *attrs)
+{
+	return dma_map_sg_attrs(dev->dma_device, sg, nents, direction, attrs);
+}
+
+static inline void ib_dma_unmap_sg_attrs(struct ib_device *dev,
+					 struct scatterlist *sg, int nents,
+					 enum dma_data_direction direction, 
+					 struct dma_attrs *attrs)
+{
+	dma_unmap_sg_attrs(dev->dma_device, sg, nents, direction, attrs);
+}
 /**
  * ib_sg_dma_address - Return the DMA address from a scatter/gather entry
  * @dev: The device for which the DMA addresses were created


^ permalink raw reply	[flat|nested] 3+ messages in thread

* Re: [PATCH 4/4] IB: introducing MTHCA_MR_DMABARRIER
  2008-01-30  5:54 [PATCH 4/4] IB: introducing MTHCA_MR_DMABARRIER akepner
@ 2008-01-31  3:00 ` Roland Dreier
  2008-02-01  2:57   ` akepner
  0 siblings, 1 reply; 3+ messages in thread
From: Roland Dreier @ 2008-01-31  3:00 UTC (permalink / raw)
  To: akepner
  Cc: Tony Luck, Grant Grundler, Jesse Barnes, Jes Sorensen,
	Randy Dunlap, James Bottomley, David Miller, Muli Ben-Yehuda,
	linux-kernel

 > @@ -72,6 +73,8 @@ static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int d
 >   * @addr: userspace virtual address to start at
 >   * @size: length of region to pin
 >   * @access: IB_ACCESS_xxx flags for memory being pinned
 > + * @dmabarrier: set a "dma barrier" so that in-flight DMA is 
 > + *  flushed when the memory region is written to
 >   */

I see... it seems some of the previous patch snuck in here...

^ permalink raw reply	[flat|nested] 3+ messages in thread

* Re: [PATCH 4/4] IB: introducing MTHCA_MR_DMABARRIER
  2008-01-31  3:00 ` Roland Dreier
@ 2008-02-01  2:57   ` akepner
  0 siblings, 0 replies; 3+ messages in thread
From: akepner @ 2008-02-01  2:57 UTC (permalink / raw)
  To: Roland Dreier
  Cc: Tony Luck, Grant Grundler, Jesse Barnes, Jes Sorensen,
	Randy Dunlap, James Bottomley, David Miller, Muli Ben-Yehuda,
	linux-kernel

On Wed, Jan 30, 2008 at 07:00:11PM -0800, Roland Dreier wrote:
>  > @@ -72,6 +73,8 @@ static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int d
>  >   * @addr: userspace virtual address to start at
>  >   * @size: length of region to pin
>  >   * @access: IB_ACCESS_xxx flags for memory being pinned
>  > + * @dmabarrier: set a "dma barrier" so that in-flight DMA is 
>  > + *  flushed when the memory region is written to
>  >   */
> 
> I see... it seems some of the previous patch snuck in here...

I'll break the next patchset up differently, and also include 
mlx4.

-- 
Arthur


^ permalink raw reply	[flat|nested] 3+ messages in thread

end of thread, other threads:[~2008-02-01  2:59 UTC | newest]

Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2008-01-30  5:54 [PATCH 4/4] IB: introducing MTHCA_MR_DMABARRIER akepner
2008-01-31  3:00 ` Roland Dreier
2008-02-01  2:57   ` akepner

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).