[PATCH] [x86_32] Add io_map_atomic using fixmaps

[PATCH] [x86_32] Add io_map_atomic using fixmaps

[Keith Packard]

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain, Size: 7561 bytes --]

This steals the code used for CONFIG_HIGHMEM memory mappings except that
it's designed for dynamic io resource mapping. These fixmaps are available
even with CONFIG_HIGHMEM turned off.

Signed-off-by: Keith Packard <keithp@keithp.com>
---
 arch/x86/Kconfig                 |    4 ++
 arch/x86/include/asm/fixmap.h    |    4 ++
 arch/x86/include/asm/fixmap_32.h |    4 --
 arch/x86/include/asm/highmem.h   |    5 +--
 arch/x86/mm/Makefile             |    2 +-
 arch/x86/mm/init_32.c            |    3 +-
 arch/x86/mm/iomap_32.c           |   59 ++++++++++++++++++++++++++++++++++++++
 include/asm-x86/iomap.h          |   30 +++++++++++++++++++
 8 files changed, 100 insertions(+), 11 deletions(-)
 create mode 100644 arch/x86/mm/iomap_32.c
 create mode 100644 include/asm-x86/iomap.h

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 350bee1..a879393 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -1890,6 +1890,10 @@ config SYSVIPC_COMPAT
 endmenu
 
 
+config HAVE_ATOMIC_IOMAP
+	def_bool y
+	depends on X86_32
+
 source "net/Kconfig"
 
 source "drivers/Kconfig"
diff --git a/arch/x86/include/asm/fixmap.h b/arch/x86/include/asm/fixmap.h
index 8668a94..23696d4 100644
--- a/arch/x86/include/asm/fixmap.h
+++ b/arch/x86/include/asm/fixmap.h
@@ -9,6 +9,10 @@
 
 extern int fixmaps_set;
 
+extern pte_t *kmap_pte;
+extern pgprot_t kmap_prot;
+extern pte_t *pkmap_page_table;
+
 void __native_set_fixmap(enum fixed_addresses idx, pte_t pte);
 void native_set_fixmap(enum fixed_addresses idx,
 		       unsigned long phys, pgprot_t flags);
diff --git a/arch/x86/include/asm/fixmap_32.h b/arch/x86/include/asm/fixmap_32.h
index 09f29ab..c7115c1 100644
--- a/arch/x86/include/asm/fixmap_32.h
+++ b/arch/x86/include/asm/fixmap_32.h
@@ -28,10 +28,8 @@ extern unsigned long __FIXADDR_TOP;
 #include <asm/acpi.h>
 #include <asm/apicdef.h>
 #include <asm/page.h>
-#ifdef CONFIG_HIGHMEM
 #include <linux/threads.h>
 #include <asm/kmap_types.h>
-#endif
 
 /*
  * Here we define all the compile-time 'special' virtual
@@ -75,10 +73,8 @@ enum fixed_addresses {
 #ifdef CONFIG_X86_CYCLONE_TIMER
 	FIX_CYCLONE_TIMER, /*cyclone timer register*/
 #endif
-#ifdef CONFIG_HIGHMEM
 	FIX_KMAP_BEGIN,	/* reserved pte's for temporary kernel mappings */
 	FIX_KMAP_END = FIX_KMAP_BEGIN+(KM_TYPE_NR*NR_CPUS)-1,
-#endif
 #ifdef CONFIG_PCI_MMCONFIG
 	FIX_PCIE_MCFG,
 #endif
diff --git a/arch/x86/include/asm/highmem.h b/arch/x86/include/asm/highmem.h
index a3b3b7c..bf9276b 100644
--- a/arch/x86/include/asm/highmem.h
+++ b/arch/x86/include/asm/highmem.h
@@ -25,14 +25,11 @@
 #include <asm/kmap_types.h>
 #include <asm/tlbflush.h>
 #include <asm/paravirt.h>
+#include <asm/fixmap.h>
 
 /* declarations for highmem.c */
 extern unsigned long highstart_pfn, highend_pfn;
 
-extern pte_t *kmap_pte;
-extern pgprot_t kmap_prot;
-extern pte_t *pkmap_page_table;
-
 /*
  * Right now we initialize only a single pte table. It can be extended
  * easily, subsequent pte tables have to be allocated in one physical
diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile
index 59f89b4..fea4565 100644
--- a/arch/x86/mm/Makefile
+++ b/arch/x86/mm/Makefile
@@ -1,7 +1,7 @@
 obj-y	:=  init_$(BITS).o fault.o ioremap.o extable.o pageattr.o mmap.o \
 	    pat.o pgtable.o gup.o
 
-obj-$(CONFIG_X86_32)		+= pgtable_32.o
+obj-$(CONFIG_X86_32)		+= pgtable_32.o iomap_32.o
 
 obj-$(CONFIG_HUGETLB_PAGE)	+= hugetlbpage.o
 obj-$(CONFIG_X86_PTDUMP)	+= dump_pagetables.o
diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c
index 8396868..c483f42 100644
--- a/arch/x86/mm/init_32.c
+++ b/arch/x86/mm/init_32.c
@@ -334,7 +334,6 @@ int devmem_is_allowed(unsigned long pagenr)
 	return 0;
 }
 
-#ifdef CONFIG_HIGHMEM
 pte_t *kmap_pte;
 pgprot_t kmap_prot;
 
@@ -357,6 +356,7 @@ static void __init kmap_init(void)
 	kmap_prot = PAGE_KERNEL;
 }
 
+#ifdef CONFIG_HIGHMEM
 static void __init permanent_kmaps_init(pgd_t *pgd_base)
 {
 	unsigned long vaddr;
@@ -436,7 +436,6 @@ static void __init set_highmem_pages_init(void)
 #endif /* !CONFIG_NUMA */
 
 #else
-# define kmap_init()				do { } while (0)
 # define permanent_kmaps_init(pgd_base)		do { } while (0)
 # define set_highmem_pages_init()	do { } while (0)
 #endif /* CONFIG_HIGHMEM */
diff --git a/arch/x86/mm/iomap_32.c b/arch/x86/mm/iomap_32.c
new file mode 100644
index 0000000..d0151d8
--- /dev/null
+++ b/arch/x86/mm/iomap_32.c
@@ -0,0 +1,59 @@
+/*
+ * Copyright © 2008 Ingo Molnar
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.
+ */
+
+#include <asm/iomap.h>
+#include <linux/module.h>
+
+/* Map 'pfn' using fixed map 'type' and protections 'prot'
+ */
+void *
+iomap_atomic_prot_pfn(unsigned long pfn, enum km_type type, pgprot_t prot)
+{
+	enum fixed_addresses idx;
+	unsigned long vaddr;
+
+	pagefault_disable();
+
+	idx = type + KM_TYPE_NR*smp_processor_id();
+	vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
+	set_pte(kmap_pte-idx, pfn_pte(pfn, prot));
+	arch_flush_lazy_mmu_mode();
+
+	return (void*) vaddr;
+}
+EXPORT_SYMBOL_GPL(iomap_atomic_prot_pfn);
+
+void
+iounmap_atomic(void *kvaddr, enum km_type type)
+{
+	unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK;
+	enum fixed_addresses idx = type + KM_TYPE_NR*smp_processor_id();
+
+	/*
+	 * Force other mappings to Oops if they'll try to access this pte
+	 * without first remap it.  Keeping stale mappings around is a bad idea
+	 * also, in case the page changes cacheability attributes or becomes
+	 * a protected page in a hypervisor.
+	 */
+	if (vaddr == __fix_to_virt(FIX_KMAP_BEGIN+idx))
+		kpte_clear_flush(kmap_pte-idx, vaddr);
+
+	arch_flush_lazy_mmu_mode();
+	pagefault_enable();
+}
+EXPORT_SYMBOL_GPL(iounmap_atomic);
diff --git a/include/asm-x86/iomap.h b/include/asm-x86/iomap.h
new file mode 100644
index 0000000..c1f0628
--- /dev/null
+++ b/include/asm-x86/iomap.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright © 2008 Ingo Molnar
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.
+ */
+
+#include <linux/fs.h>
+#include <linux/mm.h>
+#include <linux/uaccess.h>
+#include <asm/cacheflush.h>
+#include <asm/pgtable.h>
+#include <asm/tlbflush.h>
+
+void *
+iomap_atomic_prot_pfn(unsigned long pfn, enum km_type type, pgprot_t prot);
+
+void
+iounmap_atomic(void *kvaddr, enum km_type type);
-- 
1.5.6.5

[PATCH] Add io-mapping functions to dynamically map large device apertures

[Keith Packard]

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain, Size: 7959 bytes --]

Graphics devices have large PCI apertures which would consume a significant
fraction of a 32-bit address space if mapped during driver initialization.
Using ioremap at runtime is impractical as it is too slow. This new set of
interfaces uses atomic mappings on 32-bit processors and a large static
mapping on 64-bit processors to provide reasonable 32-bit performance and
optimal 64-bit performance.

The current implementation sits atop the existing CONFIG_HIGHMEM kmap_atomic
mechanism for 32-bit processors when present. When absent, it just uses
ioremap, which remains horribly inefficient. Fixing non-HIGHMEM 32-bit
kernels to provide per-CPU mappings ala HIGHMEM would resolve that
performance issue.

This includes some editorial suggestions from Randy Dunlap for
Documentation/io-mapping.txt

Signed-off-by: Keith Packard <keithp@keithp.com>
---
 Documentation/io-mapping.txt |   82 +++++++++++++++++++++++++++
 include/linux/io-mapping.h   |  125 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 207 insertions(+), 0 deletions(-)
 create mode 100644 Documentation/io-mapping.txt
 create mode 100644 include/linux/io-mapping.h

diff --git a/Documentation/io-mapping.txt b/Documentation/io-mapping.txt
new file mode 100644
index 0000000..473e43b
--- /dev/null
+++ b/Documentation/io-mapping.txt
@@ -0,0 +1,82 @@
+The io_mapping functions in linux/io-mapping.h provide an abstraction for
+efficiently mapping small regions of an I/O device to the CPU. The initial
+usage is to support the large graphics aperture on 32-bit processors where
+ioremap_wc cannot be used to statically map the entire aperture to the CPU
+as it would consume too much of the kernel address space.
+
+A mapping object is created during driver initialization using
+
+	struct io_mapping *io_mapping_create_wc(unsigned long base,
+						unsigned long size)
+
+		'base' is the bus address of the region to be made
+		mappable, while 'size' indicates how large a mapping region to
+		enable. Both are in bytes.
+
+		This _wc variant provides a mapping which may only be used
+		with the io_mapping_map_atomic_wc or io_mapping_map_wc.
+
+With this mapping object, individual pages can be mapped either atomically
+or not, depending on the necessary scheduling environment. Of course, atomic
+maps are more efficient:
+
+	void *io_mapping_map_atomic_wc(struct io_mapping *mapping,
+				       unsigned long offset)
+
+		'offset' is the offset within the defined mapping region.
+		Accessing addresses beyond the region specified in the
+		creation function yields undefined results. Using an offset
+		which is not page aligned yields an undefined result. The
+		return value points to a single page in CPU address space.
+
+		This _wc variant returns a write-combining map to the
+		page and may only be used with mappings created by
+		io_mapping_create_wc
+
+		Note that the task may not sleep while holding this page
+		mapped.
+
+	void io_mapping_unmap_atomic(void *vaddr)
+
+		'vaddr' must be the the value returned by the last
+		io_mapping_map_atomic_wc call. This unmaps the specified
+		page and allows the task to sleep once again.
+
+If you need to sleep while holding the lock, you can use the non-atomic
+variant, although they may be significantly slower.
+
+	void *io_mapping_map_wc(struct io_mapping *mapping,
+				unsigned long offset)
+
+		This works like io_mapping_map_atomic_wc except it allows
+		the task to sleep while holding the page mapped.
+
+	void io_mapping_unmap(void *vaddr)
+
+		This works like io_mapping_unmap_atomic, except it is used
+		for pages mapped with io_mapping_map_wc.
+
+At driver close time, the io_mapping object must be freed:
+
+	void io_mapping_free(struct io_mapping *mapping)
+
+Current Implementation:
+
+The initial implementation of these functions uses existing mapping
+mechanisms and so provides only an abstraction layer and no new
+functionality.
+
+On 64-bit processors, io_mapping_create_wc calls ioremap_wc for the whole
+range, creating a permanent kernel-visible mapping to the resource. The
+map_atomic and map functions add the requested offset to the base of the
+virtual address returned by ioremap_wc.
+
+On 32-bit processors with HIGHMEM defined, io_mapping_map_atomic_wc uses
+kmap_atomic_pfn to map the specified page in an atomic fashion;
+kmap_atomic_pfn isn't really supposed to be used with device pages, but it
+provides an efficient mapping for this usage.
+
+On 32-bit processors without HIGHMEM defined, io_mapping_map_atomic_wc and
+io_mapping_map_wc both use ioremap_wc, a terribly inefficient function which
+performs an IPI to inform all processors about the new mapping. This results
+in a significant performance penalty.
diff --git a/include/linux/io-mapping.h b/include/linux/io-mapping.h
new file mode 100644
index 0000000..82df317
--- /dev/null
+++ b/include/linux/io-mapping.h
@@ -0,0 +1,125 @@
+/*
+ * Copyright © 2008 Keith Packard <keithp@keithp.com>
+ *
+ * This file is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#ifndef _LINUX_IO_MAPPING_H
+#define _LINUX_IO_MAPPING_H
+
+#include <linux/types.h>
+#include <asm/io.h>
+#include <asm/page.h>
+#include <asm/iomap.h>
+
+/*
+ * The io_mapping mechanism provides an abstraction for mapping
+ * individual pages from an io device to the CPU in an efficient fashion.
+ *
+ * See Documentation/io_mapping.txt
+ */
+
+/* this struct isn't actually defined anywhere */
+struct io_mapping;
+
+#ifdef CONFIG_HAVE_ATOMIC_IOMAP
+
+/*
+ * For small address space machines, mapping large objects
+ * into the kernel virtual space isn't practical. Where
+ * available, use fixmap support to dynamically map pages
+ * of the object at run time.
+ */
+
+static inline struct io_mapping *
+io_mapping_create_wc(unsigned long base, unsigned long size)
+{
+	return (struct io_mapping *) base;
+}
+
+static inline void
+io_mapping_free(struct io_mapping *mapping)
+{
+}
+
+/* Atomic map/unmap */
+static inline void *
+io_mapping_map_atomic_wc(struct io_mapping *mapping, unsigned long offset)
+{
+	offset += (unsigned long) mapping;
+	return iomap_atomic_prot_pfn(offset >> PAGE_SHIFT, KM_USER0,
+				     __pgprot(__PAGE_KERNEL_WC));
+}
+
+static inline void
+io_mapping_unmap_atomic(void *vaddr)
+{
+	iounmap_atomic(vaddr, KM_USER0);
+}
+
+static inline void *
+io_mapping_map_wc(struct io_mapping *mapping, unsigned long offset)
+{
+	offset += (unsigned long) mapping;
+	return ioremap_wc(offset, PAGE_SIZE);
+}
+
+static inline void
+io_mapping_unmap(void *vaddr)
+{
+	iounmap(vaddr);
+}
+
+#else
+
+/* Create the io_mapping object*/
+static inline struct io_mapping *
+io_mapping_create_wc(unsigned long base, unsigned long size)
+{
+	return (struct io_mapping *) ioremap_wc(base, size);
+}
+
+static inline void
+io_mapping_free(struct io_mapping *mapping)
+{
+	iounmap(mapping);
+}
+
+/* Atomic map/unmap */
+static inline void *
+io_mapping_map_atomic_wc(struct io_mapping *mapping, unsigned long offset)
+{
+	return ((char *) mapping) + offset;
+}
+
+static inline void
+io_mapping_unmap_atomic(void *vaddr)
+{
+}
+
+/* Non-atomic map/unmap */
+static inline void *
+io_mapping_map_wc(struct io_mapping *mapping, unsigned long offset)
+{
+	return ((char *) mapping) + offset;
+}
+
+static inline void
+io_mapping_unmap(void *vaddr)
+{
+}
+
+#endif /* HAVE_ATOMIC_IOMAP */
+
+#endif /* _LINUX_IO_MAPPING_H */
-- 
1.5.6.5

[PATCH] [drm/i915] Use io-mapping interfaces instead of a variety of mapping kludges

[Keith Packard]

Switch the i915 device aperture mapping to the io-mapping interface, taking
advantage of the cleaner API to extend it across all of the mapping uses,
including both pwrite and relocation updates.

This dramatically improves performance on 64-bit kernels which were using
the same slow path as 32-bit non-HIGHMEM kernels prior to this patch.

Signed-off-by: Keith Packard <keithp@keithp.com>
---
 drivers/gpu/drm/i915/i915_drv.h |    3 +
 drivers/gpu/drm/i915/i915_gem.c |  174 ++++++++++++++++++---------------------
 2 files changed, 83 insertions(+), 94 deletions(-)

diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index cc8a9f3..572dcd0 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -31,6 +31,7 @@
 #define _I915_DRV_H_
 
 #include "i915_reg.h"
+#include <linux/io-mapping.h>
 
 /* General customization:
  */
@@ -246,6 +247,8 @@ typedef struct drm_i915_private {
 	struct {
 		struct drm_mm gtt_space;
 
+		struct io_mapping *gtt_mapping;
+
 		/**
 		 * List of objects currently involved in rendering from the
 		 * ringbuffer.
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index c1733ac..cb7060b 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -193,35 +193,50 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 	return 0;
 }
 
-/*
- * Try to write quickly with an atomic kmap. Return true on success.
- *
- * If this fails (which includes a partial write), we'll redo the whole
- * thing with the slow version.
- *
- * This is a workaround for the low performance of iounmap (approximate
- * 10% cpu cost on normal 3D workloads).  kmap_atomic on HIGHMEM kernels
- * happens to let us map card memory without taking IPIs.  When the vmap
- * rework lands we should be able to dump this hack.
+/* This is the fast write path which cannot handle
+ * page faults in the source data
  */
-static inline int fast_user_write(unsigned long pfn, char __user *user_data,
-				  int l, int o)
+
+static inline int
+fast_user_write(struct io_mapping *mapping,
+		loff_t page_base, int page_offset,
+		char __user *user_data,
+		int length)
 {
-#ifdef CONFIG_HIGHMEM
-	unsigned long unwritten;
 	char *vaddr_atomic;
+	unsigned long unwritten;
+	
+	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
+	unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
+						      user_data, length);
+	io_mapping_unmap_atomic(vaddr_atomic);
+	if (unwritten)
+		return -EFAULT;
+	return 0;
+}
 
-	vaddr_atomic = kmap_atomic_pfn(pfn, KM_USER0);
-#if WATCH_PWRITE
-	DRM_INFO("pwrite i %d o %d l %d pfn %ld vaddr %p\n",
-		 i, o, l, pfn, vaddr_atomic);
-#endif
-	unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + o, user_data, l);
-	kunmap_atomic(vaddr_atomic, KM_USER0);
-	return !unwritten;
-#else
+/* Here's the write path which can sleep for
+ * page faults
+ */
+
+static inline int
+slow_user_write(struct io_mapping *mapping,
+		loff_t page_base, int page_offset,
+		char __user *user_data,
+		int length)
+{
+	char __iomem *vaddr;
+	unsigned long unwritten;
+	
+	vaddr = io_mapping_map_wc(mapping, page_base);
+	if (vaddr == NULL)
+		return -EFAULT;
+	unwritten = __copy_from_user(vaddr + page_offset,
+				     user_data, length);
+	io_mapping_unmap(vaddr);
+	if (unwritten)
+		return -EFAULT;
 	return 0;
-#endif
 }
 
 static int
@@ -230,10 +245,12 @@ i915_gem_gtt_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
 		    struct drm_file *file_priv)
 {
 	struct drm_i915_gem_object *obj_priv = obj->driver_private;
+	drm_i915_private_t *dev_priv = dev->dev_private;
 	ssize_t remain;
-	loff_t offset;
+	loff_t offset, page_base;
 	char __user *user_data;
-	int ret = 0;
+	int page_offset, page_length;
+	int ret;
 
 	user_data = (char __user *) (uintptr_t) args->data_ptr;
 	remain = args->size;
@@ -257,57 +274,37 @@ i915_gem_gtt_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
 	obj_priv->dirty = 1;
 
 	while (remain > 0) {
-		unsigned long pfn;
-		int i, o, l;
-
 		/* Operation in this page
 		 *
-		 * i = page number
-		 * o = offset within page
-		 * l = bytes to copy
+		 * page_base = page offset within aperture
+		 * page_offset = offset within page
+		 * page_length = bytes to copy for this page
 		 */
-		i = offset >> PAGE_SHIFT;
-		o = offset & (PAGE_SIZE-1);
-		l = remain;
-		if ((o + l) > PAGE_SIZE)
-			l = PAGE_SIZE - o;
-
-		pfn = (dev->agp->base >> PAGE_SHIFT) + i;
-
-		if (!fast_user_write(pfn, user_data, l, o)) {
-			unsigned long unwritten;
-			char __iomem *vaddr;
-
-			vaddr = ioremap_wc(pfn << PAGE_SHIFT, PAGE_SIZE);
-#if WATCH_PWRITE
-			DRM_INFO("pwrite slow i %d o %d l %d "
-				 "pfn %ld vaddr %p\n",
-				 i, o, l, pfn, vaddr);
-#endif
-			if (vaddr == NULL) {
-				ret = -EFAULT;
-				goto fail;
-			}
-			unwritten = __copy_from_user(vaddr + o, user_data, l);
-#if WATCH_PWRITE
-			DRM_INFO("unwritten %ld\n", unwritten);
-#endif
-			iounmap(vaddr);
-			if (unwritten) {
-				ret = -EFAULT;
+		page_base = (offset & ~(PAGE_SIZE-1));
+		page_offset = offset & (PAGE_SIZE-1);
+		page_length = remain;
+		if ((page_offset + remain) > PAGE_SIZE)
+			page_length = PAGE_SIZE - page_offset;
+
+		ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
+				       page_offset, user_data, page_length);
+		
+		/* If we get a fault while copying data, then (presumably) our
+		 * source page isn't available. In this case, use the
+		 * non-atomic function
+		 */
+		if (ret) {
+			ret = slow_user_write (dev_priv->mm.gtt_mapping,
+					       page_base, page_offset,
+					       user_data, page_length);
+			if (ret)
 				goto fail;
-			}
 		}
 
-		remain -= l;
-		user_data += l;
-		offset += l;
+		remain -= page_length;
+		user_data += page_length;
+		offset += page_length;
 	}
-#if WATCH_PWRITE && 1
-	i915_gem_clflush_object(obj);
-	i915_gem_dump_object(obj, args->offset + args->size, __func__, ~0);
-	i915_gem_clflush_object(obj);
-#endif
 
 fail:
 	i915_gem_object_unpin(obj);
@@ -1525,12 +1522,12 @@ i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
 				 struct drm_i915_gem_exec_object *entry)
 {
 	struct drm_device *dev = obj->dev;
+	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct drm_i915_gem_relocation_entry reloc;
 	struct drm_i915_gem_relocation_entry __user *relocs;
 	struct drm_i915_gem_object *obj_priv = obj->driver_private;
 	int i, ret;
-	uint32_t last_reloc_offset = -1;
-	void __iomem *reloc_page = NULL;
+	void __iomem *reloc_page;
 
 	/* Choose the GTT offset for our buffer and put it there. */
 	ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
@@ -1653,26 +1650,11 @@ i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
 		 * perform.
 		 */
 		reloc_offset = obj_priv->gtt_offset + reloc.offset;
-		if (reloc_page == NULL ||
-		    (last_reloc_offset & ~(PAGE_SIZE - 1)) !=
-		    (reloc_offset & ~(PAGE_SIZE - 1))) {
-			if (reloc_page != NULL)
-				iounmap(reloc_page);
-
-			reloc_page = ioremap_wc(dev->agp->base +
-						(reloc_offset &
-						 ~(PAGE_SIZE - 1)),
-						PAGE_SIZE);
-			last_reloc_offset = reloc_offset;
-			if (reloc_page == NULL) {
-				drm_gem_object_unreference(target_obj);
-				i915_gem_object_unpin(obj);
-				return -ENOMEM;
-			}
-		}
-
+		reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
+						      (reloc_offset & 
+						       ~(PAGE_SIZE - 1)));
 		reloc_entry = (uint32_t __iomem *)(reloc_page +
-					   (reloc_offset & (PAGE_SIZE - 1)));
+						   (reloc_offset & (PAGE_SIZE - 1)));
 		reloc_val = target_obj_priv->gtt_offset + reloc.delta;
 
 #if WATCH_BUF
@@ -1681,6 +1663,7 @@ i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
 			  readl(reloc_entry), reloc_val);
 #endif
 		writel(reloc_val, reloc_entry);
+		io_mapping_unmap_atomic(reloc_page);
 
 		/* Write the updated presumed offset for this entry back out
 		 * to the user.
@@ -1696,9 +1679,6 @@ i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
 		drm_gem_object_unreference(target_obj);
 	}
 
-	if (reloc_page != NULL)
-		iounmap(reloc_page);
-
 #if WATCH_BUF
 	if (0)
 		i915_gem_dump_object(obj, 128, __func__, ~0);
@@ -2540,6 +2520,10 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
 	if (ret != 0)
 		return ret;
 
+	dev_priv->mm.gtt_mapping = io_mapping_create_wc(dev->agp->base,
+							dev->agp->agp_info.aper_size
+							* 1024 * 1024);
+
 	mutex_lock(&dev->struct_mutex);
 	BUG_ON(!list_empty(&dev_priv->mm.active_list));
 	BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
@@ -2557,11 +2541,13 @@ int
 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
 		       struct drm_file *file_priv)
 {
+	drm_i915_private_t *dev_priv = dev->dev_private;
 	int ret;
 
 	ret = i915_gem_idle(dev);
 	drm_irq_uninstall(dev);
 
+	io_mapping_free(dev_priv->mm.gtt_mapping);
 	return ret;
 }
 
-- 
1.5.6.5

Re: [PATCH] Add io-mapping functions to dynamically map large device apertures

[Ingo Molnar]

* Keith Packard <keithp@keithp.com> wrote:

> On Fri, 2008-10-31 at 10:21 +0100, Ingo Molnar wrote:
> 
> > it's ugly and inflexible to put x86 dependencies into generic headers. 
> > (even though with a high likelyhood 32-bit x86 will be the only arch 
> > to ever implement the iomap_atomic() APIs)
> > 
> > Instead please add a HAVE_ATOMIC_IOMAP define to arch/x86/Kconfig:
> > 
> >  config HAVE_ATOMIC_IOMAP
> >          def_bool y
> >          depends on X86_32
> > 
> > ... and use #ifndef HAVE_ATOMIC_IOMAP in include/linux/io-mapping.h 
> > instead of #ifdef CONFIG_X86_64.
> 
> Just to clarify the issue here: there are two separate 
> implementations of the io_mapping API -- one for 'large address 
> space' machines where ioremap_wc can handle the typical graphics 
> aperture within the kernel virtual map, and the other using 
> iomap_atomic_prot_pfn for machines with puny address spaces.
> 
> All large address space machines can provide the io_mapping API 
> without any archtecture-specific support. For efficient 32-bit 
> io_mapping support, we require the new iomap_atomic_prot_pfn 
> function.
> 
> So, it seems like what I want to do is use the large address space 
> code on any machine which supports it, and then use the 
> iomap_atomic_prot_pfn version for small address space machines which 
> have the iomap_atomic_prot_pfn function.

Correct.

> What I think you're suggesting is to just assume that machines 
> without iomap_atomic_prot_pfn have address spaces large enough to 
> support the ioremap_wc path. The alternative is to create a third 
> (slow) path (which I did before the iomap_atomic_prot_pfn API was 
> introduced) that uses ioremap_wc at run time for small address space 
> machines without iomap_atomic_prot_pfn.
> 
> Let me know which you'd prefer and I'll get a patch out ASAP.

Please lets keep it simple: i.e. always use ioremap_wc() when there's 
no iomap_atomic_prot_pfn() 32-bit API provided.

( and by all means ioremap_wc() will just work fine on most 32-bit
  architectures out of box: they dont go about trying to map hundreds 
  of megabytes of graphics aperture. If they nevertheless need it, 
  they can implement iomap_atomic_prot_pfn() to add support. )

	Ingo

Re: [PATCH] Add io-mapping functions to dynamically map large device apertures

[Keith Packard]

[-- Attachment #1: Type: text/plain, Size: 1752 bytes --]

On Fri, 2008-10-31 at 10:21 +0100, Ingo Molnar wrote:

> it's ugly and inflexible to put x86 dependencies into generic headers. 
> (even though with a high likelyhood 32-bit x86 will be the only arch 
> to ever implement the iomap_atomic() APIs)
> 
> Instead please add a HAVE_ATOMIC_IOMAP define to arch/x86/Kconfig:
> 
>  config HAVE_ATOMIC_IOMAP
>          def_bool y
>          depends on X86_32
> 
> ... and use #ifndef HAVE_ATOMIC_IOMAP in include/linux/io-mapping.h 
> instead of #ifdef CONFIG_X86_64.

Just to clarify the issue here: there are two separate implementations
of the io_mapping API -- one for 'large address space' machines where
ioremap_wc can handle the typical graphics aperture within the kernel
virtual map, and the other using iomap_atomic_prot_pfn for machines with
puny address spaces.

All large address space machines can provide the io_mapping API without
any archtecture-specific support. For efficient 32-bit io_mapping
support, we require the new iomap_atomic_prot_pfn function.

So, it seems like what I want to do is use the large address space code
on any machine which supports it, and then use the iomap_atomic_prot_pfn
version for small address space machines which have the
iomap_atomic_prot_pfn function.

What I think you're suggesting is to just assume that machines without
iomap_atomic_prot_pfn have address spaces large enough to support the
ioremap_wc path. The alternative is to create a third (slow) path (which
I did before the iomap_atomic_prot_pfn API was introduced) that uses
ioremap_wc at run time for small address space machines without
iomap_atomic_prot_pfn.

Let me know which you'd prefer and I'll get a patch out ASAP.

-- 
keith.packard@intel.com

[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCH] Add io-mapping functions to dynamically map large device apertures

[Ingo Molnar]

* Eric Anholt <eric@anholt.net> wrote:

> From: Keith Packard <keithp@keithp.com>
> 
> Graphics devices have large PCI apertures which would consume a significant
> fraction of a 32-bit address space if mapped during driver initialization.
> Using ioremap at runtime is impractical as it is too slow. This new set of
> interfaces uses atomic mappings on 32-bit processors and a large static
> mapping on 64-bit processors to provide reasonable 32-bit performance and
> optimal 64-bit performance.
> 
> The current implementation sits atop the io_map_atomic fixmap-based mechanism
> for 32-bit processors.
> 
> This includes some editorial suggestions from Randy Dunlap for
> Documentation/io-mapping.txt
> 
> Signed-off-by: Keith Packard <keithp@keithp.com>
> Signed-off-by: Eric Anholt <eric@anholt.net>
> ---
>  Documentation/io-mapping.txt |   76 +++++++++++++++++++++++++++
>  include/linux/io-mapping.h   |  118 ++++++++++++++++++++++++++++++++++++++++++

I've applied your three patches to tip/core/resources for testing, 
thanks!

One small detail:

> +++ b/include/linux/io-mapping.h

> +#ifdef CONFIG_X86_64

it's ugly and inflexible to put x86 dependencies into generic headers. 
(even though with a high likelyhood 32-bit x86 will be the only arch 
to ever implement the iomap_atomic() APIs)

Instead please add a HAVE_ATOMIC_IOMAP define to arch/x86/Kconfig:

 config HAVE_ATOMIC_IOMAP
         def_bool y
         depends on X86_32

... and use #ifndef HAVE_ATOMIC_IOMAP in include/linux/io-mapping.h 
instead of #ifdef CONFIG_X86_64.

( Other 32-bit architectures which need an atomic iomap implementation
  for address space reasons can then implement the iomap_atomic*() 
  APIs too and set this same flag, to gain the same generic io_mapping
  implementation. )

Please send this cleanup as a delta patch, ontop of your three 
patches.

	Ingo

[PATCH] Add io-mapping functions to dynamically map large device apertures

[Eric Anholt]

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain, Size: 7347 bytes --]

From: Keith Packard <keithp@keithp.com>

Graphics devices have large PCI apertures which would consume a significant
fraction of a 32-bit address space if mapped during driver initialization.
Using ioremap at runtime is impractical as it is too slow. This new set of
interfaces uses atomic mappings on 32-bit processors and a large static
mapping on 64-bit processors to provide reasonable 32-bit performance and
optimal 64-bit performance.

The current implementation sits atop the io_map_atomic fixmap-based mechanism
for 32-bit processors.

This includes some editorial suggestions from Randy Dunlap for
Documentation/io-mapping.txt

Signed-off-by: Keith Packard <keithp@keithp.com>
Signed-off-by: Eric Anholt <eric@anholt.net>
---
 Documentation/io-mapping.txt |   76 +++++++++++++++++++++++++++
 include/linux/io-mapping.h   |  118 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 194 insertions(+), 0 deletions(-)
 create mode 100644 Documentation/io-mapping.txt
 create mode 100644 include/linux/io-mapping.h

diff --git a/Documentation/io-mapping.txt b/Documentation/io-mapping.txt
new file mode 100644
index 0000000..cd2f726
--- /dev/null
+++ b/Documentation/io-mapping.txt
@@ -0,0 +1,76 @@
+The io_mapping functions in linux/io-mapping.h provide an abstraction for
+efficiently mapping small regions of an I/O device to the CPU. The initial
+usage is to support the large graphics aperture on 32-bit processors where
+ioremap_wc cannot be used to statically map the entire aperture to the CPU
+as it would consume too much of the kernel address space.
+
+A mapping object is created during driver initialization using
+
+	struct io_mapping *io_mapping_create_wc(unsigned long base,
+						unsigned long size)
+
+		'base' is the bus address of the region to be made
+		mappable, while 'size' indicates how large a mapping region to
+		enable. Both are in bytes.
+
+		This _wc variant provides a mapping which may only be used
+		with the io_mapping_map_atomic_wc or io_mapping_map_wc.
+
+With this mapping object, individual pages can be mapped either atomically
+or not, depending on the necessary scheduling environment. Of course, atomic
+maps are more efficient:
+
+	void *io_mapping_map_atomic_wc(struct io_mapping *mapping,
+				       unsigned long offset)
+
+		'offset' is the offset within the defined mapping region.
+		Accessing addresses beyond the region specified in the
+		creation function yields undefined results. Using an offset
+		which is not page aligned yields an undefined result. The
+		return value points to a single page in CPU address space.
+
+		This _wc variant returns a write-combining map to the
+		page and may only be used with mappings created by
+		io_mapping_create_wc
+
+		Note that the task may not sleep while holding this page
+		mapped.
+
+	void io_mapping_unmap_atomic(void *vaddr)
+
+		'vaddr' must be the the value returned by the last
+		io_mapping_map_atomic_wc call. This unmaps the specified
+		page and allows the task to sleep once again.
+
+If you need to sleep while holding the lock, you can use the non-atomic
+variant, although they may be significantly slower.
+
+	void *io_mapping_map_wc(struct io_mapping *mapping,
+				unsigned long offset)
+
+		This works like io_mapping_map_atomic_wc except it allows
+		the task to sleep while holding the page mapped.
+
+	void io_mapping_unmap(void *vaddr)
+
+		This works like io_mapping_unmap_atomic, except it is used
+		for pages mapped with io_mapping_map_wc.
+
+At driver close time, the io_mapping object must be freed:
+
+	void io_mapping_free(struct io_mapping *mapping)
+
+Current Implementation:
+
+The initial implementation of these functions uses existing mapping
+mechanisms and so provides only an abstraction layer and no new
+functionality.
+
+On 64-bit processors, io_mapping_create_wc calls ioremap_wc for the whole
+range, creating a permanent kernel-visible mapping to the resource. The
+map_atomic and map functions add the requested offset to the base of the
+virtual address returned by ioremap_wc.
+
+On 32-bit processors, io_mapping_map_atomic_wc uses io_map_atomic_prot_pfn,
+which uses the fixmaps to get us a mapping to a page using an atomic fashion.
+For io_mapping_map_wc, ioremap_wc() is used to get a mapping of the region.
diff --git a/include/linux/io-mapping.h b/include/linux/io-mapping.h
new file mode 100644
index 0000000..1b56699
--- /dev/null
+++ b/include/linux/io-mapping.h
@@ -0,0 +1,118 @@
+/*
+ * Copyright © 2008 Keith Packard <keithp@keithp.com>
+ *
+ * This file is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#ifndef _LINUX_IO_MAPPING_H
+#define _LINUX_IO_MAPPING_H
+
+#include <linux/types.h>
+#include <asm/io.h>
+#include <asm/page.h>
+#include <asm/iomap.h>
+
+/*
+ * The io_mapping mechanism provides an abstraction for mapping
+ * individual pages from an io device to the CPU in an efficient fashion.
+ *
+ * See Documentation/io_mapping.txt
+ */
+
+/* this struct isn't actually defined anywhere */
+struct io_mapping;
+
+#ifdef CONFIG_X86_64
+
+/* Create the io_mapping object*/
+static inline struct io_mapping *
+io_mapping_create_wc(unsigned long base, unsigned long size)
+{
+	return (struct io_mapping *) ioremap_wc(base, size);
+}
+
+static inline void
+io_mapping_free(struct io_mapping *mapping)
+{
+	iounmap(mapping);
+}
+
+/* Atomic map/unmap */
+static inline void *
+io_mapping_map_atomic_wc(struct io_mapping *mapping, unsigned long offset)
+{
+	return ((char *) mapping) + offset;
+}
+
+static inline void
+io_mapping_unmap_atomic(void *vaddr)
+{
+}
+
+/* Non-atomic map/unmap */
+static inline void *
+io_mapping_map_wc(struct io_mapping *mapping, unsigned long offset)
+{
+	return ((char *) mapping) + offset;
+}
+
+static inline void
+io_mapping_unmap(void *vaddr)
+{
+}
+
+#endif /* CONFIG_X86_64 */
+
+#ifdef CONFIG_X86_32
+static inline struct io_mapping *
+io_mapping_create_wc(unsigned long base, unsigned long size)
+{
+	return (struct io_mapping *) base;
+}
+
+static inline void
+io_mapping_free(struct io_mapping *mapping)
+{
+}
+
+/* Atomic map/unmap */
+static inline void *
+io_mapping_map_atomic_wc(struct io_mapping *mapping, unsigned long offset)
+{
+	offset += (unsigned long) mapping;
+	return iomap_atomic_prot_pfn(offset >> PAGE_SHIFT, KM_USER0,
+				     __pgprot(__PAGE_KERNEL_WC));
+}
+
+static inline void
+io_mapping_unmap_atomic(void *vaddr)
+{
+	iounmap_atomic(vaddr, KM_USER0);
+}
+
+static inline void *
+io_mapping_map_wc(struct io_mapping *mapping, unsigned long offset)
+{
+	offset += (unsigned long) mapping;
+	return ioremap_wc(offset, PAGE_SIZE);
+}
+
+static inline void
+io_mapping_unmap(void *vaddr)
+{
+	iounmap(vaddr);
+}
+#endif /* CONFIG_X86_32 */
+
+#endif /* _LINUX_IO_MAPPING_H */
-- 
1.5.6.5

Re: [PATCH] Add io-mapping functions to dynamically map large device apertures

[Keith Packard]

[-- Attachment #1: Type: text/plain, Size: 234 bytes --]

On Thu, 2008-10-23 at 21:49 -0700, Randy Dunlap wrote:

(A bunch of helpful edits for the io-mapping.txt file)

Thanks! They're in my tree and will get included in the next version of
this patch.

-- 
keith.packard@intel.com

[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCH] Add io-mapping functions to dynamically map large device apertures

[Randy Dunlap]

On Thu, 23 Oct 2008 00:14:46 -0700 Keith Packard wrote:

>  Documentation/io-mapping.txt |   84 ++++++++++++++++++++++++++++
>  include/linux/io-mapping.h   |  125 ++++++++++++++++++++++++++++++++++++++++++
>  2 files changed, 209 insertions(+), 0 deletions(-)
>  create mode 100644 Documentation/io-mapping.txt
>  create mode 100644 include/linux/io-mapping.h
> 
> diff --git a/Documentation/io-mapping.txt b/Documentation/io-mapping.txt
> new file mode 100644
> index 0000000..ebf6dc5
> --- /dev/null
> +++ b/Documentation/io-mapping.txt
> @@ -0,0 +1,84 @@
> +The io_mapping functions in linux/io.h provide an abstraction for

                                     io-mapping.h ?

> +efficiently mapping small regions of an io device to the CPU. The initial

                                           IO or I/O, please

> +usage is to support the large graphics aperture on 32-bit processors where
> +ioremap_wc cannot be used to statically map the entire aperture to the CPU
> +as it would consume too much of the kernel address space.
> +
> +A mapping object is created during driver initialization using
> +
> +	struct io_mapping *
> +	io_mapping_create_wc(unsigned long base, unsigned long size)
> +
> +		'base' is the bus address of the region to be made
> +		mappable, while 'size' indicates how large a mapping region to
> +		enable. Both are in bytes.
> +
> +		This _wc variant provides a mapping which may only be used
> +		with the io_mapping_map_atomic_wc or io_mapping_map_wc.
> +
> +With this mapping object, individual pages can be mapped either atomically
> +or not, depending on the necessary scheduling environment. Of course, atomic
> +maps are more efficient:
> +
> +	void *
> +	io_mapping_map_atomic_wc(struct io_mapping *mapping, unsigned long offset)
> +
> +		'offset' is the offset within the defined mapping region.
> +		Accessing addresses beyond the region specified in the
> +		creation function yields undefined results. Using an offset
> +		which is not page aligned yields an undefined result. The
> +		return value points to a single page in CPU address space.
> +
> +		This _wc variant returns a write-combining map to the
> +		page and may only be used with 

		                          with <TBD>...

> +
> +		Note that the task may not sleep while holding this page
> +		mapped.
> +
> +	void
> +	io_mapping_unmap_atomic(void *vaddr)
> +
> +		'vaddr' must be the the value returned by the last
> +		io_mapping_map_atomic_wc call. This unmaps the specified
> +		page, and allows the task to sleep once again.

s/,//

> +
> +If you need to sleep while holding the lock, you can use the non-atomic
> +variant, although they may be significantly slower;

s/;/./

> +
> +	void *
> +	io_mapping_map_wc(struct io_mapping *mapping, unsigned long offset)
> +
> +		This works like io_mapping_map_atomic_wc except it allows
> +		the task to sleep while holding the page mapped.
> +
> +	void
> +	io_mapping_unmap(void *vaddr)
> +
> +		This works like io_mapping_unmap_atomic, except it is used
> +		for pages mapped with io_mapping_map_wc.
> +
> +At driver close time, the io_mapping object must be freed:
> +
> +	void
> +	io_mapping_free(struct io_mapping *mapping)
> +
> +Current Implementation:
> +
> +The initial implementation of these functions use existing mapping

                                                 uses

> +mechanisms and so provide only an abstraction layer and no new

                     provides

> +functionality.
> +
> +On 64-bit processors, io_mapping_create_wc calls ioremap_wc for the whole
> +range, creating a permanent kernel-visible mapping to the resource. The
> +map_atomic and map functions add the requested offset to the base of the
> +virtual address returned by ioremap_wc.
> +
> +On 32-bit processors with HIGHMEM defined, io_mapping_map_atomic_wc uses
> +kmap_atomic_pfn to map the specified page in an atomic fashion;
> +kmap_atomic_pfn isn't really supposed to be used with device pages, but it
> +provides an efficient mapping for this usage.
> +
> +On 32-bit processors without HIGHMEM defined, io_mapping_map_atomic_wc and
> +io_mapping_map_wc both use ioremap_wc, a terribly inefficient function which
> +performs an IPI to inform all processors about the new mapping. This results
> +in a significant performance penalty.


And I wish you could lose that horrible (non-Linux kernel) style of function
return type on a separate line.

---
~Randy

[PATCH] Add io-mapping functions to dynamically map large device apertures

[Keith Packard]

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain, Size: 7706 bytes --]

Graphics devices have large PCI apertures which would consume a significant
fraction of a 32-bit address space if mapped during driver initialization.
Using ioremap at runtime is impractical as it is too slow. This new set of
interfaces uses atomic mappings on 32-bit processors and a large static
mapping on 64-bit processors to provide reasonable 32-bit performance and
optimal 64-bit performance.

The current implementation sits atop the existing CONFIG_HIGHMEM kmap_atomic
mechanism for 32-bit processors when present. When absent, it just uses
ioremap, which remains horribly inefficient. Fixing non-HIGHMEM 32-bit
kernels to provide per-CPU mappings ala HIGHMEM would resolve that
performance issue.

Signed-off-by: Keith Packard <keithp@keithp.com>
---
 Documentation/io-mapping.txt |   84 ++++++++++++++++++++++++++++
 include/linux/io-mapping.h   |  125 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 209 insertions(+), 0 deletions(-)
 create mode 100644 Documentation/io-mapping.txt
 create mode 100644 include/linux/io-mapping.h

diff --git a/Documentation/io-mapping.txt b/Documentation/io-mapping.txt
new file mode 100644
index 0000000..ebf6dc5
--- /dev/null
+++ b/Documentation/io-mapping.txt
@@ -0,0 +1,84 @@
+The io_mapping functions in linux/io.h provide an abstraction for
+efficiently mapping small regions of an io device to the CPU. The initial
+usage is to support the large graphics aperture on 32-bit processors where
+ioremap_wc cannot be used to statically map the entire aperture to the CPU
+as it would consume too much of the kernel address space.
+
+A mapping object is created during driver initialization using
+
+	struct io_mapping *
+	io_mapping_create_wc(unsigned long base, unsigned long size)
+
+		'base' is the bus address of the region to be made
+		mappable, while 'size' indicates how large a mapping region to
+		enable. Both are in bytes.
+
+		This _wc variant provides a mapping which may only be used
+		with the io_mapping_map_atomic_wc or io_mapping_map_wc.
+
+With this mapping object, individual pages can be mapped either atomically
+or not, depending on the necessary scheduling environment. Of course, atomic
+maps are more efficient:
+
+	void *
+	io_mapping_map_atomic_wc(struct io_mapping *mapping, unsigned long offset)
+
+		'offset' is the offset within the defined mapping region.
+		Accessing addresses beyond the region specified in the
+		creation function yields undefined results. Using an offset
+		which is not page aligned yields an undefined result. The
+		return value points to a single page in CPU address space.
+
+		This _wc variant returns a write-combining map to the
+		page and may only be used with 
+
+		Note that the task may not sleep while holding this page
+		mapped.
+
+	void
+	io_mapping_unmap_atomic(void *vaddr)
+
+		'vaddr' must be the the value returned by the last
+		io_mapping_map_atomic_wc call. This unmaps the specified
+		page, and allows the task to sleep once again.
+
+If you need to sleep while holding the lock, you can use the non-atomic
+variant, although they may be significantly slower;
+
+	void *
+	io_mapping_map_wc(struct io_mapping *mapping, unsigned long offset)
+
+		This works like io_mapping_map_atomic_wc except it allows
+		the task to sleep while holding the page mapped.
+
+	void
+	io_mapping_unmap(void *vaddr)
+
+		This works like io_mapping_unmap_atomic, except it is used
+		for pages mapped with io_mapping_map_wc.
+
+At driver close time, the io_mapping object must be freed:
+
+	void
+	io_mapping_free(struct io_mapping *mapping)
+
+Current Implementation:
+
+The initial implementation of these functions use existing mapping
+mechanisms and so provide only an abstraction layer and no new
+functionality.
+
+On 64-bit processors, io_mapping_create_wc calls ioremap_wc for the whole
+range, creating a permanent kernel-visible mapping to the resource. The
+map_atomic and map functions add the requested offset to the base of the
+virtual address returned by ioremap_wc.
+
+On 32-bit processors with HIGHMEM defined, io_mapping_map_atomic_wc uses
+kmap_atomic_pfn to map the specified page in an atomic fashion;
+kmap_atomic_pfn isn't really supposed to be used with device pages, but it
+provides an efficient mapping for this usage.
+
+On 32-bit processors without HIGHMEM defined, io_mapping_map_atomic_wc and
+io_mapping_map_wc both use ioremap_wc, a terribly inefficient function which
+performs an IPI to inform all processors about the new mapping. This results
+in a significant performance penalty.
diff --git a/include/linux/io-mapping.h b/include/linux/io-mapping.h
new file mode 100644
index 0000000..dcc24d5
--- /dev/null
+++ b/include/linux/io-mapping.h
@@ -0,0 +1,125 @@
+/*
+ * Copyright © 2008 Keith Packard <keithp@keithp.com>
+ *
+ * This file is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#ifndef _LINUX_IO_MAPPING_H
+#define _LINUX_IO_MAPPING_H
+
+#include <linux/types.h>
+#include <asm/io.h>
+#include <asm/page.h>
+#include <linux/highmem.h>
+
+/*
+ * The io_mapping mechanism provides an abstraction for mapping
+ * individual pages from an io device to the CPU in an efficient fashion.
+ *
+ * See Documentation/io_mapping.txt
+ */
+
+/* this struct isn't actually defined anywhere */
+struct io_mapping;
+
+#ifdef CONFIG_X86_64
+
+/* Create the io_mapping object*/
+static inline struct io_mapping *
+io_mapping_create_wc(unsigned long base, unsigned long size)
+{
+	return (struct io_mapping *) ioremap_wc(base, size);
+}
+
+static inline void
+io_mapping_free(struct io_mapping *mapping)
+{
+	iounmap(mapping);
+}
+
+/* Atomic map/unmap */
+static inline void *
+io_mapping_map_atomic_wc(struct io_mapping *mapping, unsigned long offset)
+{
+	return ((char *) mapping) + offset;
+}
+
+static inline void
+io_mapping_unmap_atomic(void *vaddr)
+{
+}
+
+/* Non-atomic map/unmap */
+static inline void *
+io_mapping_map_wc(struct io_mapping *mapping, unsigned long offset)
+{
+	return ((char *) mapping) + offset;
+}
+
+static inline void
+io_mapping_unmap(void *vaddr)
+{
+}
+
+#endif /* CONFIG_X86_64 */
+
+#ifdef CONFIG_X86_32
+static inline struct io_mapping *
+io_mapping_create_wc(unsigned long base, unsigned long size)
+{
+	return (struct io_mapping *) base;
+}
+
+static inline void
+io_mapping_free(struct io_mapping *mapping)
+{
+}
+
+/* Atomic map/unmap */
+static inline void *
+io_mapping_map_atomic_wc(struct io_mapping *mapping, unsigned long offset)
+{
+	offset += (unsigned long) mapping;
+#ifdef CONFIG_HIGHMEM
+	return kmap_atomic_pfn(offset >> PAGE_SHIFT, KM_USER0);
+#else
+	return ioremap_wc(offset, PAGE_SIZE);
+#endif
+}
+
+static inline void
+io_mapping_unmap_atomic(void *vaddr)
+{
+#ifdef CONFIG_HIGHMEM
+	kunmap_atomic(vaddr, KM_USER0);
+#else
+	iounmap(vaddr);
+#endif
+}
+
+static inline void *
+io_mapping_map_wc(struct io_mapping *mapping, unsigned long offset)
+{
+	offset += (unsigned long) mapping;
+	return ioremap_wc(offset, PAGE_SIZE);
+}
+
+static inline void
+io_mapping_unmap(void *vaddr)
+{
+	iounmap(vaddr);
+}
+#endif /* CONFIG_X86_32 */
+
+#endif /* _LINUX_IO_MAPPING_H */
-- 
1.5.6.5