[RFC 6/8] x86_64: Define the macros and tables for the basic UV infrastructure.

[RFC 6/8] x86_64: Define the macros and tables for the basic UV infrastructure.

[Jack Steiner]

Define the macros and tables for the basic UV infrastructure.


(NOTE: a work-in-progress. Pieces missing....)


	Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 include/asm-x86/uv_hub.h |  217 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 217 insertions(+)

Index: linux/include/asm-x86/uv_hub.h
===================================================================
--- /dev/null	1970-01-01 00:00:00.000000000 +0000
+++ linux/include/asm-x86/uv_hub.h	2008-03-21 15:48:18.000000000 -0500
@@ -0,0 +1,217 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * SGI UV architectural definitions
+ *
+ * Copyright (C) 2007 Silicon Graphics, Inc. All rights reserved.
+ */
+
+#ifndef __ASM_X86_UV_HUB_H__
+#define __ASM_X86_UV_HUB_H__
+
+#include <linux/numa.h>
+#include <linux/percpu.h>
+#include <asm/types.h>
+#include <asm/percpu.h>
+
+
+/*
+ * Addressing Terminology
+ *
+ * 	NASID - network ID of a router, Mbrick or Cbrick. Nasid values of
+ * 		routers always have low bit of 1, C/MBricks have low bit
+ * 		equal to 0. Most addressing macros that target UV hub chips
+ * 		right shift the NASID by 1 to exclude the always-zero bit.
+ *
+ *	SNASID - NASID right shifted by 1 bit.
+ *
+ *
+ *  Memory/UV-HUB Processor Socket Address Format:
+ *  +--------+---------------+---------------------+
+ *  |00..0000|    SNASID     |      NodeOffset     |
+ *  +--------+---------------+---------------------+
+ *           <--- N bits --->|<--------M bits ----->
+ *
+ *	M number of node offset bits (35 .. 40)
+ *	N number of SNASID bits (0 .. 10)
+ *
+ *		Note: M + N cannot currently exceed 44 (x86_64) or 46 (IA64).
+ *		The actual values are configuration dependent and are set at
+ *		boot time
+ *
+ * APICID format
+ * 	NOTE!!!!!! This is the current format of the APICID. However, code
+ * 	should assume that this will change in the future. Use functions
+ * 	in this file for all APICID bit manipulations and conversion.
+ *
+ * 		1111110000000000
+ * 		5432109876543210
+ *		nnnnnnnnnnlc0cch
+ *		sssssssssss
+ *
+ *			n  = snasid bits
+ *			l =  socket number on board
+ *			c  = core
+ *			h  = hyperthread
+ *			s  = bits that are in the socket CSR
+ *
+ *	Note: Processor only supports 12 bits in the APICID register. The ACPI
+ *	      tables hold all 16 bits. Software needs to be aware of this.
+ *
+ * 	      Unless otherwise specified, all references to APICID refer to
+ * 	      the FULL value contained in ACPI tables, not the subset in the
+ * 	      processor APICID register.
+ */
+
+
+/*
+ * Maximum number of bricks in all partitions and in all coherency domains.
+ * This is the total number of bricks accessible in the numalink fabric. It
+ * includes all C & M bricks. Routers are NOT included.
+ *
+ * This value is also the value of the maximum number of non-router NASIDs
+ * in the numalink fabric.
+ *
+ * NOTE: a brick may be 1 or 2 OS nodes. Don't get these confused.
+ */
+#define UV_MAX_NUMALINK_BLADES	16384
+
+/*
+ * Maximum number of C/Mbricks within a software SSI (hardware may support
+ * more).
+ */
+#define UV_MAX_SSI_BLADES	256
+
+/*
+ * The largest possible NASID of a C or M brick (+ 2)
+ */
+#define UV_MAX_NASID_VALUE	(UV_MAX_NUMALINK_NODES * 2)
+
+/*
+ * The following defines attributes of the HUB chip. These attributes are
+ * frequently referenced and are kept in the per-cpu data areas of each cpu.
+ * They are kept together in a struct to minimize cache misses.
+ */
+struct uv_hub_info_s {
+	unsigned long	global_mmr_base;
+	unsigned short	local_nasid;
+	unsigned short	gnode_upper;
+	unsigned short	coherency_domain_number;
+	unsigned short	numa_blade_id;
+	unsigned char	blade_processor_id;
+	unsigned char	m_val;
+	unsigned char	n_val;
+};
+DECLARE_PER_CPU(struct uv_hub_info_s, __uv_hub_info);
+#define uv_hub_info 		(&__get_cpu_var(__uv_hub_info))
+#define uv_cpu_hub_info(cpu)	(&per_cpu(__uv_hub_info, cpu))
+
+/* This header file is used in BIOS code that runs in physical mode */
+#ifdef __BIOS__
+#define UV_ADDR(x)		((unsigned long *)(x))
+#else
+#define UV_ADDR(x)		((unsigned long *)__va(x))
+#endif
+
+/*
+ * Local & Global MMR space macros.
+ * 	Note: macros are intended to be used ONLY by inline functions
+ * 	in this file - not by other kernel code.
+ */
+#define UV_SNASID(n)			((n) >> 1)
+#define UV_NASID(n)			((n) << 1)
+
+#define UV_LOCAL_MMR_BASE		0xf4000000UL
+#define UV_GLOBAL_MMR32_BASE		0xf8000000UL
+#define UV_GLOBAL_MMR64_BASE		(uv_hub_info->global_mmr_base)
+
+#define UV_GLOBAL_MMR32_SNASID_MASK	0x3ff
+#define UV_GLOBAL_MMR32_SNASID_SHIFT	15
+#define UV_GLOBAL_MMR64_SNASID_SHIFT	26
+
+#define UV_GLOBAL_MMR32_NASID_BITS(n)					\
+		(((UV_SNASID(n) & UV_GLOBAL_MMR32_SNASID_MASK)) <<	\
+		(UV_GLOBAL_MMR32_SNASID_SHIFT))
+
+#define UV_GLOBAL_MMR64_NASID_BITS(n)					\
+	((unsigned long)UV_SNASID(n) << UV_GLOBAL_MMR64_SNASID_SHIFT)
+
+#define UV_APIC_NASID_SHIFT	7
+
+/*
+ * Extract a NASID from an APICID (full apicid, not processor subset)
+ */
+static inline int uv_apicid_to_nasid(int apicid)
+{
+	return (UV_NASID(apicid >> UV_APIC_NASID_SHIFT));
+}
+
+/*
+ * Access global MMRs using the low memory MMR32 space. This region supports
+ * faster MMR access but not all MMRs are accessible in this space.
+ */
+static inline unsigned long *uv_global_mmr32_address(int nasid,
+				unsigned long offset)
+{
+	return UV_ADDR(UV_GLOBAL_MMR32_BASE |
+		       UV_GLOBAL_MMR32_NASID_BITS(nasid) | offset);
+}
+
+static inline void uv_write_global_mmr32(int nasid, unsigned long offset,
+				 unsigned long val)
+{
+	*uv_global_mmr32_address(nasid, offset) = val;
+}
+
+static inline unsigned long uv_read_global_mmr32(int nasid,
+						 unsigned long offset)
+{
+	return *uv_global_mmr32_address(nasid, offset);
+}
+
+/*
+ * Access Global MMR space using the MMR space located at the top of physical
+ * memory.
+ */
+static inline unsigned long *uv_global_mmr64_address(int nasid,
+				unsigned long offset)
+{
+	return UV_ADDR(UV_GLOBAL_MMR64_BASE |
+		       UV_GLOBAL_MMR64_NASID_BITS(nasid) | offset);
+}
+
+static inline void uv_write_global_mmr64(int nasid, unsigned long offset,
+				unsigned long val)
+{
+	*uv_global_mmr64_address(nasid, offset) = val;
+}
+
+static inline unsigned long uv_read_global_mmr64(int nasid,
+						 unsigned long offset)
+{
+	return *uv_global_mmr64_address(nasid, offset);
+}
+
+/*
+ * Access node local MMRs. Faster than using global space but only local MMRs
+ * are accessible.
+ */
+static inline unsigned long *uv_local_mmr_address(unsigned long offset)
+{
+	return UV_ADDR(UV_LOCAL_MMR_BASE | offset);
+}
+
+static inline unsigned long uv_read_local_mmr(unsigned long offset)
+{
+	return *uv_local_mmr_address(offset);
+}
+
+static inline void uv_write_local_mmr(unsigned long offset, unsigned long val)
+{
+	*uv_local_mmr_address(offset) = val;
+}
+
+#endif /* __ASM_X86_UV_HUB__ */
+

Re: [RFC 6/8] x86_64: Define the macros and tables for the basic UV infrastructure.

[Andi Kleen]

Jack Steiner <steiner@sgi.com> writes:

> Define the macros and tables for the basic UV infrastructure.
> 
> 
> (NOTE: a work-in-progress. Pieces missing....)

Does the kernel really need all this information? You just want
to address the UV-APIC right? I suspect you could use a much stripped
down file.

> +DECLARE_PER_CPU(struct uv_hub_info_s, __uv_hub_info);
> +#define uv_hub_info 		(&__get_cpu_var(__uv_hub_info))
> +#define uv_cpu_hub_info(cpu)	(&per_cpu(__uv_hub_info, cpu))
> +
> +/* This header file is used in BIOS code that runs in physical mode */

Not sure what physical mode is.

> +#ifdef __BIOS__
> +#define UV_ADDR(x)		((unsigned long *)(x))
> +#else
> +#define UV_ADDR(x)		((unsigned long *)__va(x))
> +#endif

But it it would be cleaner if your BIOS just supplied a suitable __va()
and then you remove these macros.

-Andi

Re: [RFC 6/8] x86_64: Define the macros and tables for the basic UV infrastructure.

[Jack Steiner]

On Tue, Mar 25, 2008 at 11:11:11AM +0100, Andi Kleen wrote:
> Jack Steiner <steiner@sgi.com> writes:
> 
> > Define the macros and tables for the basic UV infrastructure.
> > 
> > 
> > (NOTE: a work-in-progress. Pieces missing....)
> 
> Does the kernel really need all this information? You just want
> to address the UV-APIC right? I suspect you could use a much stripped
> down file.

Most of the macros will never be used by generic kernel code, but we
have UV-specific drivers that will use the information (GRU, XPC and
XPMEM drivers). All of these are getting very close to being ready to
be pushed upstream.

> 
> > +DECLARE_PER_CPU(struct uv_hub_info_s, __uv_hub_info);
> > +#define uv_hub_info 		(&__get_cpu_var(__uv_hub_info))
> > +#define uv_cpu_hub_info(cpu)	(&per_cpu(__uv_hub_info, cpu))
> > +
> > +/* This header file is used in BIOS code that runs in physical mode */
> 
> Not sure what physical mode is.

Me either :-)    I fixed the comment
	"... BIOS code that runs with virtual == physical"

However, then I read the rest of your comments & will take the approach
of defining __va() in the BIOS code. That eliminates the need for
the macro.

> 
> > +#ifdef __BIOS__
> > +#define UV_ADDR(x)		((unsigned long *)(x))
> > +#else
> > +#define UV_ADDR(x)		((unsigned long *)__va(x))
> > +#endif
> 
> But it it would be cleaner if your BIOS just supplied a suitable __va()
> and then you remove these macros.
> 
> -Andi

--- jack

Re: [RFC 6/8] x86_64: Define the macros and tables for the basic UV infrastructure.

[Christoph Hellwig]

On Tue, Mar 25, 2008 at 11:11:11AM +0100, Andi Kleen wrote:
> Not sure what physical mode is.
> 
> > +#ifdef __BIOS__
> > +#define UV_ADDR(x)		((unsigned long *)(x))
> > +#else
> > +#define UV_ADDR(x)		((unsigned long *)__va(x))
> > +#endif
> 
> But it it would be cleaner if your BIOS just supplied a suitable __va()
> and then you remove these macros.

the bios should just have headers of it's own instead of placing this
burden on kernel code.

Re: [RFC 6/8] x86_64: Define the macros and tables for the basic UV infrastructure.

[Jack Steiner]

On Tue, Mar 25, 2008 at 08:09:30PM -0400, Christoph Hellwig wrote:
> On Tue, Mar 25, 2008 at 11:11:11AM +0100, Andi Kleen wrote:
> > Not sure what physical mode is.
> > 
> > > +#ifdef __BIOS__
> > > +#define UV_ADDR(x)		((unsigned long *)(x))
> > > +#else
> > > +#define UV_ADDR(x)		((unsigned long *)__va(x))
> > > +#endif
> > 
> > But it it would be cleaner if your BIOS just supplied a suitable __va()
> > and then you remove these macros.
> 
> the bios should just have headers of it's own instead of placing this
> burden on kernel code.

See mail from earlier today. The UV_ADDR macro has been eliminated.


--- jack