LKML Archive on lore.kernel.org
help / color / mirror / Atom feed
From: "tip-bot2 for Chang S. Bae" <tip-bot2@linutronix.de>
To: linux-tip-commits@vger.kernel.org
Cc: "Chang S. Bae" <chang.seok.bae@intel.com>,
	Thomas Gleixner <tglx@linutronix.de>,
	Sasha Levin <sashal@kernel.org>, x86 <x86@kernel.org>,
	LKML <linux-kernel@vger.kernel.org>
Subject: [tip: x86/fsgsbase] x86/fsgsbase/64: Enable FSGSBASE instructions in helper functions
Date: Thu, 18 Jun 2020 13:50:58 -0000	[thread overview]
Message-ID: <159248825847.16989.12376221390897516192.tip-bot2@tip-bot2> (raw)
In-Reply-To: <1557309753-24073-7-git-send-email-chang.seok.bae@intel.com>

The following commit has been merged into the x86/fsgsbase branch of tip:

Commit-ID:     58edfd2e0a93c9adc2f29902a0335af0584041a0
Gitweb:        https://git.kernel.org/tip/58edfd2e0a93c9adc2f29902a0335af0584041a0
Author:        Chang S. Bae <chang.seok.bae@intel.com>
AuthorDate:    Thu, 28 May 2020 16:13:50 -04:00
Committer:     Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Thu, 18 Jun 2020 15:47:00 +02:00

x86/fsgsbase/64: Enable FSGSBASE instructions in helper functions

Add cpu feature conditional FSGSBASE access to the relevant helper
functions. That allows to accelerate certain FS/GS base operations in
subsequent changes.

Note, that while possible, the user space entry/exit GSBASE operations are
not going to use the new FSGSBASE instructions. The reason is that it would
require additional storage for the user space value which adds more
complexity to the low level code and experiments have shown marginal
benefit. This may be revisited later but for now the SWAPGS based handling
in the entry code is preserved except for the paranoid entry/exit code.

To preserve the SWAPGS entry mechanism introduce __[rd|wr]gsbase_inactive()
helpers. Note, for Xen PV, paravirt hooks can be added later as they might
allow a very efficient but different implementation.

[ tglx: Massaged changelog, convert it to noinstr and force inline
  	native_swapgs() ]

Signed-off-by: Chang S. Bae <chang.seok.bae@intel.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Sasha Levin <sashal@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Link: https://lkml.kernel.org/r/1557309753-24073-7-git-send-email-chang.seok.bae@intel.com
Link: https://lkml.kernel.org/r/20200528201402.1708239-5-sashal@kernel.org


---
 arch/x86/include/asm/fsgsbase.h  | 27 +++++-------
 arch/x86/include/asm/processor.h |  2 +-
 arch/x86/kernel/process_64.c     | 68 +++++++++++++++++++++++++++++++-
 3 files changed, 81 insertions(+), 16 deletions(-)

diff --git a/arch/x86/include/asm/fsgsbase.h b/arch/x86/include/asm/fsgsbase.h
index fdd1177..aefd537 100644
--- a/arch/x86/include/asm/fsgsbase.h
+++ b/arch/x86/include/asm/fsgsbase.h
@@ -49,35 +49,32 @@ static __always_inline void wrgsbase(unsigned long gsbase)
 	asm volatile("wrgsbase %0" :: "r" (gsbase) : "memory");
 }
 
+#include <asm/cpufeature.h>
+
 /* Helper functions for reading/writing FS/GS base */
 
 static inline unsigned long x86_fsbase_read_cpu(void)
 {
 	unsigned long fsbase;
 
-	rdmsrl(MSR_FS_BASE, fsbase);
+	if (static_cpu_has(X86_FEATURE_FSGSBASE))
+		fsbase = rdfsbase();
+	else
+		rdmsrl(MSR_FS_BASE, fsbase);
 
 	return fsbase;
 }
 
-static inline unsigned long x86_gsbase_read_cpu_inactive(void)
-{
-	unsigned long gsbase;
-
-	rdmsrl(MSR_KERNEL_GS_BASE, gsbase);
-
-	return gsbase;
-}
-
 static inline void x86_fsbase_write_cpu(unsigned long fsbase)
 {
-	wrmsrl(MSR_FS_BASE, fsbase);
+	if (static_cpu_has(X86_FEATURE_FSGSBASE))
+		wrfsbase(fsbase);
+	else
+		wrmsrl(MSR_FS_BASE, fsbase);
 }
 
-static inline void x86_gsbase_write_cpu_inactive(unsigned long gsbase)
-{
-	wrmsrl(MSR_KERNEL_GS_BASE, gsbase);
-}
+extern unsigned long x86_gsbase_read_cpu_inactive(void);
+extern void x86_gsbase_write_cpu_inactive(unsigned long gsbase);
 
 #endif /* CONFIG_X86_64 */
 
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 42cd333..f66202d 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -575,7 +575,7 @@ native_load_sp0(unsigned long sp0)
 	this_cpu_write(cpu_tss_rw.x86_tss.sp0, sp0);
 }
 
-static inline void native_swapgs(void)
+static __always_inline void native_swapgs(void)
 {
 #ifdef CONFIG_X86_64
 	asm volatile("swapgs" ::: "memory");
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 9a97415..c41e0aa 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -150,6 +150,44 @@ enum which_selector {
 };
 
 /*
+ * Out of line to be protected from kprobes and tracing. If this would be
+ * traced or probed than any access to a per CPU variable happens with
+ * the wrong GS.
+ *
+ * It is not used on Xen paravirt. When paravirt support is needed, it
+ * needs to be renamed with native_ prefix.
+ */
+static noinstr unsigned long __rdgsbase_inactive(void)
+{
+	unsigned long gsbase;
+
+	lockdep_assert_irqs_disabled();
+
+	native_swapgs();
+	gsbase = rdgsbase();
+	native_swapgs();
+
+	return gsbase;
+}
+
+/*
+ * Out of line to be protected from kprobes and tracing. If this would be
+ * traced or probed than any access to a per CPU variable happens with
+ * the wrong GS.
+ *
+ * It is not used on Xen paravirt. When paravirt support is needed, it
+ * needs to be renamed with native_ prefix.
+ */
+static noinstr void __wrgsbase_inactive(unsigned long gsbase)
+{
+	lockdep_assert_irqs_disabled();
+
+	native_swapgs();
+	wrgsbase(gsbase);
+	native_swapgs();
+}
+
+/*
  * Saves the FS or GS base for an outgoing thread if FSGSBASE extensions are
  * not available.  The goal is to be reasonably fast on non-FSGSBASE systems.
  * It's forcibly inlined because it'll generate better code and this function
@@ -327,6 +365,36 @@ static unsigned long x86_fsgsbase_read_task(struct task_struct *task,
 	return base;
 }
 
+unsigned long x86_gsbase_read_cpu_inactive(void)
+{
+	unsigned long gsbase;
+
+	if (static_cpu_has(X86_FEATURE_FSGSBASE)) {
+		unsigned long flags;
+
+		local_irq_save(flags);
+		gsbase = __rdgsbase_inactive();
+		local_irq_restore(flags);
+	} else {
+		rdmsrl(MSR_KERNEL_GS_BASE, gsbase);
+	}
+
+	return gsbase;
+}
+
+void x86_gsbase_write_cpu_inactive(unsigned long gsbase)
+{
+	if (static_cpu_has(X86_FEATURE_FSGSBASE)) {
+		unsigned long flags;
+
+		local_irq_save(flags);
+		__wrgsbase_inactive(gsbase);
+		local_irq_restore(flags);
+	} else {
+		wrmsrl(MSR_KERNEL_GS_BASE, gsbase);
+	}
+}
+
 unsigned long x86_fsbase_read_task(struct task_struct *task)
 {
 	unsigned long fsbase;

  parent reply	other threads:[~2020-06-18 13:51 UTC|newest]

Thread overview: 63+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2019-05-08 10:02 [PATCH v7 00/18] x86: Enable FSGSBASE instructions Chang S. Bae
2019-05-08 10:02 ` [PATCH v7 01/18] x86/fsgsbase/64: Fix ARCH_SET_FS/GS behaviors for a remote task Chang S. Bae
     [not found]   ` <74F4F506-2913-4013-9D81-A0C69FA8CDF1@intel.com>
     [not found]     ` <6420E1A5-B5AD-4028-AA91-AA4D5445AC83@intel.com>
2019-06-16 15:44       ` Bae, Chang Seok
2019-06-16 16:32         ` Thomas Gleixner
2019-06-22 10:04         ` [tip:x86/cpu] x86/ptrace: Prevent ptrace from clearing the FS/GS selector tip-bot for Chang S. Bae
2020-06-18 13:50         ` [tip: x86/fsgsbase] " tip-bot2 for Chang S. Bae
2019-05-08 10:02 ` [PATCH v7 02/18] selftests/x86/fsgsbase: Test ptracer-induced GSBASE write Chang S. Bae
2019-06-22 10:04   ` [tip:x86/cpu] " tip-bot for Chang S. Bae
2019-05-08 10:02 ` [PATCH v7 03/18] x86/fsgsbase/64: Add 'unsafe_fsgsbase' to enable CR4.FSGSBASE Chang S. Bae
2019-06-22 10:05   ` [tip:x86/cpu] x86/cpu: " tip-bot for Andy Lutomirski
2020-06-18 13:50   ` [tip: x86/fsgsbase] " tip-bot2 for Andy Lutomirski
2019-05-08 10:02 ` [PATCH v7 04/18] kbuild: Raise the minimum required binutils version to 2.21 Chang S. Bae
2019-06-22 10:06   ` [tip:x86/cpu] " tip-bot for Chang S. Bae
2019-05-08 10:02 ` [PATCH v7 05/18] x86/fsgsbase/64: Add intrinsics for FSGSBASE instructions Chang S. Bae
2019-06-22 10:06   ` [tip:x86/cpu] " tip-bot for Andi Kleen
2020-06-18 13:50   ` [tip: x86/fsgsbase] " tip-bot2 for Andi Kleen
2019-05-08 10:02 ` [PATCH v7 06/18] x86/fsgsbase/64: Enable FSGSBASE instructions in the helper functions Chang S. Bae
2019-06-22 10:07   ` [tip:x86/cpu] x86/fsgsbase/64: Enable FSGSBASE instructions in " tip-bot for Chang S. Bae
2020-06-18 13:50   ` tip-bot2 for Chang S. Bae [this message]
2019-05-08 10:02 ` [PATCH v7 07/18] x86/fsgsbase/64: Preserve FS/GS state in __switch_to() if FSGSBASE is on Chang S. Bae
2019-06-21 15:22   ` Thomas Gleixner
2019-06-22 10:08   ` [tip:x86/cpu] x86/process/64: Use FSBSBASE in switch_to() if available tip-bot for Andy Lutomirski
2020-06-18 13:50   ` [tip: x86/fsgsbase] " tip-bot2 for Andy Lutomirski
2019-05-08 10:02 ` [PATCH v7 08/18] x86/fsgsbase/64: When copying a thread, use the FSGSBASE instructions Chang S. Bae
2019-06-22 10:09   ` [tip:x86/cpu] x86/process/64: Use FSGSBASE instructions on thread copy and ptrace tip-bot for Chang S. Bae
2020-06-18 13:50   ` [tip: x86/fsgsbase] " tip-bot2 for Chang S. Bae
2019-05-08 10:02 ` [PATCH v7 09/18] x86/entry/64: Add the READ_MSR_GSBASE macro Chang S. Bae
2019-05-08 10:02 ` [PATCH v7 10/18] x86/entry/64: Switch CR3 before SWAPGS on the paranoid entry Chang S. Bae
2019-06-22 10:09   ` [tip:x86/cpu] x86/entry/64: Switch CR3 before SWAPGS in " tip-bot for Chang S. Bae
2020-06-18 13:50   ` [tip: x86/fsgsbase] " tip-bot2 for Chang S. Bae
2019-05-08 10:02 ` [PATCH v7 11/18] x86/fsgsbase/64: Introduce the FIND_PERCPU_BASE macro Chang S. Bae
2019-06-22 10:10   ` [tip:x86/cpu] x86/entry/64: " tip-bot for Chang S. Bae
2020-06-18 13:50   ` [tip: x86/fsgsbase] " tip-bot2 for Chang S. Bae
2019-05-08 10:02 ` [PATCH v7 12/18] x86/fsgsbase/64: GSBASE handling with FSGSBASE in the paranoid path Chang S. Bae
2019-06-22 10:11   ` [tip:x86/cpu] x86/entry/64: Handle FSGSBASE enabled paranoid entry/exit tip-bot for Chang S. Bae
2019-06-29  7:21   ` [PATCH v7 12/18] x86/fsgsbase/64: GSBASE handling with FSGSBASE in the paranoid path Bae, Chang Seok
2019-06-29  7:37     ` Thomas Gleixner
2020-06-18 13:50   ` [tip: x86/fsgsbase] x86/entry/64: Handle FSGSBASE enabled paranoid entry/exit tip-bot2 for Chang S. Bae
2019-05-08 10:02 ` [PATCH v7 13/18] x86/fsgsbase/64: Document GSBASE handling in the paranoid path Chang S. Bae
2019-06-22 10:11   ` [tip:x86/cpu] x86/entry/64: " tip-bot for Chang S. Bae
2019-05-08 10:02 ` [PATCH v7 14/18] selftests/x86/fsgsbase: Test WRGSBASE Chang S. Bae
2019-06-22 10:12   ` [tip:x86/cpu] selftests/x86/fsgsbase: Test RD/WRGSBASE tip-bot for Andy Lutomirski
2019-05-08 10:02 ` [PATCH v7 15/18] selftests/x86/fsgsbase: Test ptracer-induced GSBASE write with FSGSBASE Chang S. Bae
2019-06-22 10:13   ` [tip:x86/cpu] " tip-bot for Chang S. Bae
2019-05-08 10:02 ` [PATCH v7 16/18] x86/fsgsbase/64: Enable FSGSBASE by default and add a chicken bit Chang S. Bae
2019-06-22 10:14   ` [tip:x86/cpu] x86/cpu: Enable FSGSBASE on 64bit " tip-bot for Andy Lutomirski
2020-06-18 13:50   ` [tip: x86/fsgsbase] " tip-bot2 for Andy Lutomirski
2019-05-08 10:02 ` [PATCH v7 17/18] x86/elf: Enumerate kernel FSGSBASE capability in AT_HWCAP2 Chang S. Bae
2019-06-22 10:14   ` [tip:x86/cpu] " tip-bot for Andi Kleen
2020-06-18 13:50   ` [tip: x86/fsgsbase] " tip-bot2 for Andi Kleen
2019-05-08 10:02 ` [PATCH v7 18/18] x86/fsgsbase/64: Add documentation for FSGSBASE Chang S. Bae
2019-06-14  6:54   ` Thomas Gleixner
2019-06-14 20:07     ` Bae, Chang Seok
     [not found]       ` <89BE934A-A392-4CED-83E5-CA4FADDAE6DF@intel.com>
2019-06-16  8:39         ` Thomas Gleixner
2019-06-16 12:34           ` Thomas Gleixner
2019-06-16 15:34             ` Bae, Chang Seok
2019-06-16 16:05               ` Thomas Gleixner
2019-06-16 20:48                 ` Bae, Chang Seok
2019-06-16 22:00                   ` Thomas Gleixner
     [not found]                     ` <8E2E84B6-BCCC-424D-A1A7-604828B389FB@intel.com>
2019-06-17  5:18                       ` Thomas Gleixner
2019-06-16 15:54     ` Randy Dunlap
2019-06-16 16:07       ` Thomas Gleixner
2019-06-22 10:15     ` [tip:x86/cpu] Documentation/x86/64: Add documentation for GS/FS addressing mode tip-bot for Thomas Gleixner

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=159248825847.16989.12376221390897516192.tip-bot2@tip-bot2 \
    --to=tip-bot2@linutronix.de \
    --cc=chang.seok.bae@intel.com \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-tip-commits@vger.kernel.org \
    --cc=sashal@kernel.org \
    --cc=tglx@linutronix.de \
    --cc=x86@kernel.org \
    --subject='Re: [tip: x86/fsgsbase] x86/fsgsbase/64: Enable FSGSBASE instructions in helper functions' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).