LKML Archive on lore.kernel.org
help / color / mirror / Atom feed
From: Andy Lutomirski <luto@amacapital.net>
To: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>,
	Steven Rostedt <rostedt@goodmis.org>,
	Ingo Molnar <mingo@kernel.org>, Borislav Petkov <bp@alien8.de>,
	"H. Peter Anvin" <hpa@zytor.com>, Oleg Nesterov <oleg@redhat.com>,
	Frederic Weisbecker <fweisbec@gmail.com>,
	Alexei Starovoitov <ast@plumgrid.com>,
	Will Drewry <wad@chromium.org>, Kees Cook <keescook@chromium.org>,
	X86 ML <x86@kernel.org>,
	"linux-kernel@vger.kernel.org" <linux-kernel@vger.kernel.org>
Subject: Re: [PATCH 1/4] x86: save r11 into pt_regs->flags on SYSCALL64 fastpath
Date: Mon, 9 Mar 2015 13:02:11 -0700	[thread overview]
Message-ID: <CALCETrVUHPNcWfi1-tQCXdRGP8w2TLqvNpCq1ttsAX=_NwGrRw@mail.gmail.com> (raw)
In-Reply-To: <1425926364-9526-2-git-send-email-dvlasenk@redhat.com>

On Mon, Mar 9, 2015 at 11:39 AM, Denys Vlasenko <dvlasenk@redhat.com> wrote:
> Before this patch, r11 was saved in pt_regs->r11.
> Which looks natural, but requires messy shuffling to/from iret frame
> whenever ptrace or e.g. sys_iopl wants to modify flags - because
> that's how this register is used by SYSCALL/SYSRET.
>
> This patch saves r11 in pt_regs->flags,
> and uses that value for SYSRET64 insn. Shuffling is eliminated.
>
> FIXUP/RESTORE_TOP_OF_STACK are simplified.
>
> stub_iopl is no longer needed: pt_regs->flags needs no fixing up.
>
> Testing shows that syscall fast path is ~54.3 ns before
> and after the patch (on 2.7 GHz Sandy Bridge CPU).

Acked-by: Andy Lutomirski <luto@amacapital.net>

>
> Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
> CC: Linus Torvalds <torvalds@linux-foundation.org>
> CC: Steven Rostedt <rostedt@goodmis.org>
> CC: Ingo Molnar <mingo@kernel.org>
> CC: Borislav Petkov <bp@alien8.de>
> CC: "H. Peter Anvin" <hpa@zytor.com>
> CC: Andy Lutomirski <luto@amacapital.net>
> CC: Oleg Nesterov <oleg@redhat.com>
> CC: Frederic Weisbecker <fweisbec@gmail.com>
> CC: Alexei Starovoitov <ast@plumgrid.com>
> CC: Will Drewry <wad@chromium.org>
> CC: Kees Cook <keescook@chromium.org>
> CC: x86@kernel.org
> CC: linux-kernel@vger.kernel.org
> ---
>  arch/x86/include/asm/calling.h | 20 ++++++++++++++------
>  arch/x86/kernel/entry_64.S     | 24 +++++++++++-------------
>  2 files changed, 25 insertions(+), 19 deletions(-)
>
> diff --git a/arch/x86/include/asm/calling.h b/arch/x86/include/asm/calling.h
> index f1a962f..4b5f7bf 100644
> --- a/arch/x86/include/asm/calling.h
> +++ b/arch/x86/include/asm/calling.h
> @@ -95,9 +95,11 @@ For 32-bit we have the following conventions - kernel is built with
>         CFI_ADJUST_CFA_OFFSET 15*8+\addskip
>         .endm
>
> -       .macro SAVE_C_REGS_HELPER offset=0 rax=1 rcx=1 r8plus=1
> -       .if \r8plus
> +       .macro SAVE_C_REGS_HELPER offset=0 rax=1 rcx=1 r8910=1 r11=1
> +       .if \r11
>         movq_cfi r11, 6*8+\offset
> +       .endif
> +       .if \r8910
>         movq_cfi r10, 7*8+\offset
>         movq_cfi r9,  8*8+\offset
>         movq_cfi r8,  9*8+\offset
> @@ -113,16 +115,19 @@ For 32-bit we have the following conventions - kernel is built with
>         movq_cfi rdi, 14*8+\offset
>         .endm
>         .macro SAVE_C_REGS offset=0
> -       SAVE_C_REGS_HELPER \offset, 1, 1, 1
> +       SAVE_C_REGS_HELPER \offset, 1, 1, 1, 1
>         .endm
>         .macro SAVE_C_REGS_EXCEPT_RAX_RCX offset=0
> -       SAVE_C_REGS_HELPER \offset, 0, 0, 1
> +       SAVE_C_REGS_HELPER \offset, 0, 0, 1, 1
>         .endm
>         .macro SAVE_C_REGS_EXCEPT_R891011
> -       SAVE_C_REGS_HELPER 0, 1, 1, 0
> +       SAVE_C_REGS_HELPER 0, 1, 1, 0, 0
>         .endm
>         .macro SAVE_C_REGS_EXCEPT_RCX_R891011
> -       SAVE_C_REGS_HELPER 0, 1, 0, 0
> +       SAVE_C_REGS_HELPER 0, 1, 0, 0, 0
> +       .endm
> +       .macro SAVE_C_REGS_EXCEPT_RAX_RCX_R11
> +       SAVE_C_REGS_HELPER 0, 0, 0, 1, 0
>         .endm
>
>         .macro SAVE_EXTRA_REGS offset=0
> @@ -179,6 +184,9 @@ For 32-bit we have the following conventions - kernel is built with
>         .macro RESTORE_C_REGS_EXCEPT_R11
>         RESTORE_C_REGS_HELPER 1,1,0,1,1
>         .endm
> +       .macro RESTORE_C_REGS_EXCEPT_RCX_R11
> +       RESTORE_C_REGS_HELPER 1,0,0,1,1
> +       .endm
>         .macro RESTORE_RSI_RDI
>         RESTORE_C_REGS_HELPER 0,0,0,0,0
>         .endm
> diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
> index 5117a2b..324200a 100644
> --- a/arch/x86/kernel/entry_64.S
> +++ b/arch/x86/kernel/entry_64.S
> @@ -121,14 +121,12 @@ ENDPROC(native_usergs_sysret64)
>  #endif
>
>  /*
> - * C code is not supposed to know about undefined top of stack. Every time
> - * a C function with an pt_regs argument is called from the SYSCALL based
> - * fast path FIXUP_TOP_OF_STACK is needed.
> + * C code is not supposed to know that the iret frame is not populated.
> + * Every time a C function with an pt_regs argument is called from
> + * the SYSCALL based fast path FIXUP_TOP_OF_STACK is needed.
>   * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
>   * manipulation.
>   */
> -
> -       /* %rsp:at FRAMEEND */
>         .macro FIXUP_TOP_OF_STACK tmp offset=0
>         movq PER_CPU_VAR(old_rsp),\tmp
>         movq \tmp,RSP+\offset(%rsp)
> @@ -136,15 +134,13 @@ ENDPROC(native_usergs_sysret64)
>         movq $__USER_CS,CS+\offset(%rsp)
>         movq RIP+\offset(%rsp),\tmp  /* get rip */
>         movq \tmp,RCX+\offset(%rsp)  /* copy it to rcx as sysret would do */
> -       movq R11+\offset(%rsp),\tmp  /* get eflags */
> -       movq \tmp,EFLAGS+\offset(%rsp)
> +       movq EFLAGS+\offset(%rsp),\tmp /* ditto for rflags->r11 */
> +       movq \tmp,R11+\offset(%rsp)
>         .endm
>
>         .macro RESTORE_TOP_OF_STACK tmp offset=0
>         movq RSP+\offset(%rsp),\tmp
>         movq \tmp,PER_CPU_VAR(old_rsp)
> -       movq EFLAGS+\offset(%rsp),\tmp
> -       movq \tmp,R11+\offset(%rsp)
>         .endm
>
>  /*
> @@ -257,9 +253,10 @@ GLOBAL(system_call_after_swapgs)
>          */
>         ENABLE_INTERRUPTS(CLBR_NONE)
>         ALLOC_PT_GPREGS_ON_STACK 8              /* +8: space for orig_ax */
> -       SAVE_C_REGS_EXCEPT_RAX_RCX
> +       SAVE_C_REGS_EXCEPT_RAX_RCX_R11
>         movq    $-ENOSYS,RAX(%rsp)
>         movq_cfi rax,ORIG_RAX
> +       movq    %r11,EFLAGS(%rsp)
>         movq    %rcx,RIP(%rsp)
>         CFI_REL_OFFSET rip,RIP
>         testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP)
> @@ -277,7 +274,7 @@ system_call_fastpath:
>         movq %rax,RAX(%rsp)
>  /*
>   * Syscall return path ending with SYSRET (fast path)
> - * Has incomplete stack frame and undefined top of stack.
> + * Has incompletely filled pt_regs, iret frame is also incomplete.
>   */
>  ret_from_sys_call:
>         testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP)
> @@ -291,9 +288,10 @@ ret_from_sys_call:
>          * sysretq will re-enable interrupts:
>          */
>         TRACE_IRQS_ON
> -       RESTORE_C_REGS_EXCEPT_RCX
> -       movq RIP(%rsp),%rcx
> +       RESTORE_C_REGS_EXCEPT_RCX_R11
> +       movq    RIP(%rsp),%rcx
>         CFI_REGISTER    rip,rcx
> +       movq    EFLAGS(%rsp),%r11
>         /*CFI_REGISTER  rflags,r11*/
>         movq    PER_CPU_VAR(old_rsp), %rsp
>         /*
> --
> 1.8.1.4
>



-- 
Andy Lutomirski
AMA Capital Management, LLC

  reply	other threads:[~2015-03-09 20:02 UTC|newest]

Thread overview: 23+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2015-03-09 18:39 [PATCH 0/4 v2] x86: entry_64.S: steps towards simpler iret frame handling Denys Vlasenko
2015-03-09 18:39 ` [PATCH 1/4] x86: save r11 into pt_regs->flags on SYSCALL64 fastpath Denys Vlasenko
2015-03-09 20:02   ` Andy Lutomirski [this message]
2015-03-16 12:04   ` [tip:x86/asm] x86/asm/entry/64: Save R11 into pt_regs-> flags " tip-bot for Denys Vlasenko
2015-03-09 18:39 ` [PATCH 3/4] x86: save user rsp in pt_regs->sp " Denys Vlasenko
2015-03-09 20:11   ` Andy Lutomirski
2015-03-09 20:32     ` Denys Vlasenko
2015-03-09 20:43       ` Andy Lutomirski
2015-03-10 12:51   ` Ingo Molnar
2015-03-10 13:10     ` Andy Lutomirski
2015-03-10 13:18     ` Denys Vlasenko
2015-03-10 13:20       ` Andy Lutomirski
2015-03-10 13:26         ` Ingo Molnar
2015-03-10 13:21       ` Ingo Molnar
2015-03-10 13:26         ` Andy Lutomirski
2015-03-10 14:00           ` Denys Vlasenko
2015-03-10 14:02             ` Andy Lutomirski
2015-03-10 14:09               ` Denys Vlasenko
2015-03-10 13:28         ` Ingo Molnar
2015-03-10 13:50         ` Denys Vlasenko
2015-03-16  9:44           ` Ingo Molnar
2015-03-16 12:05   ` [tip:x86/asm] x86/asm/entry/64: Save user RSP in pt_regs-> sp " tip-bot for Denys Vlasenko
2015-03-10  6:00 ` [PATCH 0/4 v2] x86: entry_64.S: steps towards simpler iret frame handling Ingo Molnar

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to='CALCETrVUHPNcWfi1-tQCXdRGP8w2TLqvNpCq1ttsAX=_NwGrRw@mail.gmail.com' \
    --to=luto@amacapital.net \
    --cc=ast@plumgrid.com \
    --cc=bp@alien8.de \
    --cc=dvlasenk@redhat.com \
    --cc=fweisbec@gmail.com \
    --cc=hpa@zytor.com \
    --cc=keescook@chromium.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=mingo@kernel.org \
    --cc=oleg@redhat.com \
    --cc=rostedt@goodmis.org \
    --cc=torvalds@linux-foundation.org \
    --cc=wad@chromium.org \
    --cc=x86@kernel.org \
    --subject='Re: [PATCH 1/4] x86: save r11 into pt_regs->flags on SYSCALL64 fastpath' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).