[PATCH] sysrq show-all-cpus

[PATCH] sysrq show-all-cpus

[Rik van Riel]

SysRQ-P is not always useful on SMP systems, since it usually ends up showing the
backtrace of a CPU that is doing just fine, instead of the backtrace of the CPU
that is having problems.

This patch adds SysRQ show-all-cpus(L), which shows the backtrace of every active
CPU in the system.  It skips idle CPUs because some SMP systems are just too large
and we already know what the backtrace of the idle task looks like.

Signed-off-by: Rik van Riel <riel@redhat.com>

diff -up linux-2.6.25-rc3-mm1/drivers/char/sysrq.c.multicpu linux-2.6.25-rc3-mm1/drivers/char/sysrq.c
--- linux-2.6.25-rc3-mm1/drivers/char/sysrq.c.multicpu	2008-03-09 20:22:17.000000000 -0400
+++ linux-2.6.25-rc3-mm1/drivers/char/sysrq.c	2008-03-10 13:03:00.000000000 -0400
@@ -196,6 +196,44 @@ static struct sysrq_key_op sysrq_showloc
 #define sysrq_showlocks_op (*(struct sysrq_key_op *)0)
 #endif
 
+#ifdef CONFIG_SMP
+static DEFINE_SPINLOCK(show_lock);
+static void showacpu(void *dummy)
+{
+	unsigned long flags;
+
+	/* Idle CPUs have no interesting backtrace. */
+	if (idle_cpu(smp_processor_id()))
+		return;
+
+	spin_lock_irqsave(&show_lock, flags);
+	printk("CPU%d:\n", smp_processor_id());
+	show_stack(NULL, NULL);
+	spin_unlock_irqrestore(&show_lock, flags);
+}
+static void sysrq_showregs_othercpus(struct work_struct *dummy)
+{
+	smp_call_function(showacpu, NULL, 0, 0);
+}
+static DECLARE_WORK(sysrq_showallcpus, sysrq_showregs_othercpus);
+
+static void sysrq_handle_showallcpus(int key, struct tty_struct *tty)
+{
+	struct pt_regs *regs = get_irq_regs();
+	if (regs) {
+		printk("CPU%d:\n", smp_processor_id());
+		show_regs(regs);
+	}
+	schedule_work(&sysrq_showallcpus);
+}
+static struct sysrq_key_op sysrq_showallcpus_op = {
+	.handler	= sysrq_handle_showallcpus,
+	.help_msg	= "show-all-cpus(L)",
+	.action_msg	= "Show backtrace of all active CPUs",
+	.enable_mask	= SYSRQ_ENABLE_DUMP,
+};
+#endif
+
 static void sysrq_handle_showregs(int key, struct tty_struct *tty)
 {
 	struct pt_regs *regs = get_irq_regs();
@@ -340,7 +378,11 @@ static struct sysrq_key_op *sysrq_key_ta
 	&sysrq_kill_op,			/* i */
 	NULL,				/* j */
 	&sysrq_SAK_op,			/* k */
+#ifdef CONFIG_SMP
+	&sysrq_showallcpus_op,		/* l */
+#else
 	NULL,				/* l */
+#endif
 	&sysrq_showmem_op,		/* m */
 	&sysrq_unrt_op,			/* n */
 	/* o: This will often be registered as 'Off' at init time */

Re: [PATCH] sysrq show-all-cpus

[Andrew Morton]

On Mon, 10 Mar 2008 13:32:42 -0400
Rik van Riel <riel@redhat.com> wrote:

> SysRQ-P is not always useful on SMP systems, since it usually ends up showing the
> backtrace of a CPU that is doing just fine, instead of the backtrace of the CPU
> that is having problems.
> 
> This patch adds SysRQ show-all-cpus(L), which shows the backtrace of every active
> CPU in the system.  It skips idle CPUs because some SMP systems are just too large
> and we already know what the backtrace of the idle task looks like.
> 
> Signed-off-by: Rik van Riel <riel@redhat.com>
> 
> diff -up linux-2.6.25-rc3-mm1/drivers/char/sysrq.c.multicpu linux-2.6.25-rc3-mm1/drivers/char/sysrq.c
> --- linux-2.6.25-rc3-mm1/drivers/char/sysrq.c.multicpu	2008-03-09 20:22:17.000000000 -0400
> +++ linux-2.6.25-rc3-mm1/drivers/char/sysrq.c	2008-03-10 13:03:00.000000000 -0400
> @@ -196,6 +196,44 @@ static struct sysrq_key_op sysrq_showloc
>  #define sysrq_showlocks_op (*(struct sysrq_key_op *)0)
>  #endif
>  
> +#ifdef CONFIG_SMP
> +static DEFINE_SPINLOCK(show_lock);
> +static void showacpu(void *dummy)
> +{
> +	unsigned long flags;
> +
> +	/* Idle CPUs have no interesting backtrace. */
> +	if (idle_cpu(smp_processor_id()))
> +		return;
> +
> +	spin_lock_irqsave(&show_lock, flags);
> +	printk("CPU%d:\n", smp_processor_id());
> +	show_stack(NULL, NULL);
> +	spin_unlock_irqrestore(&show_lock, flags);
> +}
> +static void sysrq_showregs_othercpus(struct work_struct *dummy)
> +{
> +	smp_call_function(showacpu, NULL, 0, 0);
> +}
> +static DECLARE_WORK(sysrq_showallcpus, sysrq_showregs_othercpus);
> +
> +static void sysrq_handle_showallcpus(int key, struct tty_struct *tty)
> +{
> +	struct pt_regs *regs = get_irq_regs();
> +	if (regs) {
> +		printk("CPU%d:\n", smp_processor_id());
> +		show_regs(regs);
> +	}
> +	schedule_work(&sysrq_showallcpus);
> +}
> +static struct sysrq_key_op sysrq_showallcpus_op = {
> +	.handler	= sysrq_handle_showallcpus,
> +	.help_msg	= "show-all-cpus(L)",
> +	.action_msg	= "Show backtrace of all active CPUs",
> +	.enable_mask	= SYSRQ_ENABLE_DUMP,
> +};
> +#endif

So the CPU which took the interrupt is always the first to display - that's
nice.

Even if it's idle.  I guess that's OK.

How come the sysrq-handling CPU uses show_regs() and the others use
show_stack()?  Do we not set up the get_irq_regs() data for IPIs?

>  static void sysrq_handle_showregs(int key, struct tty_struct *tty)
>  {
>  	struct pt_regs *regs = get_irq_regs();
> @@ -340,7 +378,11 @@ static struct sysrq_key_op *sysrq_key_ta
>  	&sysrq_kill_op,			/* i */
>  	NULL,				/* j */
>  	&sysrq_SAK_op,			/* k */
> +#ifdef CONFIG_SMP
> +	&sysrq_showallcpus_op,		/* l */
> +#else
>  	NULL,				/* l */
> +#endif
>  	&sysrq_showmem_op,		/* m */
>  	&sysrq_unrt_op,			/* n */
>  	/* o: This will often be registered as 'Off' at init time */

Re: [PATCH] sysrq show-all-cpus

[Rik van Riel]

On Mon, 10 Mar 2008 10:41:54 -0700
Andrew Morton <akpm@linux-foundation.org> wrote:

> How come the sysrq-handling CPU uses show_regs() and the others use
> show_stack()?  Do we not set up the get_irq_regs() data for IPIs?

I tried it that way, and the other CPUs never printed anything.

I guess get_irq_regs() data is not present when in an IPI.

-- 
All rights reversed.

Re: [PATCH] sysrq show-all-cpus

[Randy Dunlap]

On Mon, 10 Mar 2008 13:32:42 -0400 Rik van Riel wrote:

> SysRQ-P is not always useful on SMP systems, since it usually ends up showing the
> backtrace of a CPU that is doing just fine, instead of the backtrace of the CPU
> that is having problems.
> 
> This patch adds SysRQ show-all-cpus(L), which shows the backtrace of every active
> CPU in the system.  It skips idle CPUs because some SMP systems are just too large
> and we already know what the backtrace of the idle task looks like.
> 
> Signed-off-by: Rik van Riel <riel@redhat.com>

Hi,
Please update Documentation/sysrq.txt also.

---
~Randy

[PATCH] sysrq show-all-cpus

[Rik van Riel]

SysRQ-P is not always useful on SMP systems, since it usually ends up showing the
backtrace of a CPU that is doing just fine, instead of the backtrace of the CPU
that is having problems.

This patch adds SysRQ show-all-cpus(L), which shows the backtrace of every active
CPU in the system.  It skips idle CPUs because some SMP systems are just too large
and we already know what the backtrace of the idle task looks like.
 
Signed-off-by: Rik van Riel <riel@redhat.com>

diff -up linux-2.6.25-rc3-mm1/Documentation/sysrq.txt.multicpu linux-2.6.25-rc3-mm1/Documentation/sysrq.txt
--- linux-2.6.25-rc3-mm1/Documentation/sysrq.txt.multicpu	2008-03-10 13:49:18.000000000 -0400
+++ linux-2.6.25-rc3-mm1/Documentation/sysrq.txt	2008-03-10 13:51:09.000000000 -0400
@@ -85,6 +85,8 @@ On all -  write a character to /proc/sys
 'k'     - Secure Access Key (SAK) Kills all programs on the current virtual
           console. NOTE: See important comments below in SAK section.
 
+'l'     - Shows a stack backtrace for all active CPUs.
+
 'm'     - Will dump current memory info to your console.
 
 'n'	- Used to make RT tasks nice-able
diff -up linux-2.6.25-rc3-mm1/drivers/char/sysrq.c.multicpu linux-2.6.25-rc3-mm1/drivers/char/sysrq.c
--- linux-2.6.25-rc3-mm1/drivers/char/sysrq.c.multicpu	2008-03-09 20:22:17.000000000 -0400
+++ linux-2.6.25-rc3-mm1/drivers/char/sysrq.c	2008-03-10 13:03:00.000000000 -0400
@@ -196,6 +196,44 @@ static struct sysrq_key_op sysrq_showloc
 #define sysrq_showlocks_op (*(struct sysrq_key_op *)0)
 #endif
 
+#ifdef CONFIG_SMP
+static DEFINE_SPINLOCK(show_lock);
+static void showacpu(void *dummy)
+{
+	unsigned long flags;
+
+	/* Idle CPUs have no interesting backtrace. */
+	if (idle_cpu(smp_processor_id()))
+		return;
+
+	spin_lock_irqsave(&show_lock, flags);
+	printk("CPU%d:\n", smp_processor_id());
+	show_stack(NULL, NULL);
+	spin_unlock_irqrestore(&show_lock, flags);
+}
+static void sysrq_showregs_othercpus(struct work_struct *dummy)
+{
+	smp_call_function(showacpu, NULL, 0, 0);
+}
+static DECLARE_WORK(sysrq_showallcpus, sysrq_showregs_othercpus);
+
+static void sysrq_handle_showallcpus(int key, struct tty_struct *tty)
+{
+	struct pt_regs *regs = get_irq_regs();
+	if (regs) {
+		printk("CPU%d:\n", smp_processor_id());
+		show_regs(regs);
+	}
+	schedule_work(&sysrq_showallcpus);
+}
+static struct sysrq_key_op sysrq_showallcpus_op = {
+	.handler	= sysrq_handle_showallcpus,
+	.help_msg	= "aLlcpus",
+	.action_msg	= "Show backtrace of all active CPUs",
+	.enable_mask	= SYSRQ_ENABLE_DUMP,
+};
+#endif
+
 static void sysrq_handle_showregs(int key, struct tty_struct *tty)
 {
 	struct pt_regs *regs = get_irq_regs();
@@ -340,7 +378,11 @@ static struct sysrq_key_op *sysrq_key_ta
 	&sysrq_kill_op,			/* i */
 	NULL,				/* j */
 	&sysrq_SAK_op,			/* k */
+#ifdef CONFIG_SMP
+	&sysrq_showallcpus_op,		/* l */
+#else
 	NULL,				/* l */
+#endif
 	&sysrq_showmem_op,		/* m */
 	&sysrq_unrt_op,			/* n */
 	/* o: This will often be registered as 'Off' at init time */

Re: [PATCH] sysrq show-all-cpus

[Randy Dunlap]

Rik van Riel wrote:
> SysRQ-P is not always useful on SMP systems, since it usually ends up showing the
> backtrace of a CPU that is doing just fine, instead of the backtrace of the CPU
> that is having problems.
> 
> This patch adds SysRQ show-all-cpus(L), which shows the backtrace of every active
> CPU in the system.  It skips idle CPUs because some SMP systems are just too large
> and we already know what the backtrace of the idle task looks like.
>  
> Signed-off-by: Rik van Riel <riel@redhat.com>
> 
> diff -up linux-2.6.25-rc3-mm1/Documentation/sysrq.txt.multicpu linux-2.6.25-rc3-mm1/Documentation/sysrq.txt
> --- linux-2.6.25-rc3-mm1/Documentation/sysrq.txt.multicpu	2008-03-10 13:49:18.000000000 -0400
> +++ linux-2.6.25-rc3-mm1/Documentation/sysrq.txt	2008-03-10 13:51:09.000000000 -0400
> @@ -85,6 +85,8 @@ On all -  write a character to /proc/sys
>  'k'     - Secure Access Key (SAK) Kills all programs on the current virtual
>            console. NOTE: See important comments below in SAK section.
>  
> +'l'     - Shows a stack backtrace for all active CPUs.
> +
>  'm'     - Will dump current memory info to your console.
>  
>  'n'	- Used to make RT tasks nice-able

Yep, thanks.
A diffstat is also appreciated (in the future).

-- 
~Randy

Re: [PATCH] sysrq show-all-cpus

[Jon Masters]

On Mon, 2008-03-10 at 14:03 -0400, Rik van Riel wrote:
> SysRQ-P is not always useful on SMP systems, since it usually ends up showing the
> backtrace of a CPU that is doing just fine, instead of the backtrace of the CPU
> that is having problems.
> 
> This patch adds SysRQ show-all-cpus(L), which shows the backtrace of every active
> CPU in the system.  It skips idle CPUs because some SMP systems are just too large
> and we already know what the backtrace of the idle task looks like.
>  
> Signed-off-by: Rik van Riel <riel@redhat.com>

Why not have it just print the local CPU on non-SMP anyway?

Jon.

Re: [PATCH] sysrq show-all-cpus

[Rik van Riel]

On Tue, 11 Mar 2008 03:18:24 -0400
Jon Masters <jonathan@jonmasters.org> wrote:

> On Mon, 2008-03-10 at 14:03 -0400, Rik van Riel wrote:
> > SysRQ-P is not always useful on SMP systems, since it usually ends up showing the
> > backtrace of a CPU that is doing just fine, instead of the backtrace of the CPU
> > that is having problems.
> > 
> > This patch adds SysRQ show-all-cpus(L), which shows the backtrace of every active
> > CPU in the system.  It skips idle CPUs because some SMP systems are just too large
> > and we already know what the backtrace of the idle task looks like.
> >  
> > Signed-off-by: Rik van Riel <riel@redhat.com>
> 
> Why not have it just print the local CPU on non-SMP anyway?

Sysrq p already does exactly that.

-- 
All rights reversed.

Re: [PATCH] sysrq show-all-cpus

[Jon Masters]

On Tue, 2008-03-11 at 10:07 -0400, Rik van Riel wrote:
> On Tue, 11 Mar 2008 03:18:24 -0400
> Jon Masters <jonathan@jonmasters.org> wrote:
> 
> > On Mon, 2008-03-10 at 14:03 -0400, Rik van Riel wrote:
> > > SysRQ-P is not always useful on SMP systems, since it usually ends up showing the
> > > backtrace of a CPU that is doing just fine, instead of the backtrace of the CPU
> > > that is having problems.
> > > 
> > > This patch adds SysRQ show-all-cpus(L), which shows the backtrace of every active
> > > CPU in the system.  It skips idle CPUs because some SMP systems are just too large
> > > and we already know what the backtrace of the idle task looks like.
> > >  
> > > Signed-off-by: Rik van Riel <riel@redhat.com>
> > 
> > Why not have it just print the local CPU on non-SMP anyway?
> 
> Sysrq p already does exactly that.

That's my point. I understand why there's a new command, because it is
different from 'p', but probably worth doing something on non-SMP too.

Jon.

Re: [PATCH] sysrq show-all-cpus

[Nick Piggin]

On Tuesday 11 March 2008 05:03, Rik van Riel wrote:
> SysRQ-P is not always useful on SMP systems, since it usually ends up
> showing the backtrace of a CPU that is doing just fine, instead of the
> backtrace of the CPU that is having problems.
>
> This patch adds SysRQ show-all-cpus(L), which shows the backtrace of every
> active CPU in the system.  It skips idle CPUs because some SMP systems are
> just too large and we already know what the backtrace of the idle task
> looks like.
>

Yeah we need something like this. I prefer mine though ;) It shows
registers as well as stack...

http://groups.google.com.au/group/fa.linux.kernel/msg/3cd0288f7fd3be2c?dmode=source&output=gplain

It prints just a single CPU at a time, but cycles through each one.
An alternative might be to have one sysrq key used to cycle between
CPUs, and sysrq+P could then just print out a trace from the selected
CPU...