LKML Archive on lore.kernel.org
help / color / mirror / Atom feed
From: Jens Axboe <jens.axboe@oracle.com>
To: linux-kernel@vger.kernel.org
Cc: npiggin@suse.de, dgc@sgi.com, Jens Axboe <jens.axboe@oracle.com>
Subject: [PATCH 2/7] x86-64: speedup and tweak smp_call_function_single()
Date: Wed, 12 Mar 2008 12:55:35 +0100	[thread overview]
Message-ID: <1205322940-20127-3-git-send-email-jens.axboe@oracle.com> (raw)
In-Reply-To: <1205322940-20127-1-git-send-email-jens.axboe@oracle.com>

Add a __smp_call_function_single() that allows passing in the
caller data to avoid an allocation.

It's OK to have interrupts disabled, as long as we don't wait for
the IPI call to finish.

Get rid of the fallback data and pass back an error instead. Callers
that don't want to handle errors must either use wait == 1, or pre-allocate
the data and use the __smp_call_function_single() variant.

Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
---
 arch/x86/kernel/smp_64.c |  117 +++++++++++++++++++++++----------------------
 include/linux/smp.h      |    8 +++
 2 files changed, 68 insertions(+), 57 deletions(-)

diff --git a/arch/x86/kernel/smp_64.c b/arch/x86/kernel/smp_64.c
index 1196a12..b1a3d3c 100644
--- a/arch/x86/kernel/smp_64.c
+++ b/arch/x86/kernel/smp_64.c
@@ -298,6 +298,8 @@ void smp_send_reschedule(int cpu)
 
 #define CALL_WAIT		0x01
 #define CALL_FALLBACK		0x02
+#define CALL_DATA_ALLOC		0x04
+
 /*
  * Structure and data for smp_call_function(). This is designed to minimise
  * static memory requirements. It also looks cleaner.
@@ -330,23 +332,12 @@ void unlock_ipi_call_lock(void)
 	spin_unlock_irq(&call_lock);
 }
 
-
-struct call_single_data {
-	struct list_head list;
-	void (*func) (void *info);
-	void *info;
-	unsigned int flags;
-};
-
 struct call_single_queue {
 	spinlock_t lock;
 	struct list_head list;
 };
 static DEFINE_PER_CPU(struct call_single_queue, call_single_queue);
 
-static unsigned long call_single_fallback_used;
-static struct call_single_data call_single_data_fallback;
-
 int __cpuinit init_smp_call(void)
 {
 	int i;
@@ -416,7 +407,8 @@ int smp_call_function_mask(cpumask_t mask,
 		data = &call_data_fallback;
 		flags |= CALL_FALLBACK;
 		/* XXX: can IPI all to "synchronize" RCU? */
-	}
+	} else
+		flags |= CALL_DATA_ALLOC;
 
 	spin_lock_init(&data->lock);
 	data->func = func;
@@ -446,7 +438,7 @@ int smp_call_function_mask(cpumask_t mask,
 		/* Wait for response */
 		while (data->flags)
 			cpu_relax();
-		if (likely(!(flags & CALL_FALLBACK)))
+		if (flags & CALL_DATA_ALLOC)
 			free_call_data(data);
 		else
 			clear_bit_unlock(0, &call_fallback_used);
@@ -457,6 +449,45 @@ int smp_call_function_mask(cpumask_t mask,
 EXPORT_SYMBOL(smp_call_function_mask);
 
 /*
+ * __smp_call_function_single - Run a function on a specific CPU
+ * @data: Associated data
+ *
+ * Retrurns 0 on success, else a negative status code.
+ *
+ * Does not return until the remote CPU is nearly ready to execute <func>
+ * or is or has executed. Also see smp_call_function_single()
+ */
+void __smp_call_function_single(int cpu, struct call_single_data *data)
+{
+	cpumask_t mask = cpumask_of_cpu(cpu);
+	struct call_single_queue *dst;
+	unsigned long flags;
+	/* prevent preemption and reschedule on another processor */
+	int ipi;
+
+	/* Can deadlock when called with interrupts disabled */
+	WARN_ON((data->flags & CALL_WAIT) && irqs_disabled());
+
+	INIT_LIST_HEAD(&data->list);
+	dst = &per_cpu(call_single_queue, cpu);
+
+	spin_lock_irqsave(&dst->lock, flags);
+	ipi = list_empty(&dst->list);
+	list_add_tail(&data->list, &dst->list);
+	spin_unlock_irqrestore(&dst->lock, flags);
+
+	if (ipi)
+		send_IPI_mask(mask, CALL_FUNCTION_SINGLE_VECTOR);
+
+	if (data->flags & CALL_WAIT) {
+		/* Wait for response */
+		while (data->flags)
+			cpu_relax();
+	}
+}
+EXPORT_SYMBOL(__smp_call_function_single);
+
+/*
  * smp_call_function_single - Run a function on a specific CPU
  * @func: The function to run. This must be fast and non-blocking.
  * @info: An arbitrary pointer to pass to the function.
@@ -468,68 +499,44 @@ EXPORT_SYMBOL(smp_call_function_mask);
  * Does not return until the remote CPU is nearly ready to execute <func>
  * or is or has executed.
  */
-
 int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
 			      int nonatomic, int wait)
 {
+	unsigned long flags;
 	/* prevent preemption and reschedule on another processor */
 	int me = get_cpu();
+	int ret = 0;
 
 	/* Can deadlock when called with interrupts disabled */
-	WARN_ON(irqs_disabled());
+	WARN_ON(wait && irqs_disabled());
 
 	if (cpu == me) {
-		local_irq_disable();
+		local_irq_save(flags);
 		func(info);
-		local_irq_enable();
+		local_irq_restore(flags);
 	} else {
 		struct call_single_data d;
 		struct call_single_data *data;
-		struct call_single_queue *dst;
-		cpumask_t mask = cpumask_of_cpu(cpu);
-		unsigned int flags = wait ? CALL_WAIT : 0;
-		int ipi;
 
 		if (!wait) {
-			data = kmalloc(sizeof(struct call_single_data), GFP_ATOMIC);
+			data = kmalloc(sizeof(*data), GFP_ATOMIC);
 			if (unlikely(!data)) {
-				while (test_and_set_bit_lock(0, &call_single_fallback_used))
-					cpu_relax();
-				data = &call_single_data_fallback;
-				flags |= CALL_FALLBACK;
+				ret = -ENOMEM;
+				goto out;
 			}
+			data->flags = CALL_DATA_ALLOC;
 		} else {
 			data = &d;
+			data->flags = CALL_WAIT;
 		}
 
 		data->func = func;
 		data->info = info;
-		data->flags = flags;
-		dst = &per_cpu(call_single_queue, cpu);
-
-		local_irq_disable();
-		while (!spin_trylock(&dst->lock)) {
-			local_irq_enable();
-			cpu_relax();
-			local_irq_disable();
-		}
-		ipi = list_empty(&dst->list);
-		list_add_tail(&data->list, &dst->list);
-		spin_unlock(&dst->lock);
-		local_irq_enable();
-
-		if (ipi)
-			send_IPI_mask(mask, CALL_FUNCTION_SINGLE_VECTOR);
-
-		if (wait) {
-			/* Wait for response */
-			while (data->flags)
-				cpu_relax();
-		}
+		__smp_call_function_single(cpu, data);
 	}
-
+out:
 	put_cpu();
-	return 0;
+	return ret;
 }
 EXPORT_SYMBOL(smp_call_function_single);
 
@@ -626,7 +633,7 @@ asmlinkage void smp_call_function_interrupt(void)
 				smp_wmb();
 				data->flags = 0;
 			} else {
-				if (likely(!(data->flags & CALL_FALLBACK)))
+				if (likely(data->flags & CALL_DATA_ALLOC))
 					free_call_data(data);
 				else
 					clear_bit_unlock(0, &call_fallback_used);
@@ -662,12 +669,8 @@ asmlinkage void smp_call_function_single_interrupt(void)
 		if (data->flags & CALL_WAIT) {
 			smp_wmb();
 			data->flags = 0;
-		} else {
-			if (likely(!(data->flags & CALL_FALLBACK)))
-				kfree(data);
-			else
-				clear_bit_unlock(0, &call_single_fallback_used);
-		}
+		} else if (data->flags & CALL_DATA_ALLOC)
+			kfree(data);
 	}
 	add_pda(irq_call_count, 1);
 	irq_exit();
diff --git a/include/linux/smp.h b/include/linux/smp.h
index c938d26..629a44a 100644
--- a/include/linux/smp.h
+++ b/include/linux/smp.h
@@ -49,12 +49,20 @@ extern int __cpu_up(unsigned int cpunum);
  */
 extern void smp_cpus_done(unsigned int max_cpus);
 
+struct call_single_data {
+	struct list_head list;
+	void (*func) (void *info);
+	void *info;
+	unsigned int flags;
+};
+
 /*
  * Call a function on all other processors
  */
 int smp_call_function(void(*func)(void *info), void *info, int retry, int wait);
 int smp_call_function_single(int cpuid, void (*func) (void *info), void *info,
 				int retry, int wait);
+void __smp_call_function_single(int cpuid, struct call_single_data *data);
 
 /*
  * Call a function on all processors
-- 
1.5.4.GIT


  parent reply	other threads:[~2008-03-12 11:58 UTC|newest]

Thread overview: 19+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2008-03-12 11:55 [PATCH 0/7] IO CPU affinity testing series Jens Axboe
2008-03-12 11:55 ` [PATCH 1/7] x86-64: introduce fast variant of smp_call_function_single() Jens Axboe
2008-03-14 18:21   ` Jeremy Fitzhardinge
2008-03-16 18:45     ` Jens Axboe
2008-03-16 22:58       ` Jeremy Fitzhardinge
2008-03-17  2:24         ` Nick Piggin
2008-03-17  7:25         ` Jens Axboe
2008-03-12 11:55 ` Jens Axboe [this message]
2008-03-12 11:55 ` [PATCH 3/7] x86: add fast smp_call_function_single() Jens Axboe
2008-03-12 11:55 ` [PATCH 4/7] block: split softirq handling into blk-softirq.c Jens Axboe
2008-03-12 11:55 ` [PATCH 5/7] Add interface for queuing work on a specific CPU Jens Axboe
2008-03-12 11:55 ` [PATCH 6/7] block: make kblockd_schedule_work() take the queue as parameter Jens Axboe
2008-03-12 11:55 ` [PATCH 7/7] block: add test code for testing CPU affinity Jens Axboe
2008-03-12 16:41 ` [PATCH 0/7] IO CPU affinity testing series Alan D. Brunelle
2008-03-12 17:54   ` Jens Axboe
2008-03-12 20:37 ` Max Krasnyanskiy
2008-03-13 12:13   ` Jens Axboe
2008-03-13 14:54 ` Alan D. Brunelle
2008-03-13 15:00   ` Jens Axboe

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=1205322940-20127-3-git-send-email-jens.axboe@oracle.com \
    --to=jens.axboe@oracle.com \
    --cc=dgc@sgi.com \
    --cc=linux-kernel@vger.kernel.org \
    --cc=npiggin@suse.de \
    --subject='Re: [PATCH 2/7] x86-64: speedup and tweak smp_call_function_single()' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).