Name: Simplify skip_callipi Code
Author: Rusty Russell
Status: Experimental
Depends: Hotcpu/hotcpu-i386.patch.gz

D: Play with the skip_callipi code.  I suspect that some of the
D: various race fixes we've had are not really required now Vatsa has
D: fixed the real IPI race by putting explicit code in the sending
D: side.

diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .25609-linux-2.6.2-rc1/arch/i386/kernel/process.c .25609-linux-2.6.2-rc1.updated/arch/i386/kernel/process.c
--- .25609-linux-2.6.2-rc1/arch/i386/kernel/process.c	2004-01-23 14:05:51.000000000 +1100
+++ .25609-linux-2.6.2-rc1.updated/arch/i386/kernel/process.c	2004-01-23 14:40:42.000000000 +1100
@@ -153,6 +153,11 @@ static inline void check_cpu_quiescent(v
 		BUG_ON(!cpu_isset(cpu, cpu_active_map));
 		cpu_clear(cpu, cpu_active_map);
 
+		/* Death loop */
+
+		while (__get_cpu_var(cpu_state) != CPU_UP_PREPARE)
+			cpu_relax();
+
 		/*
 		 * It is possible that smp_call_function running on
 		 * some other CPU had noticed us to be in the active map
@@ -169,29 +174,22 @@ static inline void check_cpu_quiescent(v
 		 * Hence when we come online, we have to discard such an IPI
 		 * that got queued up while we were in the death loop.
 		 */
-		
-		/* Death loop */
-
-		while (__get_cpu_var(cpu_state) != CPU_UP_PREPARE)
-			cpu_relax();
+		local_irq_enable();
+		while (skip_callipi_cpu == smp_processor_id())
+			mb();
+		local_irq_disable();
 
-		/* from hereon we're ready to do work */
+		cpu_set(smp_processor_id(), cpu_online_map);
+		cpu_set(smp_processor_id(), cpu_active_map);
+		/* Ack __cpu_up(). */
 		__get_cpu_var(cpu_state) = CPU_ONLINE;
-		wmb();
-
-		//printk("Cpu %u arisen\n", smp_processor_id());
 
-		/* Put ourselves online before doing ___flush_tlb_all,
-		 * so we avoid losing one to a race. */
-		cpu_set(smp_processor_id(), cpu_active_map);
-		cpu_set(smp_processor_id(), cpu_online_map);
+		/* Make sure we're seen as active before we flush, so
+		   we don't miss any updates.  Might be overly
+		   paranoid. */
 		wmb();
 		__flush_tlb_all();
 		local_irq_enable();
-
-		while (cpu_isset(cpu, skip_callipi_map))
-			cpu_relax();
-
 		preempt_enable();
 	}
 }
diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .25609-linux-2.6.2-rc1/arch/i386/kernel/smp.c .25609-linux-2.6.2-rc1.updated/arch/i386/kernel/smp.c
--- .25609-linux-2.6.2-rc1/arch/i386/kernel/smp.c	2004-01-23 14:05:51.000000000 +1100
+++ .25609-linux-2.6.2-rc1.updated/arch/i386/kernel/smp.c	2004-01-23 14:40:46.000000000 +1100
@@ -561,7 +561,9 @@ int smp_call_function (void (*func) (voi
 		 * Hence, by setting the CPU in skip_callipi_map, we will "ask"
 		 * it to ignore the IPI whenever it comes up next.
 		 */
-		cpu_set(first_cpu(data.not_started), skip_callipi_map);
+		BUG_ON(skip_callipi_cpu != NR_CPUS);
+		skip_callipi_cpu = first_cpu(data.not_started);
+		BUG_ON(next_cpu(skip_callipi_cpu,data.not_started) != NR_CPUS);
 	}
 #endif
 	
@@ -625,9 +627,9 @@ asmlinkage void smp_call_function_interr
 	ack_APIC_irq();
 
 #ifdef CONFIG_HOTPLUG_CPU
-	if (unlikely(cpu_isset(cpu, skip_callipi_map))) {
+	if (unlikely(cpu == skip_callipi_cpu)) {
 		printk ("%d: Ignoring Queued IPI \n", cpu);
-		cpu_clear(cpu, skip_callipi_map);
+		skip_callipi_cpu = NR_CPUS;
 		return;
 	}
 #endif
diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .25609-linux-2.6.2-rc1/arch/i386/kernel/smpboot.c .25609-linux-2.6.2-rc1.updated/arch/i386/kernel/smpboot.c
--- .25609-linux-2.6.2-rc1/arch/i386/kernel/smpboot.c	2004-01-23 14:05:51.000000000 +1100
+++ .25609-linux-2.6.2-rc1.updated/arch/i386/kernel/smpboot.c	2004-01-23 15:01:30.000000000 +1100
@@ -71,8 +71,8 @@ cpumask_t cpu_online_map;
 cpumask_t cpu_active_map;
 #endif
 
-/* CPUs that have to skip processing of smp_call_function interrupt */
-cpumask_t skip_callipi_map;
+/* This CPU has to skip processing of smp_call_function interrupt */
+unsigned int skip_callipi_cpu = NR_CPUS;
 
 /* Initialize, although master cpu never calls in */
 static volatile cpumask_t cpu_callin_map;
diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .25609-linux-2.6.2-rc1/include/asm-i386/smp.h .25609-linux-2.6.2-rc1.updated/include/asm-i386/smp.h
--- .25609-linux-2.6.2-rc1/include/asm-i386/smp.h	2004-01-23 14:05:51.000000000 +1100
+++ .25609-linux-2.6.2-rc1.updated/include/asm-i386/smp.h	2004-01-23 14:07:04.000000000 +1100
@@ -52,7 +52,7 @@ extern void zap_low_mappings (void);
  */
 #define smp_processor_id() (current_thread_info()->cpu)
 
-extern cpumask_t skip_callipi_map;
+extern unsigned int skip_callipi_cpu;
 extern cpumask_t cpu_callout_map;
 #define cpu_possible_map cpu_callout_map
 
diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .25609-linux-2.6.2-rc1/kernel/cpu.c .25609-linux-2.6.2-rc1.updated/kernel/cpu.c
--- .25609-linux-2.6.2-rc1/kernel/cpu.c	2004-01-23 14:05:50.000000000 +1100
+++ .25609-linux-2.6.2-rc1.updated/kernel/cpu.c	2004-01-23 15:03:29.000000000 +1100
@@ -47,10 +47,10 @@ static inline void check_for_tasks(int c
 
 	write_lock_irq(&tasklist_lock);
 	for_each_process(p) {
-		int dying = p->state & (TASK_ZOMBIE | TASK_DEAD);
-		if (p->thread_info->cpu == cpu && !dying)
+		if (p->thread_info->cpu == cpu)
 			printk(KERN_WARNING "Task %s is on cpu %d, "
-				"not dying\n", p->comm, cpu);
+				"flags %d state %d\n", p->comm, cpu,
+			       p->flags, p->state);
 	}
 	write_unlock_irq(&tasklist_lock);
 }