Name: Hot-plug CPU Boot Changes
Author: Rusty Russell
Status: Experimental
Depends: Hotcpu/nonlinear-cpus.patch.gz Hotcpu/do-fork.patch.gz Hotcpu/clone-pid.patch.gz

D: This patch alters the boot sequence to "plug in" each CPU.

diff -urN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.19.29524/include/linux/notifier.h linux-2.5.19.29524.updated/include/linux/notifier.h
--- linux-2.5.19.29524/include/linux/notifier.h	Fri Mar  1 01:08:18 2002
+++ linux-2.5.19.29524.updated/include/linux/notifier.h	Thu May 30 11:41:17 2002
@@ -58,5 +58,7 @@
 #define SYS_HALT	0x0002	/* Notify of system halt */
 #define SYS_POWER_OFF	0x0003	/* Notify of system power off */
 
+#define CPU_ONLINE	0x0002 /* CPU (unsigned)v coming up */
+
 #endif /* __KERNEL__ */
 #endif /* _LINUX_NOTIFIER_H */
diff -urN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.19.29524/include/linux/sched.h linux-2.5.19.29524.updated/include/linux/sched.h
--- linux-2.5.19.29524/include/linux/sched.h	Thu May 30 10:00:59 2002
+++ linux-2.5.19.29524.updated/include/linux/sched.h	Thu May 30 11:41:17 2002
@@ -152,7 +152,6 @@
 extern void update_one_process(struct task_struct *p, unsigned long user,
 			       unsigned long system, int cpu);
 extern void scheduler_tick(int user_tick, int system);
-extern void migration_init(void);
 extern unsigned long cache_decay_ticks;
 
 
diff -urN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.19.29524/include/linux/smp.h linux-2.5.19.29524.updated/include/linux/smp.h
--- linux-2.5.19.29524/include/linux/smp.h	Thu May 30 11:41:09 2002
+++ linux-2.5.19.29524.updated/include/linux/smp.h	Thu May 30 11:41:17 2002
@@ -32,19 +32,14 @@
 
 
 /*
- * Boot processor call to load the other CPU's
- */
-extern void smp_boot_cpus(void);
-
-/*
- * Processor call in. Must hold processors until ..
+ * Prepare machine for booting other CPUs.
  */
-extern void smp_callin(void);
+extern void smp_prepare_cpus(unsigned int max_cpus);
 
 /*
- * Multiprocessors may now schedule
+ * Final polishing of CPUs
  */
-extern void smp_commence(void);
+extern void smp_cpus_done(unsigned int max_cpus);
 
 /*
  * Call a function on all other processors
@@ -71,6 +66,14 @@
 #define MSG_RESCHEDULE		0x0003	/* Reschedule request from master CPU*/
 #define MSG_CALL_FUNCTION       0x0004  /* Call function on all other CPUs */
 
+struct notifier_block;
+
+/* Need to know about CPUs going up/down? */
+extern int register_cpu_notifier(struct notifier_block *nb);
+extern void unregister_cpu_notifier(struct notifier_block *nb);
+
+/* Bring a CPU up */
+int cpu_up(unsigned int cpu);
 #else /* !SMP */
 
 /*
@@ -91,6 +94,10 @@
 #define __per_cpu_data
 #define per_cpu(var, cpu)			var
 #define this_cpu(var)				var
+
+/* Need to know about CPUs going up/down? */
+#define register_cpu_notifier(nb) 0
+#define unregister_cpu_notifier(nb) do { } while(0)
 
 #endif /* !SMP */
 
diff -urN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.19.29524/init/main.c linux-2.5.19.29524.updated/init/main.c
--- linux-2.5.19.29524/init/main.c	Sat May 25 14:35:00 2002
+++ linux-2.5.19.29524.updated/init/main.c	Thu May 30 11:41:17 2002
@@ -94,6 +94,35 @@
 
 char *execute_command;
 
+/* Setup configured maximum number of CPUs to activate */
+static unsigned int max_cpus = UINT_MAX;
+
+/*
+ * Setup routine for controlling SMP activation
+ *
+ * Command-line option of "nosmp" or "maxcpus=0" will disable SMP
+ * activation entirely (the MPS table probe still happens, though).
+ *
+ * Command-line option of "maxcpus=<NUM>", where <NUM> is an integer
+ * greater than 0, limits the maximum number of CPUs activated in
+ * SMP mode to <NUM>.
+ */
+static int __init nosmp(char *str)
+{
+	max_cpus = 0;
+	return 1;
+}
+
+__setup("nosmp", nosmp);
+
+static int __init maxcpus(char *str)
+{
+	get_option(&str, &max_cpus);
+	return 1;
+}
+
+__setup("maxcpus=", maxcpus);
+
 static char * argv_init[MAX_INIT_ARGS+2] = { "init", NULL, };
 char * envp_init[MAX_INIT_ENVS+2] = { "HOME=/", "TERM=linux", NULL, };
 
@@ -274,6 +303,7 @@
 #endif
 
 static inline void setup_per_cpu_areas(void) { }
+static inline void smp_prepare_cpus(unsigned int maxcpus) { }
 
 #else
 
@@ -304,11 +334,27 @@
 /* Called by boot processor to activate the rest. */
 static void __init smp_init(void)
 {
+	unsigned int i;
+
+	/* FIXME: This should be done in userspace --RR */
+	for (i = 0; i < NR_CPUS; i++) {
+		if (num_online_cpus() >= max_cpus)
+			break;
+		if (cpu_possible(i) && !cpu_online(i)) {
+			printk("Bringing up %i\n", i);
+			cpu_up(i);
+		}
+	}
+
+	/* Any cleanup work */
+	printk("CPUS done %u\n", max_cpus);
+	smp_cpus_done(max_cpus);
+#if 0
 	/* Get other processors into their bootup holding patterns. */
-	smp_boot_cpus();
 
 	smp_threads_ready=1;
 	smp_commence();
+#endif
 }
 
 #endif
@@ -406,14 +452,12 @@
 	check_bugs();
 	printk("POSIX conformance testing by UNIFIX\n");
 
-	init_idle(current, smp_processor_id());
-
 	/* 
 	 *	We count on the initial thread going ok 
 	 *	Like idlers init is an unlocked kernel thread, which will
 	 *	make syscalls (and thus be locked).
 	 */
-	smp_init();
+	init_idle(current, smp_processor_id());
 
 	/* Do the rest non-__init'ed, we're now alive */
 	rest_init();
@@ -445,12 +489,6 @@
 static void __init do_basic_setup(void)
 {
 	/*
-	 * Let the per-CPU migration threads start up:
-	 */
-#if CONFIG_SMP
-	migration_init();
-#endif
-	/*
 	 * Tell the world that we're going to be the grim
 	 * reaper of innocent orphaned children.
 	 *
@@ -494,7 +532,10 @@
 	static char * argv_sh[] = { "sh", NULL, };
 
 	lock_kernel();
+	/* Sets up cpus_possible() */
+	smp_prepare_cpus(max_cpus);
 	do_basic_setup();
+	smp_init();
 
 	prepare_namespace();
 
diff -urN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.19.29524/kernel/Makefile linux-2.5.19.29524.updated/kernel/Makefile
--- linux-2.5.19.29524/kernel/Makefile	Sat May 25 14:35:00 2002
+++ linux-2.5.19.29524.updated/kernel/Makefile	Thu May 30 11:46:18 2002
@@ -17,6 +17,7 @@
 	    sysctl.o capability.o ptrace.o timer.o user.o \
 	    signal.o sys.o kmod.o context.o futex.o platform.o
 
+obj-$(CONFIG_SMP) += cpu.o
 obj-$(CONFIG_UID16) += uid16.o
 obj-$(CONFIG_MODULES) += ksyms.o
 obj-$(CONFIG_PM) += pm.o
diff -urN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.19.29524/kernel/cpu.c linux-2.5.19.29524.updated/kernel/cpu.c
--- linux-2.5.19.29524/kernel/cpu.c	Thu Jan  1 10:00:00 1970
+++ linux-2.5.19.29524.updated/kernel/cpu.c	Thu May 30 11:41:17 2002
@@ -0,0 +1,64 @@
+/* CPU control.
+ * (C) 2001 Rusty Russell
+ * This code is licenced under the GPL.
+ */
+#include <linux/proc_fs.h>
+#include <linux/smp.h>
+#include <linux/init.h>
+#include <linux/notifier.h>
+#include <linux/sched.h>
+#include <linux/unistd.h>
+#include <asm/semaphore.h>
+
+/* This protects CPUs going up and down... */
+DECLARE_MUTEX(cpucontrol);
+
+static struct notifier_block *cpu_chain = NULL;
+
+/* Need to know about CPUs going up/down? */
+int register_cpu_notifier(struct notifier_block *nb)
+{
+	return notifier_chain_register(&cpu_chain, nb);
+}
+
+void unregister_cpu_notifier(struct notifier_block *nb)
+{
+	notifier_chain_unregister(&cpu_chain,nb);
+}
+
+int __init cpu_up(unsigned int cpu)
+{
+	int ret;
+
+	if ((ret = down_interruptible(&cpucontrol)) != 0) 
+		return ret;
+
+	if (cpu_online(cpu)) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	/* Arch-specific enabling code. */
+	ret = __cpu_up(cpu);
+	if (ret != 0) goto out;
+	if (!cpu_online(cpu))
+		BUG();
+
+	printk("CPU %u IS NOW UP!\n", cpu);
+	/* Now call notifier in preparation. */
+	notifier_call_chain(&cpu_chain, CPU_ONLINE, (void *)cpu);
+
+	printk("Synchronizing kernel!\n");
+	/* Friendly to make sure everyone knows it's up before we
+	   return */
+#if 0
+	sync_kernel();
+#else
+	schedule_timeout(HZ);
+#endif
+
+ out:
+	printk("UP %u done: %i.\n", cpu, ret);
+	up(&cpucontrol);
+	return ret;
+}
diff -urN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.19.29524/kernel/sched.c linux-2.5.19.29524.updated/kernel/sched.c
--- linux-2.5.19.29524/kernel/sched.c	Thu May 30 11:41:09 2002
+++ linux-2.5.19.29524.updated/kernel/sched.c	Thu May 30 11:41:17 2002
@@ -26,6 +26,8 @@
 #include <linux/interrupt.h>
 #include <linux/completion.h>
 #include <linux/kernel_stat.h>
+#include <linux/notifier.h>
+#include <linux/delay.h>
 
 /*
  * Convert user-nice values [ -20 ... 0 ... 19 ]
@@ -501,6 +503,11 @@
 	prio_array_t *array;
 	list_t *head, *curr;
 
+	/* CPU going down is a special case: we don't pull more tasks
+	   onboard */
+	if (unlikely(!cpu_online(this_cpu)))
+		return;
+
 	/*
 	 * We search all runqueues to find the most busy one.
 	 * We do this lockless to reduce cache-bouncing overhead,
@@ -1670,9 +1677,11 @@
 	migration_req_t req;
 	runqueue_t *rq;
 
+#if 0 /* FIXME: Grab cpu_lock, return error on this case. --RR */
 	new_mask &= cpu_online_map;
 	if (!new_mask)
 		BUG();
+#endif
 
 	preempt_disable();
 	rq = task_rq_lock(p, &flags);
@@ -1718,17 +1727,6 @@
 	sigfillset(&current->blocked);
 	set_fs(KERNEL_DS);
 
-	/* FIXME: First CPU may not be zero, but this crap code
-           vanishes with hotplug cpu patch anyway. --RR */
-	/*
-	 * The first migration thread is started on CPU #0. This one can migrate
-	 * the other migration threads to their destination CPUs.
-	 */
-	if (cpu != 0) {
-		while (!cpu_rq(0)->migration_thread)
-			yield();
-		set_cpus_allowed(current, 1UL << cpu);
-	}
 	printk("migration_task %d on cpu=%d\n",cpu,smp_processor_id());
 	ret = setscheduler(0, SCHED_FIFO, &param);
 
@@ -1806,4 +1804,36 @@
 			schedule_timeout(2);
 	}
 }
+static int migration_call(struct notifier_block *nfb,
+			  unsigned long action,
+			  void *hcpu)
+{
+	printk("Migration hotplug action %lu (cpu %li)\n",
+	       action, (long)hcpu);
+	switch (action) {
+	case CPU_ONLINE:
+		printk("Starting migration thread for cpu %li\n",
+		       (long)hcpu);
+		kernel_thread(migration_thread, hcpu,
+			      CLONE_FS | CLONE_FILES | CLONE_SIGNAL);
+		break;
+	case CPU_OFFLINE:
+		BUG();
+		break;
+	}
+	return NOTIFY_OK;
+}
+
+static struct notifier_block migration_notifier = { &migration_call, NULL, 0 };
+
+int __init migration_init(void)
+{
+	/* Start one for boot CPU. */
+	kernel_thread(migration_thread, (void *)smp_processor_id(),
+		      CLONE_FS | CLONE_FILES | CLONE_SIGNAL);
+	register_cpu_notifier(&migration_notifier);
+	return 0;
+}
+
+__initcall(migration_init);
 #endif
diff -urN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.19.29524/kernel/softirq.c linux-2.5.19.29524.updated/kernel/softirq.c
--- linux-2.5.19.29524/kernel/softirq.c	Thu May 30 11:41:09 2002
+++ linux-2.5.19.29524.updated/kernel/softirq.c	Thu May 30 11:41:17 2002
@@ -17,6 +17,7 @@
 #include <linux/init.h>
 #include <linux/tqueue.h>
 #include <linux/percpu.h>
+#include <linux/notifier.h>
 
 /*
    - No shared variables, all the data are CPU local.
@@ -395,20 +396,33 @@
 	}
 }
 
-static __init int spawn_ksoftirqd(void)
+static int __devinit softirq_cpu_callback(struct notifier_block *nfb,
+					  unsigned long action,
+					  void *hcpu)
 {
-	int cpu;
+	int hotcpu = (unsigned long) hcpu;
 
-	for (cpu = 0; cpu < NR_CPUS; cpu++) {
-		if (!cpu_online(cpu))
-			continue;
-		if (kernel_thread(ksoftirqd, (void *) (long) cpu,
+	if (action == CPU_ONLINE) {
+		if (kernel_thread(ksoftirqd, (void *) (long) hotcpu,
 				  CLONE_FS | CLONE_FILES | CLONE_SIGNAL) < 0)
-			printk("spawn_ksoftirqd() failed for cpu %d\n", cpu);
-		else
-			while (!ksoftirqd_task(cpu))
-				yield();
-	}
+			return NOTIFY_BAD;
+
+		while (!ksoftirqd_task(hotcpu))
+			yield();
+		return NOTIFY_OK;
+ 	}
+	return NOTIFY_BAD;
+}
+
+static struct notifier_block cpu_callback_nfb
+= { &softirq_cpu_callback, NULL, 0 };
+
+static __init int spawn_ksoftirqd(void)
+{
+	if (kernel_thread(ksoftirqd, (void *) (long) smp_processor_id(),
+			  CLONE_FS | CLONE_FILES | CLONE_SIGNAL) < 0)
+		printk("spawn_ksoftirqd() failed\n");
+	register_cpu_notifier(&cpu_callback_nfb);
 	return 0;
 }
 
}