Name: Make Generic irq_stat Structure Use per-cpu Author: Rusty Russell Status: Tested on 2.5.74 Depends: Percpu/local_softirq_pending.patch.gz Changes the generic irq_stat structure to use DECLARE_PER_CPU. This saves a little space, and is more efficient on some archs. Only i386 is fixed here. Some archs refer to this in assembler: this is easy on UP (append __per_cpu to the symbol name), but more complex on SMP (you need to add __per_cpu_offset[smp_processor_id()]). If that's too hard then you can just stop using the generic structure. Please remember to drop the ____cacheline_aligned from the typedef of irq_cpustat_t! Index: linux-2.6.10-rc2-bk13-Percpu/arch/i386/kernel/nmi.c =================================================================== --- linux-2.6.10-rc2-bk13-Percpu.orig/arch/i386/kernel/nmi.c 2004-10-19 14:33:50.000000000 +1000 +++ linux-2.6.10-rc2-bk13-Percpu/arch/i386/kernel/nmi.c 2004-12-02 11:39:54.014729824 +1100 @@ -110,7 +110,7 @@ printk(KERN_INFO "testing NMI watchdog ... "); for (cpu = 0; cpu < NR_CPUS; cpu++) - prev_nmi_count[cpu] = irq_stat[cpu].__nmi_count; + prev_nmi_count[cpu] = nmi_count(cpu); local_irq_enable(); mdelay((10*1000)/nmi_hz); // wait 10 ticks @@ -479,7 +479,7 @@ */ int sum, cpu = smp_processor_id(); - sum = irq_stat[cpu].apic_timer_irqs; + sum = __get_cpu_var(irq_stat).apic_timer_irqs; if (last_irq_sums[cpu] == sum) { /* Index: linux-2.6.10-rc2-bk13-Percpu/include/asm-ppc/hardirq.h =================================================================== --- linux-2.6.10-rc2-bk13-Percpu.orig/include/asm-ppc/hardirq.h 2004-11-16 15:30:03.000000000 +1100 +++ linux-2.6.10-rc2-bk13-Percpu/include/asm-ppc/hardirq.h 2004-12-02 11:39:54.014729824 +1100 @@ -19,7 +19,8 @@ #include /* Standard mappings for irq_cpustat_t above */ -#define last_jiffy_stamp(cpu) __IRQ_STAT((cpu), __last_jiffy_stamp) +#define local_last_jiffy_stamp() __LOCAL_IRQ_STAT(__last_jiffy_stamp) +#define last_jiffy_stamp(cpu) per_cpu(irq_stat, (cpu)).__last_jiffy_stamp static inline void ack_bad_irq(int irq) { Index: linux-2.6.10-rc2-bk13-Percpu/arch/i386/kernel/irq.c =================================================================== --- linux-2.6.10-rc2-bk13-Percpu.orig/arch/i386/kernel/irq.c 2004-11-16 15:29:02.000000000 +1100 +++ linux-2.6.10-rc2-bk13-Percpu/arch/i386/kernel/irq.c 2004-12-02 11:45:15.354878696 +1100 @@ -246,7 +246,7 @@ for (j = 0; j < NR_CPUS; j++) if (cpu_online(j)) seq_printf(p, "%10u ", - irq_stat[j].apic_timer_irqs); + __IRQ_STAT(j, apic_timer_irqs)); seq_putc(p, '\n'); #endif seq_printf(p, "ERR: %10u\n", atomic_read(&irq_err_count)); Index: linux-2.6.10-rc2-bk13-Percpu/arch/x86_64/kernel/nmi.c =================================================================== --- linux-2.6.10-rc2-bk13-Percpu.orig/arch/x86_64/kernel/nmi.c 2004-11-16 15:29:16.000000000 +1100 +++ linux-2.6.10-rc2-bk13-Percpu/arch/x86_64/kernel/nmi.c 2004-12-02 11:39:54.015729672 +1100 @@ -125,17 +125,17 @@ printk(KERN_INFO "testing NMI watchdog ... "); for (cpu = 0; cpu < NR_CPUS; cpu++) - counts[cpu] = cpu_pda[cpu].__nmi_count; + counts[cpu] = nmi_count(cpu); local_irq_enable(); mdelay((10*1000)/nmi_hz); // wait 10 ticks for (cpu = 0; cpu < NR_CPUS; cpu++) { if (!cpu_online(cpu)) continue; - if (cpu_pda[cpu].__nmi_count - counts[cpu] <= 5) { + if (nmi_count(cpu) - counts[cpu] <= 5) { printk("CPU#%d: NMI appears to be stuck (%d)!\n", cpu, - cpu_pda[cpu].__nmi_count); + nmi_count(cpu)); nmi_active = 0; lapic_nmi_owner &= ~LAPIC_NMI_WATCHDOG; return -1; Index: linux-2.6.10-rc2-bk13-Percpu/arch/i386/kernel/traps.c =================================================================== --- linux-2.6.10-rc2-bk13-Percpu.orig/arch/i386/kernel/traps.c 2004-11-30 12:45:10.000000000 +1100 +++ linux-2.6.10-rc2-bk13-Percpu/arch/i386/kernel/traps.c 2004-12-02 11:39:54.016729520 +1100 @@ -624,7 +624,7 @@ nmi_enter(); cpu = smp_processor_id(); - ++nmi_count(cpu); + ++local_nmi_count(); if (!nmi_callback(regs, cpu)) default_do_nmi(regs); Index: linux-2.6.10-rc2-bk13-Percpu/include/linux/irq_cpustat.h =================================================================== --- linux-2.6.10-rc2-bk13-Percpu.orig/include/linux/irq_cpustat.h 2004-12-02 11:37:52.084266072 +1100 +++ linux-2.6.10-rc2-bk13-Percpu/include/linux/irq_cpustat.h 2004-12-02 11:43:55.159070312 +1100 @@ -10,6 +10,7 @@ */ #include +#include /* * Simple wrappers reducing source bloat. Define all irq_stat fields @@ -18,14 +19,16 @@ */ #ifndef __ARCH_IRQ_STAT -extern irq_cpustat_t irq_stat[]; /* defined in asm/hardirq.h */ -#define __IRQ_STAT(cpu, member) (irq_stat[cpu].member) +DECLARE_PER_CPU(irq_cpustat_t, irq_stat); /* defined in asm/hardirq.h */ +#define __LOCAL_IRQ_STAT(member) __get_cpu_var(irq_stat).member +#define __IRQ_STAT(cpu, member) per_cpu(irq_stat, (cpu)).member #endif /* arch independent irq_stat fields */ -#define local_softirq_pending() __IRQ_STAT(smp_processor_id(), __softirq_pending) +#define local_softirq_pending() __LOCAL_IRQ_STAT(__softirq_pending) /* arch dependent irq_stat fields */ -#define nmi_count(cpu) __IRQ_STAT((cpu), __nmi_count) /* i386 */ +#define local_nmi_count() __LOCAL_IRQ_STAT(__nmi_count) +#define nmi_count(cpu) __IRQ_STAT((cpu), __nmi_count) #endif /* __irq_cpustat_h */ Index: linux-2.6.10-rc2-bk13-Percpu/kernel/softirq.c =================================================================== --- linux-2.6.10-rc2-bk13-Percpu.orig/kernel/softirq.c 2004-11-16 15:30:10.000000000 +1100 +++ linux-2.6.10-rc2-bk13-Percpu/kernel/softirq.c 2004-12-02 11:39:54.016729520 +1100 @@ -37,8 +37,8 @@ */ #ifndef __ARCH_IRQ_STAT -irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned; -EXPORT_SYMBOL(irq_stat); +DEFINE_PER_CPU(irq_cpustat_t, irq_stat); +EXPORT_PER_CPU_SYMBOL(irq_stat); #endif static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp; Index: linux-2.6.10-rc2-bk13-Percpu/arch/ppc/platforms/pmac_smp.c =================================================================== --- linux-2.6.10-rc2-bk13-Percpu.orig/arch/ppc/platforms/pmac_smp.c 2004-11-16 15:29:10.000000000 +1100 +++ linux-2.6.10-rc2-bk13-Percpu/arch/ppc/platforms/pmac_smp.c 2004-12-02 11:39:54.016729520 +1100 @@ -587,7 +587,7 @@ set_dec(tb_ticks_per_jiffy); set_tb(0, 0); - last_jiffy_stamp(smp_processor_id()) = 0; + local_last_jiffy_stamp() = 0; mb(); sec_tb_reset = 1; @@ -608,7 +608,7 @@ set_dec(tb_ticks_per_jiffy); set_tb(0, 0); - last_jiffy_stamp(smp_processor_id()) = 0; + local_last_jiffy_stamp() = 0; mb(); /* Now, restart the timebase by leaving the GPIO to an open collector */ Index: linux-2.6.10-rc2-bk13-Percpu/arch/i386/kernel/io_apic.c =================================================================== --- linux-2.6.10-rc2-bk13-Percpu.orig/arch/i386/kernel/io_apic.c 2004-11-16 15:29:02.000000000 +1100 +++ linux-2.6.10-rc2-bk13-Percpu/arch/i386/kernel/io_apic.c 2004-12-02 11:39:54.018729216 +1100 @@ -274,7 +274,7 @@ #define IRQ_DELTA(cpu,irq) (irq_cpu_data[cpu].irq_delta[irq]) #define IDLE_ENOUGH(cpu,now) \ - (idle_cpu(cpu) && ((now) - irq_stat[(cpu)].idle_timestamp > 1)) + (idle_cpu(cpu) && ((now) - per_cpu(irq_stat,(cpu)).idle_timestamp > 1)) #define IRQ_ALLOWED(cpu, allowed_mask) cpu_isset(cpu, allowed_mask) Index: linux-2.6.10-rc2-bk13-Percpu/arch/ppc/kernel/time.c =================================================================== --- linux-2.6.10-rc2-bk13-Percpu.orig/arch/ppc/kernel/time.c 2004-11-16 15:29:09.000000000 +1100 +++ linux-2.6.10-rc2-bk13-Percpu/arch/ppc/kernel/time.c 2004-12-02 11:39:54.018729216 +1100 @@ -184,7 +184,7 @@ } if ( !disarm_decr[smp_processor_id()] ) set_dec(next_dec); - last_jiffy_stamp(cpu) = jiffy_stamp; + local_last_jiffy_stamp() = jiffy_stamp; #ifdef CONFIG_SMP smp_local_timer_interrupt(regs); @@ -260,7 +260,7 @@ * guarantees an error < 1 jiffy even if they are off by eons, * still reasonable when gettimeofday resolution is 1 jiffy. */ - tb_delta = tb_ticks_since(last_jiffy_stamp(smp_processor_id())); + tb_delta = tb_ticks_since(local_last_jiffy_stamp()); tb_delta += (jiffies - wall_jiffies) * tb_ticks_per_jiffy; new_nsec -= 1000 * mulhwu(tb_to_us, tb_delta); @@ -334,7 +334,7 @@ /* No update now, we just read the time from the RTC ! */ last_rtc_update = xtime.tv_sec; } - last_jiffy_stamp(0) = tb_last_stamp = stamp; + local_last_jiffy_stamp() = tb_last_stamp = stamp; /* Not exact, but the timer interrupt takes care of this */ set_dec(tb_ticks_per_jiffy); Index: linux-2.6.10-rc2-bk13-Percpu/arch/i386/kernel/apic.c =================================================================== --- linux-2.6.10-rc2-bk13-Percpu.orig/arch/i386/kernel/apic.c 2004-11-30 12:45:10.000000000 +1100 +++ linux-2.6.10-rc2-bk13-Percpu/arch/i386/kernel/apic.c 2004-12-02 11:39:54.019729064 +1100 @@ -1151,12 +1151,10 @@ fastcall void smp_apic_timer_interrupt(struct pt_regs *regs) { - int cpu = smp_processor_id(); - /* * the NMI deadlock-detector uses this. */ - irq_stat[cpu].apic_timer_irqs++; + __get_cpu_var(irq_stat).apic_timer_irqs++; /* * NOTE! We'd better ACK the irq immediately, Index: linux-2.6.10-rc2-bk13-Percpu/arch/ppc/platforms/pmac_cpufreq.c =================================================================== --- linux-2.6.10-rc2-bk13-Percpu.orig/arch/ppc/platforms/pmac_cpufreq.c 2004-11-30 12:45:10.000000000 +1100 +++ linux-2.6.10-rc2-bk13-Percpu/arch/ppc/platforms/pmac_cpufreq.c 2004-12-02 11:39:54.019729064 +1100 @@ -96,7 +96,7 @@ /* No currently-supported powerbook has a 601, * so use get_tbl, not native */ - last_jiffy_stamp(0) = tb_last_stamp = get_tbl(); + local_last_jiffy_stamp() = tb_last_stamp = get_tbl(); } #ifdef DEBUG_FREQ Index: linux-2.6.10-rc2-bk13-Percpu/drivers/macintosh/via-pmu.c =================================================================== --- linux-2.6.10-rc2-bk13-Percpu.orig/drivers/macintosh/via-pmu.c 2004-11-30 12:45:13.000000000 +1100 +++ linux-2.6.10-rc2-bk13-Percpu/drivers/macintosh/via-pmu.c 2004-12-02 11:39:54.021728760 +1100 @@ -595,7 +595,7 @@ /* No currently-supported powerbook has a 601, * so use get_tbl, not native */ - last_jiffy_stamp(0) = tb_last_stamp = get_tbl(); + local_last_jiffy_stamp() = tb_last_stamp = get_tbl(); } #endif Index: linux-2.6.10-rc2-bk13-Percpu/arch/i386/kernel/process.c =================================================================== --- linux-2.6.10-rc2-bk13-Percpu.orig/arch/i386/kernel/process.c 2004-11-16 15:29:02.000000000 +1100 +++ linux-2.6.10-rc2-bk13-Percpu/arch/i386/kernel/process.c 2004-12-02 11:39:54.021728760 +1100 @@ -157,7 +157,7 @@ if (!idle) idle = default_idle; - irq_stat[smp_processor_id()].idle_timestamp = jiffies; + __get_cpu_var(irq_stat).idle_timestamp = jiffies; idle(); rcu_read_unlock(); } Index: linux-2.6.10-rc2-bk13-Percpu/include/asm-i386/hardirq.h =================================================================== --- linux-2.6.10-rc2-bk13-Percpu.orig/include/asm-i386/hardirq.h 2004-11-16 15:29:59.000000000 +1100 +++ linux-2.6.10-rc2-bk13-Percpu/include/asm-i386/hardirq.h 2004-12-02 11:39:54.022728608 +1100 @@ -10,7 +10,7 @@ unsigned long idle_timestamp; unsigned int __nmi_count; /* arch dependent */ unsigned int apic_timer_irqs; /* arch dependent */ -} ____cacheline_aligned irq_cpustat_t; +} irq_cpustat_t; #include /* Standard mappings for irq_cpustat_t above */ Index: linux-2.6.10-rc2-bk13-Percpu/include/asm-x86_64/hardirq.h =================================================================== --- linux-2.6.10-rc2-bk13-Percpu.orig/include/asm-x86_64/hardirq.h 2004-11-16 15:30:06.000000000 +1100 +++ linux-2.6.10-rc2-bk13-Percpu/include/asm-x86_64/hardirq.h 2004-12-02 11:39:54.022728608 +1100 @@ -11,7 +11,7 @@ /* Generate a lvalue for a pda member. Should fix softirq.c instead to use special access macros. This would generate better code. */ -#define __IRQ_STAT(cpu,member) (read_pda(me)->member) +#define __LOCAL_IRQ_STAT(member) (read_pda(me)->member) #include /* Standard mappings for irq_cpustat_t above */