Name: Initial kmalloc_percpu patch Author: Rusty Russell Status: Experimental D: This patch allows an unlimited number of per-cpu allocations, with D: some caveats. It is more complex than the "simply allocate a big D: chunk up front" approach of Misc/kmalloc_percpu.patch.gz. diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .8622-2.5.66-bk2-kmalloc_percpu.pre/arch/ia64/kernel/setup.c .8622-2.5.66-bk2-kmalloc_percpu/arch/ia64/kernel/setup.c --- .8622-2.5.66-bk2-kmalloc_percpu.pre/arch/ia64/kernel/setup.c 2003-03-18 12:21:31.000000000 +1100 +++ .8622-2.5.66-bk2-kmalloc_percpu/arch/ia64/kernel/setup.c 2003-03-27 18:31:19.000000000 +1100 @@ -660,6 +660,8 @@ cpu_init (void) __per_cpu_offset[cpu] = (char *) cpu_data - __per_cpu_start; cpu_data += PERCPU_PAGE_SIZE; } + percpu_block_size = PERCPU_PAGE_SIZE; + percpu_base_offset = __per_cpu_offset[0]; } cpu_data = __per_cpu_start + __per_cpu_offset[smp_processor_id()]; #else /* !CONFIG_SMP */ diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .8622-2.5.66-bk2-kmalloc_percpu.pre/include/asm-generic/percpu.h .8622-2.5.66-bk2-kmalloc_percpu/include/asm-generic/percpu.h --- .8622-2.5.66-bk2-kmalloc_percpu.pre/include/asm-generic/percpu.h 2003-01-02 12:32:47.000000000 +1100 +++ .8622-2.5.66-bk2-kmalloc_percpu/include/asm-generic/percpu.h 2003-03-27 18:31:19.000000000 +1100 @@ -17,18 +17,11 @@ extern unsigned long __per_cpu_offset[NR #define per_cpu(var, cpu) (*RELOC_HIDE(&var##__per_cpu, __per_cpu_offset[cpu])) #define __get_cpu_var(var) per_cpu(var, smp_processor_id()) -#else /* ! SMP */ - -/* Can't define per-cpu variables in modules. Sorry --RR */ -#ifndef MODULE -#define DEFINE_PER_CPU(type, name) \ - __typeof__(type) name##__per_cpu -#endif - -#define per_cpu(var, cpu) ((void)cpu, var##__per_cpu) -#define __get_cpu_var(var) var##__per_cpu +#define per_cpu_ptr(ptr, cpu) ((__typeof__(ptr))(RELOC_HIDE(ptr, __per_cpu_offset[cpu]))) +#define __get_cpu_ptr(var) per_cpu_ptr(ptr, smp_processor_id()) -#endif /* SMP */ +void setup_per_cpu_areas(void); +#endif /* SMP */ #define DECLARE_PER_CPU(type, name) extern __typeof__(type) name##__per_cpu diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .8622-2.5.66-bk2-kmalloc_percpu.pre/include/linux/genhd.h .8622-2.5.66-bk2-kmalloc_percpu/include/linux/genhd.h --- .8622-2.5.66-bk2-kmalloc_percpu.pre/include/linux/genhd.h 2003-03-18 12:21:40.000000000 +1100 +++ .8622-2.5.66-bk2-kmalloc_percpu/include/linux/genhd.h 2003-03-27 18:31:19.000000000 +1100 @@ -164,7 +164,7 @@ static inline void disk_stat_set_all(str #ifdef CONFIG_SMP static inline int init_disk_stats(struct gendisk *disk) { - disk->dkstats = kmalloc_percpu(sizeof (struct disk_stats), GFP_KERNEL); + disk->dkstats = kmalloc_percpu(struct disk_stats); if (!disk->dkstats) return 0; disk_stat_set_all(disk, 0); diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .8622-2.5.66-bk2-kmalloc_percpu.pre/include/linux/percpu.h .8622-2.5.66-bk2-kmalloc_percpu/include/linux/percpu.h --- .8622-2.5.66-bk2-kmalloc_percpu.pre/include/linux/percpu.h 2003-02-07 19:20:01.000000000 +1100 +++ .8622-2.5.66-bk2-kmalloc_percpu/include/linux/percpu.h 2003-03-27 18:31:19.000000000 +1100 @@ -1,42 +1,52 @@ #ifndef __LINUX_PERCPU_H #define __LINUX_PERCPU_H #include /* For preempt_disable() */ -#include /* For kmalloc_percpu() */ +#include /* For kmalloc() */ +#include +#include #include -/* Must be an lvalue. */ +/* Maximum size for kmalloc_percpu */ +#define PERCPU_MAX 4096 + +/* For variables declared with DECLARE_PER_CPU()/DEFINE_PER_CPU(). */ #define get_cpu_var(var) (*({ preempt_disable(); &__get_cpu_var(var); })) #define put_cpu_var(var) preempt_enable() +/* Also, per_cpu(var, cpu) to get another cpu's value. */ -#ifdef CONFIG_SMP - -struct percpu_data { - void *ptrs[NR_CPUS]; - void *blkp; -}; +/* For ptrs allocated with kmalloc_percpu */ +#define get_cpu_ptr(ptr) ({ preempt_disable(); __get_cpu_ptr(ptr); }) +#define put_cpu_ptr(ptr) preempt_enable() +/* Also, per_cpu_ptr(ptr, cpu) to get another cpu's value. */ -/* - * Use this to get to a cpu's version of the per-cpu object allocated using - * kmalloc_percpu. If you want to get "this cpu's version", maybe you want - * to use get_cpu_ptr... - */ -#define per_cpu_ptr(ptr, cpu) \ -({ \ - struct percpu_data *__p = (struct percpu_data *)~(unsigned long)(ptr); \ - (__typeof__(ptr))__p->ptrs[(cpu)]; \ -}) +#ifdef CONFIG_SMP -extern void *kmalloc_percpu(size_t size, int flags); +/* Most are defined in asm/percpu.h */ +extern void *__kmalloc_percpu(size_t size, size_t align); extern void kfree_percpu(const void *); extern void kmalloc_percpu_init(void); -#else /* CONFIG_SMP */ +extern unsigned long percpu_base_offset, percpu_block_size; -#define per_cpu_ptr(ptr, cpu) (ptr) +#else /* !CONFIG_SMP */ -static inline void *kmalloc_percpu(size_t size, int flags) +/* Can't define per-cpu variables in modules. Sorry --RR */ +#ifndef MODULE +#define DEFINE_PER_CPU(type, name) \ + __typeof__(type) name##__per_cpu +#endif + +#define per_cpu(var, cpu) ((void)(cpu), var##__per_cpu) +#define __get_cpu_var(var) var##__per_cpu +#define per_cpu_ptr(ptr, cpu) ((void)(cpu), (ptr)) +#define __get_cpu_ptr(ptr) (ptr) + +static inline void *__kmalloc_percpu(size_t size, size_t align) { - return(kmalloc(size, flags)); + /* kmalloc always cacheline aligns. */ + BUG_ON(align > SMP_CACHE_BYTES); + BUG_ON(size > PERCPU_MAX); + return kmalloc(size, GFP_KERNEL); } static inline void kfree_percpu(const void *ptr) { @@ -46,26 +56,8 @@ static inline void kmalloc_percpu_init(v #endif /* CONFIG_SMP */ -/* - * Use these with kmalloc_percpu. If - * 1. You want to operate on memory allocated by kmalloc_percpu (dereference - * and read/modify/write) AND - * 2. You want "this cpu's version" of the object AND - * 3. You want to do this safely since: - * a. On multiprocessors, you don't want to switch between cpus after - * you've read the current processor id due to preemption -- this would - * take away the implicit advantage to not have any kind of traditional - * serialization for per-cpu data - * b. On uniprocessors, you don't want another kernel thread messing - * up with the same per-cpu data due to preemption - * - * So, Use get_cpu_ptr to disable preemption and get pointer to the - * local cpu version of the per-cpu object. Use put_cpu_ptr to enable - * preemption. Operations on per-cpu data between get_ and put_ is - * then considered to be safe. And ofcourse, "Thou shalt not sleep between - * get_cpu_ptr and put_cpu_ptr" - */ -#define get_cpu_ptr(ptr) per_cpu_ptr(ptr, get_cpu()) -#define put_cpu_ptr(ptr) put_cpu() +/* Simple wrapper for the common case. */ +#define kmalloc_percpu(type) \ + ((type *)(__kmalloc_percpu(sizeof(type), __alignof__(type)))) #endif /* __LINUX_PERCPU_H */ diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .8622-2.5.66-bk2-kmalloc_percpu.pre/init/main.c .8622-2.5.66-bk2-kmalloc_percpu/init/main.c --- .8622-2.5.66-bk2-kmalloc_percpu.pre/init/main.c 2003-03-18 12:21:40.000000000 +1100 +++ .8622-2.5.66-bk2-kmalloc_percpu/init/main.c 2003-03-27 18:31:19.000000000 +1100 @@ -305,30 +305,6 @@ static inline void smp_prepare_cpus(unsi #else -#ifdef __GENERIC_PER_CPU -unsigned long __per_cpu_offset[NR_CPUS]; - -static void __init setup_per_cpu_areas(void) -{ - unsigned long size, i; - char *ptr; - /* Created by linker magic */ - extern char __per_cpu_start[], __per_cpu_end[]; - - /* Copy section for each CPU (we discard the original) */ - size = ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES); - if (!size) - return; - - ptr = alloc_bootmem(size * NR_CPUS); - - for (i = 0; i < NR_CPUS; i++, ptr += size) { - __per_cpu_offset[i] = ptr - __per_cpu_start; - memcpy(ptr, __per_cpu_start, size); - } -} -#endif /* !__GENERIC_PER_CPU */ - /* Called by boot processor to activate the rest. */ static void __init smp_init(void) { diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .8622-2.5.66-bk2-kmalloc_percpu.pre/kernel/ksyms.c .8622-2.5.66-bk2-kmalloc_percpu/kernel/ksyms.c --- .8622-2.5.66-bk2-kmalloc_percpu.pre/kernel/ksyms.c 2003-03-25 12:17:31.000000000 +1100 +++ .8622-2.5.66-bk2-kmalloc_percpu/kernel/ksyms.c 2003-03-27 18:31:19.000000000 +1100 @@ -98,7 +98,7 @@ EXPORT_SYMBOL(remove_shrinker); EXPORT_SYMBOL(kmalloc); EXPORT_SYMBOL(kfree); #ifdef CONFIG_SMP -EXPORT_SYMBOL(kmalloc_percpu); +EXPORT_SYMBOL(__kmalloc_percpu); EXPORT_SYMBOL(kfree_percpu); #endif EXPORT_SYMBOL(vfree); diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .8622-2.5.66-bk2-kmalloc_percpu.pre/mm/Makefile .8622-2.5.66-bk2-kmalloc_percpu/mm/Makefile --- .8622-2.5.66-bk2-kmalloc_percpu.pre/mm/Makefile 2003-02-11 14:26:20.000000000 +1100 +++ .8622-2.5.66-bk2-kmalloc_percpu/mm/Makefile 2003-03-27 18:31:19.000000000 +1100 @@ -12,3 +12,4 @@ obj-y := bootmem.o filemap.o mempool.o slab.o swap.o truncate.o vcache.o vmscan.o $(mmu-y) obj-$(CONFIG_SWAP) += page_io.o swap_state.o swapfile.o +obj-$(CONFIG_SMP) += percpu.o diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .8622-2.5.66-bk2-kmalloc_percpu.pre/mm/percpu.c .8622-2.5.66-bk2-kmalloc_percpu/mm/percpu.c --- .8622-2.5.66-bk2-kmalloc_percpu.pre/mm/percpu.c 1970-01-01 10:00:00.000000000 +1000 +++ .8622-2.5.66-bk2-kmalloc_percpu/mm/percpu.c 2003-03-27 18:31:19.000000000 +1100 @@ -0,0 +1,391 @@ +/* + * Dynamic per-cpu allocation. + * Originally by Dipankar Sarma + * This version (C) 2003 Rusty Russell, IBM Corporation. + */ + +/* Simple linked list allocator: we don't stress it hard, but do want + it space-efficient. We keep the bookkeeping separately. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static DECLARE_MUTEX(pcpu_lock); +static LIST_HEAD(pcpu_blocks); +unsigned long percpu_base_offset, percpu_block_size; + +struct pcpu_block +{ + struct list_head list; + + void *base_ptr; + + /* Number of blocks used and allocated. */ + unsigned short num_used, num_allocated; + + /* Size of each block. -ve means used. */ + int size[0]; +}; + +static struct pcpu_block *split_block(struct pcpu_block *b, + unsigned int i, + unsigned short size) +{ + /* Reallocation required? */ + if (b->num_used + 1 > b->num_allocated) { + struct pcpu_block *new; + + new = kmalloc(sizeof(*b) + + sizeof(b->size[0]) * b->num_allocated*2, + GFP_KERNEL); + if (!new) + return NULL; + new->base_ptr = b->base_ptr; + new->num_used = b->num_used; + new->num_allocated = b->num_allocated * 2; + memcpy(new->size, b->size, sizeof(b->size[0])*b->num_used); + list_del(&b->list); + list_add(&new->list, &pcpu_blocks); + kfree(b); + b = new; + } + + /* Insert a new subblock */ + memmove(&b->size[i+1], &b->size[i], + sizeof(b->size[0]) * (b->num_used - i)); + b->num_used++; + + b->size[i+1] -= size; + b->size[i] = size; + return b; +} + +#define CPU_INIT_BLOCKS 4 + +static int new_block(void) +{ + struct pcpu_block *b; + + b = kmalloc(sizeof(*b) + CPU_INIT_BLOCKS*sizeof(int), GFP_KERNEL); + if (!b) + return 0; + + /* FIXME: vmallocing here is *horrible*: increase kmalloc limit, + or drop NR_CPUS. */ + if (percpu_block_size * NR_CPUS > 128*1024) + b->base_ptr = vmalloc(percpu_block_size * NR_CPUS); + else + b->base_ptr = kmalloc(percpu_block_size * NR_CPUS, GFP_KERNEL); + + if (!b->base_ptr) { + kfree(b); + return 0; + } + + b->num_allocated = CPU_INIT_BLOCKS; + b->num_used = 1; + b->size[0] = percpu_block_size; + + down(&pcpu_lock); + list_add(&b->list, &pcpu_blocks); + up(&pcpu_lock); + return 1; +} + +static inline unsigned int abs(int val) +{ + if (val < 0) + return -val; + return val; +} + +void *__kmalloc_percpu(size_t size, size_t align) +{ + struct pcpu_block *b; + unsigned long extra; + void *ret; + + BUG_ON(align > SMP_CACHE_BYTES); + BUG_ON(size > PERCPU_MAX); + + again: + down(&pcpu_lock); + list_for_each_entry(b, &pcpu_blocks, list) { + unsigned int i; + void *ptr = b->base_ptr; + + for (i = 0; i < b->num_used; ptr += abs(b->size[i]), i++) { + /* Extra for alignment requirement. */ + extra = ALIGN((unsigned long)ptr, align) + - (unsigned long)ptr; + BUG_ON(i == 0 && extra != 0); + + if (b->size[i] < 0 || b->size[i] < extra + size) + continue; + + /* Transfer extra to previous block. */ + if (b->size[i-1] < 0) + b->size[i-1] -= extra; + else + b->size[i-1] += extra; + b->size[i] -= extra; + ptr += extra; + + /* Split block if warranted */ + if (b->size[i] - size > sizeof(unsigned long)) { + struct pcpu_block *realloc; + realloc = split_block(b, i, size); + if (!realloc) + continue; + b = realloc; + } + + /* Mark allocated */ + b->size[i] = -b->size[i]; + /* Pointer will be offset by this: compensate. */ + ret = RELOC_HIDE(ptr, -percpu_base_offset); + goto done; + } + } + + if (!new_block()) + return NULL; + goto again; + + done: + up(&pcpu_lock); + return ret; +} + +static void free_block(struct pcpu_block *b, const void *freeme) +{ + unsigned int i; + void *ptr = b->base_ptr; + + for (i = 0; i < b->num_used; ptr += abs(b->size[i]), i++) { + if (ptr == freeme) { + b->size[i] = -b->size[i]; + goto free; + } + } + BUG(); + + free: + /* Merge with previous? */ + if (i > 0 && b->size[i-1] >= 0) { + b->size[i-1] += b->size[i]; + b->num_used--; + memmove(&b->size[i], &b->size[i+1], + (b->num_used - i) * sizeof(b->size[0])); + i--; + } + /* Merge with next? */ + if (i+1 < b->num_used && b->size[i+1] >= 0) { + b->size[i] += b->size[i+1]; + b->num_used--; + memmove(&b->size[i+1], &b->size[i+2], + (b->num_used - (i+1)) * sizeof(b->size[0])); + } + /* Empty? */ + if (b->num_used == 1) { + list_del(&b->list); + if (percpu_block_size * NR_CPUS > 128*1024) + vfree(b->base_ptr); + else + kfree(b->base_ptr); + kfree(b); + } +} + +void kfree_percpu(const void *ptr) +{ + struct pcpu_block *b; + + /* Pointer will be offset by this amount: compensate. */ + ptr = RELOC_HIDE(ptr, percpu_base_offset); + down(&pcpu_lock); + list_for_each_entry(b, &pcpu_blocks, list) { + if (b->base_ptr <= ptr && b->base_ptr+percpu_block_size > ptr){ + free_block(b, ptr); + up(&pcpu_lock); + return; + } + } + up(&pcpu_lock); + BUG(); +} + +#if 1 +#include + +static void check_values(unsigned int *ptrs[], + unsigned int sizes[], + unsigned int num) +{ + unsigned int cpu, i; + unsigned char *ptr; + + if (!ptrs[num]) + return; + + for (cpu = 0; cpu < NR_CPUS; cpu++) { + ptr = (unsigned char *)per_cpu_ptr(ptrs[num], cpu); + for (i = 0; i < sizes[num]; i++) { + if (ptr[i] != (unsigned char)(num + cpu)) + BUG(); + } + } +} + +static void check_blocklen(void) +{ + struct pcpu_block *i; + unsigned int count = 0; + + list_for_each_entry(i, &pcpu_blocks, list) + count++; + + printk("Total blocks = %u\n", count); +} + +static void dump_blocks(void) +{ + struct pcpu_block *i; + + list_for_each_entry(i, &pcpu_blocks, list) { + if (i->num_used != 1) { + printk("Block %p still has %u subs\n", i, i->num_used); + } + } +} + +static void mymemset(void *ptr, int c, unsigned long len) +{ + unsigned char *p = ptr; + while (len > 0) { + *p = c; + p++; + len--; + } +} + +static int random(void) +{ + unsigned short s; + + get_random_bytes(&s, sizeof(s)); + return s; +} + +static int test_percpu(void) +{ + unsigned int i; + unsigned int *ptr; + unsigned int *ptrs[PERCPU_MAX], sizes[PERCPU_MAX]; + + ptr = __kmalloc_percpu(4, 4); + printk("This cpu = %p (%u)\n", + __get_cpu_ptr(ptr), *__get_cpu_ptr(ptr)); + for (i = 0; i < NR_CPUS; i++) { + printk("&ptr[i] == %p (%u)\n", + per_cpu_ptr(ptr, i), *per_cpu_ptr(ptr, i)); + *per_cpu_ptr(ptr, i) = i; + } + kfree_percpu(ptr); + + for (i = 4; i < PERCPU_MAX; i+=27) { + unsigned int j; + ptrs[i] = __kmalloc_percpu(i, 4); + for (j = 0; j < NR_CPUS; j++) { + mymemset(per_cpu_ptr(ptrs[i], j), 0, i); + *per_cpu_ptr(ptrs[i], j) = i; + } + } + + for (i = 4; i < PERCPU_MAX; i+=27) { + unsigned int j; + for (j = 0; j < NR_CPUS; j++) + if (*per_cpu_ptr(ptrs[i], j) != i) + BUG(); + } + for (i = 4; i < PERCPU_MAX; i+=27) { + kfree_percpu(ptrs[i]); + ptrs[i] = NULL; + } + + /* Randomized test. */ + for (i = 0; i < 10000; i++) { + unsigned int j = random() % PERCPU_MAX; + if (!ptrs[j]) { + unsigned int cpu; + + sizes[j] = random() % PERCPU_MAX; + if (sizes[j] < 4) + sizes[j] = 4; + ptrs[j] = __kmalloc_percpu(sizes[j], 1<<(random()%L1_CACHE_SHIFT)); + + for (cpu = 0; cpu < NR_CPUS; cpu++) + memset(per_cpu_ptr(ptrs[j], cpu), j+cpu, + sizes[j]); + } else { + if (random() % 1000 == 0) { + printk("c\n"); + for (j = 0; j < PERCPU_MAX; j++) + check_values(ptrs, sizes, j); + } else { + check_values(ptrs, sizes, j); + kfree_percpu(ptrs[j]); + ptrs[j] = NULL; + } + } + if (i % (10000/10) == 0) + printk(".\n"); + } + check_blocklen(); + + for (i = 0; i < PERCPU_MAX; i++) { + if (ptrs[i]) { + kfree_percpu(ptrs[i]); + ptrs[i] = NULL; + } + } + dump_blocks(); +} +late_initcall(test_percpu); +#endif + +#ifdef __GENERIC_PER_CPU +unsigned long __per_cpu_offset[NR_CPUS]; +EXPORT_SYMBOL(__per_cpu_offset); + +void __init setup_per_cpu_areas(void) +{ + unsigned long i; + char *ptr; + /* Created by linker magic */ + extern char __per_cpu_start[], __per_cpu_end[]; + + /* Copy section for each CPU (we discard the original) */ + percpu_block_size = ALIGN(__per_cpu_end - __per_cpu_start, + SMP_CACHE_BYTES); + /* We guarantee at least this much. */ + if (percpu_block_size < PERCPU_MAX) + percpu_block_size = PERCPU_MAX; + + ptr = alloc_bootmem(percpu_block_size * NR_CPUS); + + for (i = 0; i < NR_CPUS; i++, ptr += percpu_block_size) { + __per_cpu_offset[i] = ptr - __per_cpu_start; + memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start); + } + percpu_base_offset = __per_cpu_offset[0]; + +} +#endif /* !__GENERIC_PER_CPU */ diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .8622-2.5.66-bk2-kmalloc_percpu.pre/mm/slab.c .8622-2.5.66-bk2-kmalloc_percpu/mm/slab.c --- .8622-2.5.66-bk2-kmalloc_percpu.pre/mm/slab.c 2003-03-25 12:17:32.000000000 +1100 +++ .8622-2.5.66-bk2-kmalloc_percpu/mm/slab.c 2003-03-27 18:31:19.000000000 +1100 @@ -1961,54 +1961,6 @@ void * kmalloc (size_t size, int flags) return NULL; } -#ifdef CONFIG_SMP -/** - * kmalloc_percpu - allocate one copy of the object for every present - * cpu in the system. - * Objects should be dereferenced using per_cpu_ptr/get_cpu_ptr - * macros only. - * - * @size: how many bytes of memory are required. - * @flags: the type of memory to allocate. - * The @flags argument may be one of: - * - * %GFP_USER - Allocate memory on behalf of user. May sleep. - * - * %GFP_KERNEL - Allocate normal kernel ram. May sleep. - * - * %GFP_ATOMIC - Allocation will not sleep. Use inside interrupt handlers. - */ -void * -kmalloc_percpu(size_t size, int flags) -{ - int i; - struct percpu_data *pdata = kmalloc(sizeof (*pdata), flags); - - if (!pdata) - return NULL; - - for (i = 0; i < NR_CPUS; i++) { - if (!cpu_possible(i)) - continue; - pdata->ptrs[i] = kmalloc(size, flags); - if (!pdata->ptrs[i]) - goto unwind_oom; - } - - /* Catch derefs w/o wrappers */ - return (void *) (~(unsigned long) pdata); - -unwind_oom: - while (--i >= 0) { - if (!cpu_possible(i)) - continue; - kfree(pdata->ptrs[i]); - } - kfree(pdata); - return NULL; -} -#endif - /** * kmem_cache_free - Deallocate an object * @cachep: The cache the allocation was from. @@ -2047,28 +1999,6 @@ void kfree (const void *objp) local_irq_restore(flags); } -#ifdef CONFIG_SMP -/** - * kfree_percpu - free previously allocated percpu memory - * @objp: pointer returned by kmalloc_percpu. - * - * Don't free memory not originally allocated by kmalloc_percpu() - * The complemented objp is to check for that. - */ -void -kfree_percpu(const void *objp) -{ - int i; - struct percpu_data *p = (struct percpu_data *) (~(unsigned long) objp); - - for (i = 0; i < NR_CPUS; i++) { - if (!cpu_possible(i)) - continue; - kfree(p->ptrs[i]); - } -} -#endif - unsigned int kmem_cache_size(kmem_cache_t *cachep) { unsigned int objlen = cachep->objsize; diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .8622-2.5.66-bk2-kmalloc_percpu.pre/net/ipv4/af_inet.c .8622-2.5.66-bk2-kmalloc_percpu/net/ipv4/af_inet.c --- .8622-2.5.66-bk2-kmalloc_percpu.pre/net/ipv4/af_inet.c 2003-03-25 12:17:32.000000000 +1100 +++ .8622-2.5.66-bk2-kmalloc_percpu/net/ipv4/af_inet.c 2003-03-27 18:31:19.000000000 +1100 @@ -1062,20 +1062,16 @@ static int __init init_ipv4_mibs(void) { int i; - net_statistics[0] = - kmalloc_percpu(sizeof (struct linux_mib), GFP_KERNEL); - net_statistics[1] = - kmalloc_percpu(sizeof (struct linux_mib), GFP_KERNEL); - ip_statistics[0] = kmalloc_percpu(sizeof (struct ip_mib), GFP_KERNEL); - ip_statistics[1] = kmalloc_percpu(sizeof (struct ip_mib), GFP_KERNEL); - icmp_statistics[0] = - kmalloc_percpu(sizeof (struct icmp_mib), GFP_KERNEL); - icmp_statistics[1] = - kmalloc_percpu(sizeof (struct icmp_mib), GFP_KERNEL); - tcp_statistics[0] = kmalloc_percpu(sizeof (struct tcp_mib), GFP_KERNEL); - tcp_statistics[1] = kmalloc_percpu(sizeof (struct tcp_mib), GFP_KERNEL); - udp_statistics[0] = kmalloc_percpu(sizeof (struct udp_mib), GFP_KERNEL); - udp_statistics[1] = kmalloc_percpu(sizeof (struct udp_mib), GFP_KERNEL); + net_statistics[0] = kmalloc_percpu(struct linux_mib); + net_statistics[1] = kmalloc_percpu(struct linux_mib); + ip_statistics[0] = kmalloc_percpu(struct ip_mib); + ip_statistics[1] = kmalloc_percpu(struct ip_mib); + icmp_statistics[0] = kmalloc_percpu(struct icmp_mib); + icmp_statistics[1] = kmalloc_percpu(struct icmp_mib); + tcp_statistics[0] = kmalloc_percpu(struct tcp_mib); + tcp_statistics[1] = kmalloc_percpu(struct tcp_mib); + udp_statistics[0] = kmalloc_percpu(struct udp_mib); + udp_statistics[1] = kmalloc_percpu(struct udp_mib); if (! (net_statistics[0] && net_statistics[1] && ip_statistics[0] && ip_statistics[1] && tcp_statistics[0] && tcp_statistics[1] diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .8622-2.5.66-bk2-kmalloc_percpu.pre/net/ipv4/route.c .8622-2.5.66-bk2-kmalloc_percpu/net/ipv4/route.c --- .8622-2.5.66-bk2-kmalloc_percpu.pre/net/ipv4/route.c 2003-03-25 12:17:33.000000000 +1100 +++ .8622-2.5.66-bk2-kmalloc_percpu/net/ipv4/route.c 2003-03-27 18:31:19.000000000 +1100 @@ -2655,8 +2655,7 @@ int __init ip_rt_init(void) ipv4_dst_ops.gc_thresh = (rt_hash_mask + 1); ip_rt_max_size = (rt_hash_mask + 1) * 16; - rt_cache_stat = kmalloc_percpu(sizeof (struct rt_cache_stat), - GFP_KERNEL); + rt_cache_stat = kmalloc_percpu(struct rt_cache_stat); if (!rt_cache_stat) goto out_enomem1; for (i = 0; i < NR_CPUS; i++) { diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .8622-2.5.66-bk2-kmalloc_percpu.pre/net/ipv6/af_inet6.c .8622-2.5.66-bk2-kmalloc_percpu/net/ipv6/af_inet6.c --- .8622-2.5.66-bk2-kmalloc_percpu.pre/net/ipv6/af_inet6.c 2003-03-25 12:17:33.000000000 +1100 +++ .8622-2.5.66-bk2-kmalloc_percpu/net/ipv6/af_inet6.c 2003-03-27 18:31:19.000000000 +1100 @@ -631,30 +631,24 @@ static int __init init_ipv6_mibs(void) { int i; - ipv6_statistics[0] = kmalloc_percpu(sizeof (struct ipv6_mib), - GFP_KERNEL); + ipv6_statistics[0] = kmalloc_percpu(struct ipv6_mib); if (!ipv6_statistics[0]) goto err_ip_mib0; - ipv6_statistics[1] = kmalloc_percpu(sizeof (struct ipv6_mib), - GFP_KERNEL); + ipv6_statistics[1] = kmalloc_percpu(struct ipv6_mib); if (!ipv6_statistics[1]) goto err_ip_mib1; - icmpv6_statistics[0] = kmalloc_percpu(sizeof (struct icmpv6_mib), - GFP_KERNEL); + icmpv6_statistics[0] = kmalloc_percpu(struct icmpv6_mib); if (!icmpv6_statistics[0]) goto err_icmp_mib0; - icmpv6_statistics[1] = kmalloc_percpu(sizeof (struct icmpv6_mib), - GFP_KERNEL); + icmpv6_statistics[1] = kmalloc_percpu(struct icmpv6_mib); if (!icmpv6_statistics[1]) goto err_icmp_mib1; - udp_stats_in6[0] = kmalloc_percpu(sizeof (struct udp_mib), - GFP_KERNEL); + udp_stats_in6[0] = kmalloc_percpu(struct udp_mib); if (!udp_stats_in6[0]) goto err_udp_mib0; - udp_stats_in6[1] = kmalloc_percpu(sizeof (struct udp_mib), - GFP_KERNEL); + udp_stats_in6[1] = kmalloc_percpu(struct udp_mib); if (!udp_stats_in6[1]) goto err_udp_mib1; diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .8622-2.5.66-bk2-kmalloc_percpu.pre/net/sctp/protocol.c .8622-2.5.66-bk2-kmalloc_percpu/net/sctp/protocol.c --- .8622-2.5.66-bk2-kmalloc_percpu.pre/net/sctp/protocol.c 2003-03-25 12:17:36.000000000 +1100 +++ .8622-2.5.66-bk2-kmalloc_percpu/net/sctp/protocol.c 2003-03-27 18:31:19.000000000 +1100 @@ -837,12 +837,10 @@ static int __init init_sctp_mibs(void) { int i; - sctp_statistics[0] = kmalloc_percpu(sizeof (struct sctp_mib), - GFP_KERNEL); + sctp_statistics[0] = kmalloc_percpu(struct sctp_mib); if (!sctp_statistics[0]) return -ENOMEM; - sctp_statistics[1] = kmalloc_percpu(sizeof (struct sctp_mib), - GFP_KERNEL); + sctp_statistics[1] = kmalloc_percpu(struct sctp_mib); if (!sctp_statistics[1]) { kfree_percpu(sctp_statistics[0]); return -ENOMEM;