diff -ur --exclude-from /home/axboe/exclude /opt/kernel/linux-2.4.2-pre1/drivers/block/ll_rw_blk.c linux/drivers/block/ll_rw_blk.c --- /opt/kernel/linux-2.4.2-pre1/drivers/block/ll_rw_blk.c Mon Jan 29 21:11:32 2001 +++ linux/drivers/block/ll_rw_blk.c Sat Feb 3 16:14:01 2001 @@ -1247,9 +1247,6 @@ #ifdef CONFIG_BLK_DEV_RAM rd_init(); #endif -#ifdef CONFIG_BLK_DEV_LOOP - loop_init(); -#endif #ifdef CONFIG_ISP16_CDI isp16_init(); #endif diff -ur --exclude-from /home/axboe/exclude /opt/kernel/linux-2.4.2-pre1/drivers/block/loop.c linux/drivers/block/loop.c --- /opt/kernel/linux-2.4.2-pre1/drivers/block/loop.c Fri Dec 29 23:07:21 2000 +++ linux/drivers/block/loop.c Mon Feb 5 16:43:30 2001 @@ -31,11 +31,12 @@ * max_loop=<1-255> to the kernel on boot. * Erik I. Bolsų, , Oct 31, 1999 * + * Rip out request handling and turn loop into a block remapper. + * Jens Axboe , Nov 2000 + * * Still To Fix: * - Advisory locking is ignored here. * - Should use an own CAP_* category instead of CAP_SYS_ADMIN - * - Should use the underlying filesystems/devices read function if possible - * to support read ahead (and for write) * * WARNING/FIXME: * - The block number as IV passing to low level transfer functions is broken: @@ -56,9 +57,12 @@ #include #include #include - +#include +#include +#include #include #include +#include #include @@ -66,40 +70,27 @@ #define MAJOR_NR LOOP_MAJOR -#define DEVICE_NAME "loop" -#define DEVICE_REQUEST do_lo_request -#define DEVICE_NR(device) (MINOR(device)) -#define DEVICE_ON(device) -#define DEVICE_OFF(device) -#define DEVICE_NO_RANDOM -#define TIMEOUT_VALUE (6 * HZ) -#include - #include static int max_loop = 8; static struct loop_device *loop_dev; static int *loop_sizes; static int *loop_blksizes; static devfs_handle_t devfs_handle; /* For the directory */ +static kmem_cache_t *loop_bhp; -#define FALSE 0 -#define TRUE (!FALSE) - -/* - * Transfer functions - */ static int transfer_none(struct loop_device *lo, int cmd, char *raw_buf, - char *loop_buf, int size, int real_block) + char *loop_buf, int size, int real_block) { if (cmd == READ) memcpy(loop_buf, raw_buf, size); else memcpy(raw_buf, loop_buf, size); + return 0; } static int transfer_xor(struct loop_device *lo, int cmd, char *raw_buf, - char *loop_buf, int size, int real_block) + char *loop_buf, int size, int real_block) { char *in, *out, *key; int i, keysize; @@ -111,17 +102,18 @@ in = loop_buf; out = raw_buf; } + key = lo->lo_encrypt_key; keysize = lo->lo_encrypt_key_size; - for (i=0; i < size; i++) + for (i = 0; i < size; i++) *out++ = *in++ ^ key[(i & 511) % keysize]; return 0; } static int none_status(struct loop_device *lo, struct loop_info *info) { - return 0; -} + return 0; +} static int xor_status(struct loop_device *lo, struct loop_info *info) { @@ -133,7 +125,7 @@ struct loop_func_table none_funcs = { number: LO_CRYPT_NONE, transfer: transfer_none, - init: none_status + init: none_status, }; struct loop_func_table xor_funcs = { @@ -168,8 +160,7 @@ loop_sizes[lo->lo_number] = size; } -static int lo_send(struct loop_device *lo, char *data, int len, loff_t pos, - int blksize) +static int lo_send(struct loop_device *lo, char *data, int len, loff_t pos) { struct file *file = lo->lo_backing_file; /* kudos to NFsckingS */ struct address_space *mapping = lo->lo_dentry->d_inode->i_mapping; @@ -182,7 +173,8 @@ index = pos >> PAGE_CACHE_SHIFT; offset = pos & (PAGE_CACHE_SIZE - 1); while (len > 0) { - int IV = index * (PAGE_CACHE_SIZE/blksize) + offset/blksize; + int IV = index * (PAGE_CACHE_SIZE/lo->lo_blksize) + offset/lo->lo_blksize; + IV >>= 2; size = PAGE_CACHE_SIZE - offset; if (size > len) size = len; @@ -193,7 +185,8 @@ if (aops->prepare_write(file, page, offset, offset+size)) goto unlock; kaddr = page_address(page); - if ((lo->transfer)(lo, WRITE, kaddr+offset, data, size, IV)) + flush_dcache_page(page); + if (lo_do_transfer(lo, WRITE, kaddr + offset, data, size, IV)) goto write_fail; if (aops->commit_write(file, page, offset, offset+size)) goto unlock; @@ -203,6 +196,7 @@ index++; pos += size; UnlockPage(page); + deactivate_page(page); page_cache_release(page); } return 0; @@ -213,6 +207,7 @@ kunmap(page); unlock: UnlockPage(page); + deactivate_page(page); page_cache_release(page); fail: return -1; @@ -221,7 +216,6 @@ struct lo_read_data { struct loop_device *lo; char *data; - int blksize; }; static int lo_read_actor(read_descriptor_t * desc, struct page *page, unsigned long offset, unsigned long size) @@ -230,16 +224,17 @@ unsigned long count = desc->count; struct lo_read_data *p = (struct lo_read_data*)desc->buf; struct loop_device *lo = p->lo; - int IV = page->index * (PAGE_CACHE_SIZE/p->blksize) + offset/p->blksize; + int IV = page->index * (PAGE_CACHE_SIZE/lo->lo_blksize) + offset/lo->lo_blksize; + + IV >>= 2; if (size > count) size = count; kaddr = kmap(page); - if ((lo->transfer)(lo,READ,kaddr+offset,p->data,size,IV)) { + if (lo_do_transfer(lo, READ, kaddr + offset, p->data, size, IV)) { size = 0; - printk(KERN_ERR "loop: transfer error block %ld\n", - page->index); + printk(KERN_ERR "loop: transfer error block %ld\n",page->index); desc->error = -EINVAL; } kunmap(page); @@ -250,8 +245,7 @@ return size; } -static int lo_receive(struct loop_device *lo, char *data, int len, loff_t pos, - int blksize) +static int lo_receive(struct loop_device *lo, char *data, int len, loff_t pos) { struct file *file = lo->lo_backing_file; struct lo_read_data cookie; @@ -259,7 +253,6 @@ cookie.lo = lo; cookie.data = data; - cookie.blksize = blksize; desc.written = 0; desc.count = len; desc.buf = (char*)&cookie; @@ -268,135 +261,278 @@ return desc.error; } -static void do_lo_request(request_queue_t * q) +static int do_bh_filebacked(struct loop_device *lo, struct buffer_head *bh, int rw) { - int block, offset, len, blksize, size; - char *dest_addr; - struct loop_device *lo; - struct buffer_head *bh; - struct request *current_request; loff_t pos; + int ret; + + pos = ((loff_t) bh->b_rsector << 9) + lo->lo_offset; + + if (rw == WRITE) + ret = lo_send(lo, bh->b_data, bh->b_size, pos); + else + ret = lo_receive(lo, bh->b_data, bh->b_size, pos); + + return ret; +} + +static void loop_put_buffer(struct buffer_head *bh) +{ + if (bh) { + kunmap(bh->b_page); + __free_page(bh->b_page); + kmem_cache_free(loop_bhp, bh); + } +} + +/* + * Add buffer_head to back of pending list + */ +static void loop_add_bh(struct loop_device *lo, struct buffer_head *bh) +{ + unsigned long flags; + + if (bh->b_reqnext) { + printk("loop: add_bh for bh with b_reqnext\n"); + bh->b_reqnext = NULL; + } + + spin_lock_irqsave(&lo->lo_lock, flags); + if (lo->lo_bhtail) { + lo->lo_bhtail->b_reqnext = bh; + lo->lo_bhtail = bh; + } else + lo->lo_bh = lo->lo_bhtail = bh; + spin_unlock_irqrestore(&lo->lo_lock, flags); + + wake_up_process(lo->lo_tsk); +} -repeat: - INIT_REQUEST; - current_request=CURRENT; - blkdev_dequeue_request(current_request); - if (MINOR(current_request->rq_dev) >= max_loop) - goto error_out; - lo = &loop_dev[MINOR(current_request->rq_dev)]; - if (!lo->lo_dentry || !lo->transfer) - goto error_out; - if (current_request->cmd == WRITE) { +/* + * Grab first pending buffer + */ +static struct buffer_head *loop_get_bh(struct loop_device *lo) +{ + struct buffer_head *bh; + + spin_lock_irq(&lo->lo_lock); + if ((bh = lo->lo_bh)) { + if (bh == lo->lo_bhtail) + lo->lo_bhtail = NULL; + lo->lo_bh = bh->b_reqnext; + bh->b_reqnext = NULL; + } + spin_unlock_irq(&lo->lo_lock); + + return bh; +} + +/* + * when buffer i/o has completed. if BH_Dirty is set, this was a WRITE + * and lo->transfer stuff has already been done. if not, it was a READ + * so queue it for the loop thread and let it do the transfer out of + * b_end_io context (we don't want to do decrypt of a page with irqs + * disabled) + */ +static void loop_end_io_transfer(struct buffer_head *bh, int uptodate) +{ + struct loop_device *lo = &loop_dev[MINOR(bh->b_dev)]; + + if (!uptodate || test_bit(BH_Dirty, &bh->b_state)) { + struct buffer_head *rbh = bh->b_private; + + rbh->b_end_io(rbh, uptodate); + loop_put_buffer(bh); + } else + loop_add_bh(lo, bh); +} + +static struct buffer_head *loop_get_buffer(struct loop_device *lo, + struct buffer_head *rbh) +{ + struct buffer_head *bh; + + do { + bh = kmem_cache_alloc(loop_bhp, SLAB_BUFFER); + if (bh) + break; + + run_task_queue(&tq_disk); + schedule_timeout(HZ); + } while (1); + memset(bh, 0, sizeof(*bh)); + + bh->b_size = rbh->b_size; + bh->b_dev = rbh->b_dev; + bh->b_rdev = lo->lo_device; + bh->b_state = (1 << BH_Req) | (1 << BH_Mapped) | (1 << BH_Lock); + + /* + * easy way out, although it does waste some memory for < PAGE_SIZE + * blocks... if highmem bounce buffering can get away with it, + * so can we :-) + */ + bh->b_page = alloc_page(GFP_BUFFER); + bh->b_data = kmap(bh->b_page); + + bh->b_end_io = loop_end_io_transfer; + bh->b_rsector = rbh->b_rsector + (lo->lo_offset >> 9); + init_waitqueue_head(&bh->b_wait); + + return bh; +} + +static int loop_make_request(request_queue_t *q, int rw, struct buffer_head *rbh) +{ + struct buffer_head *bh = NULL; + struct loop_device *lo; + unsigned long IV; + + if (!buffer_locked(rbh)) + BUG(); + + if (MINOR(rbh->b_dev) >= max_loop) + goto err; + + lo = &loop_dev[MINOR(rbh->b_dev)]; + if (!lo->lo_dentry) + goto err; + + if (rw == WRITE) { if (lo->lo_flags & LO_FLAGS_READ_ONLY) - goto error_out; - } else if (current_request->cmd != READ) { - printk(KERN_ERR "unknown loop device command (%d)?!?", - current_request->cmd); - goto error_out; + goto err; + } else if (rw == READA) { + rw = READ; + } else if (rw != READ) { + printk(KERN_ERR "loop: unknown command (%d)\n", rw); + goto err; } - dest_addr = current_request->buffer; - len = current_request->current_nr_sectors << 9; +#if CONFIG_HIGHMEM + rbh = create_bounce(rw, rbh); +#endif + + if (lo->lo_blksize != rbh->b_size) + lo->lo_blksize = rbh->b_size; - blksize = BLOCK_SIZE; - if (blksize_size[MAJOR(lo->lo_device)]) { - blksize = blksize_size[MAJOR(lo->lo_device)][MINOR(lo->lo_device)]; - if (!blksize) - blksize = BLOCK_SIZE; + /* + * file backed, queue for loop_thread to handle + */ + if (lo->lo_flags & LO_FLAGS_DO_BMAP) { + if (rw == WRITE) + set_bit(BH_Dirty, &rbh->b_state); + loop_add_bh(lo, rbh); + return 0; } - if (lo->lo_flags & LO_FLAGS_DO_BMAP) - goto file_backed; + /* + * piggy old buffer on original, and submit for I/O + */ + bh = loop_get_buffer(lo, rbh); + bh->b_private = rbh; + IV = (bh->b_rsector / (lo->lo_blksize >> 9)); + IV += lo->lo_offset / lo->lo_blksize; + IV >>= 2; + if (rw == WRITE) { + set_bit(BH_Dirty, &bh->b_state); + if (lo_do_transfer(lo, WRITE, bh->b_data, rbh->b_data, bh->b_size, IV)) + goto err; + } - if (blksize < 512) { - block = current_request->sector * (512/blksize); - offset = 0; + generic_make_request(rw, bh); + return 0; + +err: + loop_put_buffer(bh); + buffer_IO_error(rbh); + return 0; +} + +static inline void loop_handle_bh(struct loop_device *lo,struct buffer_head *bh) +{ + int ret; + + /* + * For block backed loop, we know this is a READ + */ + if (lo->lo_flags & LO_FLAGS_DO_BMAP) { + int rw = !!test_and_clear_bit(BH_Dirty, &bh->b_state); + + ret = do_bh_filebacked(lo, bh, rw); + bh->b_end_io(bh, !ret); } else { - block = current_request->sector / (blksize >> 9); - offset = (current_request->sector % (blksize >> 9)) << 9; - } - block += lo->lo_offset / blksize; - offset += lo->lo_offset % blksize; - if (offset >= blksize) { - block++; - offset -= blksize; + struct buffer_head *rbh = bh->b_private; + unsigned long IV; + + IV = (bh->b_rsector / (bh->b_size >> 9)); + IV += lo->lo_offset / bh->b_size; + + ret = lo_do_transfer(lo, READ, bh->b_data, rbh->b_data, + bh->b_size, IV); + + rbh->b_end_io(rbh, !ret); + loop_put_buffer(bh); } - spin_unlock_irq(&io_request_lock); +} - while (len > 0) { +/* + * worker thread that handles reads/writes to file backed loop devices, + * to avoid blocking in our make_request_fn. it also does loop decrypting + * on reads for block backed loop, as that is too heavy to do from + * b_end_io context where irqs may be disabled. + */ +static int loop_thread(void *data) +{ + struct loop_device *lo = data; + struct buffer_head *bh; + int i; - size = blksize - offset; - if (size > len) - size = len; + daemonize(); + exit_files(current); - bh = getblk(lo->lo_device, block, blksize); - if (!bh) { - printk(KERN_ERR "loop: device %s: getblk(-, %d, %d) returned NULL", - kdevname(lo->lo_device), - block, blksize); - goto error_out_lock; - } - if (!buffer_uptodate(bh) && ((current_request->cmd == READ) || - (offset || (len < blksize)))) { - ll_rw_block(READ, 1, &bh); - wait_on_buffer(bh); - if (!buffer_uptodate(bh)) { - brelse(bh); - goto error_out_lock; + sprintf(current->comm, "loop%d", lo->lo_number); + + spin_lock_irq(¤t->sigmask_lock); + siginitsetinv(¤t->blocked, sigmask(SIGKILL)); + flush_signals(current); + spin_unlock_irq(¤t->sigmask_lock); + + current->policy = SCHED_OTHER; + current->nice = -20; + + lo->lo_tsk = current; + + for (i = 0;; i++) { + bh = loop_get_bh(lo); + set_task_state(lo->lo_tsk, TASK_INTERRUPTIBLE); + if (bh) { + loop_handle_bh(lo, bh); + /* + * give kswapd/bdflush a chance to run... + */ + if (i > 1024) { + set_task_state(lo->lo_tsk,TASK_UNINTERRUPTIBLE); + schedule_timeout(1); + i = 0; } + continue; } - if ((lo->transfer)(lo, current_request->cmd, - bh->b_data + offset, - dest_addr, size, block)) { - printk(KERN_ERR "loop: transfer error block %d\n", - block); - brelse(bh); - goto error_out_lock; - } + schedule(); + i = 0; - if (current_request->cmd == WRITE) { - mark_buffer_uptodate(bh, 1); - mark_buffer_dirty(bh); - } - brelse(bh); - dest_addr += size; - len -= size; - offset = 0; - block++; + if (signal_pending(lo->lo_tsk)) + break; } - goto done; -file_backed: - pos = ((loff_t)current_request->sector << 9) + lo->lo_offset; - spin_unlock_irq(&io_request_lock); - if (current_request->cmd == WRITE) { - if (lo_send(lo, dest_addr, len, pos, blksize)) - goto error_out_lock; - } else { - if (lo_receive(lo, dest_addr, len, pos, blksize)) - goto error_out_lock; - } -done: - spin_lock_irq(&io_request_lock); - current_request->sector += current_request->current_nr_sectors; - current_request->nr_sectors -= current_request->current_nr_sectors; - list_add(¤t_request->queue, &q->queue_head); - end_request(1); - goto repeat; -error_out_lock: - spin_lock_irq(&io_request_lock); -error_out: - list_add(¤t_request->queue, &q->queue_head); - end_request(0); - goto repeat; + return 0; } static int loop_set_fd(struct loop_device *lo, kdev_t dev, unsigned int arg) { struct file *file; struct inode *inode; - int error; + int error, bs; MOD_INC_USE_COUNT; @@ -412,13 +548,19 @@ error = -EINVAL; inode = file->f_dentry->d_inode; + spin_lock_init(&lo->lo_lock); + lo->lo_bh = lo->lo_bhtail = NULL; + lo->lo_pid = kernel_thread(loop_thread, lo, CLONE_FS | CLONE_FILES | + CLONE_SIGHAND); + if (S_ISBLK(inode->i_mode)) { /* dentry will be wired, so... */ error = blkdev_get(inode->i_bdev, file->f_mode, file->f_flags, BDEV_FILE); lo->lo_device = inode->i_rdev; - lo->lo_flags = 0; + if (is_read_only(lo->lo_device)) + lo->lo_flags |= LO_FLAGS_READ_ONLY; /* Backed by a block device - don't need to hold onto a file structure */ @@ -427,9 +569,7 @@ if (error) goto out_putf; } else if (S_ISREG(inode->i_mode)) { - struct address_space_operations *aops; - - aops = inode->i_mapping->a_ops; + struct address_space_operations *aops = inode->i_mapping->a_ops; /* * If we can't read - sorry. If we only can't write - well, * it's going to be read-only. @@ -478,14 +618,27 @@ if (IS_RDONLY (inode) || is_read_only(lo->lo_device)) lo->lo_flags |= LO_FLAGS_READ_ONLY; - set_device_ro(dev, (lo->lo_flags & LO_FLAGS_READ_ONLY)!=0); + bs = 0; + if (blksize_size[MAJOR(inode->i_rdev)]) + bs = blksize_size[MAJOR(inode->i_rdev)][MINOR(inode->i_rdev)]; + if (!bs) + bs = BLOCK_SIZE; + + lo->lo_blksize = bs; + set_blocksize(dev, bs); + + set_device_ro(dev, lo->lo_flags & LO_FLAGS_READ_ONLY); lo->lo_dentry = dget(file->f_dentry); lo->transfer = NULL; lo->ioctl = NULL; + lo->old_gfp_mask = inode->i_mapping->gfp_mask; + inode->i_mapping->gfp_mask = GFP_BUFFER; figure_loop_size(lo); out_putf: + if (error) + send_sig(SIGKILL, lo->lo_tsk, 1); fput(file); out: if (error) @@ -526,12 +679,15 @@ static int loop_clr_fd(struct loop_device *lo, kdev_t dev) { struct dentry *dentry = lo->lo_dentry; + int gfp = lo->old_gfp_mask; if (!dentry) return -ENXIO; if (lo->lo_refcnt > 1) /* we needed one fd for the ioctl */ return -EBUSY; + send_sig(SIGKILL, lo->lo_tsk, 1); + if (S_ISBLK(dentry->d_inode->i_mode)) blkdev_put(dentry->d_inode->i_bdev, BDEV_FILE); @@ -554,10 +710,12 @@ lo->lo_encrypt_type = 0; lo->lo_offset = 0; lo->lo_encrypt_key_size = 0; + lo->lo_flags = 0; memset(lo->lo_encrypt_key, 0, LO_KEY_SIZE); memset(lo->lo_name, 0, LO_NAME_SIZE); loop_sizes[lo->lo_number] = 0; invalidate_buffers(dev); + dentry->d_inode->i_mapping->gfp_mask = gfp; MOD_DEC_USE_COUNT; return 0; } @@ -656,7 +814,7 @@ return loop_set_status(lo, (struct loop_info *) arg); case LOOP_GET_STATUS: return loop_get_status(lo, (struct loop_info *) arg); - case BLKGETSIZE: /* Return device size */ + case BLKGETSIZE: if (!lo->lo_dentry) return -ENXIO; if (!arg) @@ -673,7 +831,6 @@ struct loop_device *lo; int dev, type; - if (!inode) return -EINVAL; if (MAJOR(inode->i_rdev) != MAJOR_NR) { @@ -681,9 +838,9 @@ return -ENODEV; } dev = MINOR(inode->i_rdev); - if (dev >= max_loop) { + if (dev >= max_loop) return -ENODEV; - } + lo = &loop_dev[dev]; type = lo->lo_encrypt_type; @@ -697,7 +854,7 @@ static int lo_release(struct inode *inode, struct file *file) { struct loop_device *lo; - int dev; + int dev, type; if (!inode) return 0; @@ -709,17 +866,14 @@ dev = MINOR(inode->i_rdev); if (dev >= max_loop) return 0; + lo = &loop_dev[dev]; - if (lo->lo_refcnt <= 0) - printk(KERN_ERR "lo_release: refcount(%d) <= 0\n", - lo->lo_refcnt); - else { - int type = lo->lo_encrypt_type; - --lo->lo_refcnt; - if (xfer_funcs[type] && xfer_funcs[type]->unlock) - xfer_funcs[type]->unlock(lo); - MOD_DEC_USE_COUNT; - } + type = lo->lo_encrypt_type; + --lo->lo_refcnt; + if (xfer_funcs[type] && xfer_funcs[type]->unlock) + xfer_funcs[type]->unlock(lo); + + MOD_DEC_USE_COUNT; return 0; } @@ -732,11 +886,8 @@ /* * And now the modules code and kernel interface. */ -#ifdef MODULE -#define loop_init init_module MODULE_PARM(max_loop, "i"); MODULE_PARM_DESC(max_loop, "Maximum number of loop devices (1-255)"); -#endif int loop_register_transfer(struct loop_func_table *funcs) { @@ -767,57 +918,50 @@ EXPORT_SYMBOL(loop_register_transfer); EXPORT_SYMBOL(loop_unregister_transfer); -static void no_plug_device(request_queue_t *q, kdev_t device) -{ -} - int __init loop_init(void) { int i; - if (devfs_register_blkdev(MAJOR_NR, "loop", &lo_fops)) { - printk(KERN_WARNING "Unable to get major number %d for loop device\n", - MAJOR_NR); - return -EIO; - } - devfs_handle = devfs_mk_dir (NULL, "loop", NULL); - devfs_register_series (devfs_handle, "%u", max_loop, DEVFS_FL_DEFAULT, - MAJOR_NR, 0, - S_IFBLK | S_IRUSR | S_IWUSR | S_IRGRP, - &lo_fops, NULL); - if ((max_loop < 1) || (max_loop > 255)) { - printk (KERN_WARNING "loop: invalid max_loop (must be between 1 and 255), using default (8)\n"); + printk(KERN_WARNING "loop: invalid max_loop (must be between" + " 1 and 255), using default (8)\n"); max_loop = 8; } - printk(KERN_INFO "loop: enabling %d loop devices\n", max_loop); + if (devfs_register_blkdev(MAJOR_NR, "loop", &lo_fops)) { + printk(KERN_WARNING "Unable to get major number %d for loop" + " device\n", MAJOR_NR); + return -EIO; + } - loop_dev = kmalloc (max_loop * sizeof(struct loop_device), GFP_KERNEL); - if (!loop_dev) { - printk (KERN_ERR "loop: Unable to create loop_dev\n"); + loop_bhp = kmem_cache_create("loop_buffers", sizeof(struct buffer_head), + 0, SLAB_HWCACHE_ALIGN, NULL, NULL); + if (!loop_bhp) { + printk(KERN_WARNING "loop: unable to create slab cache\n"); return -ENOMEM; } + + devfs_handle = devfs_mk_dir(NULL, "loop", NULL); + devfs_register_series(devfs_handle, "%u", max_loop, DEVFS_FL_DEFAULT, + MAJOR_NR, 0, + S_IFBLK | S_IRUSR | S_IWUSR | S_IRGRP, + &lo_fops, NULL); + + loop_dev = kmalloc(max_loop * sizeof(struct loop_device), GFP_KERNEL); + if (!loop_dev) + goto out_dev; loop_sizes = kmalloc(max_loop * sizeof(int), GFP_KERNEL); - if (!loop_sizes) { - printk (KERN_ERR "loop: Unable to create loop_sizes\n"); - kfree (loop_dev); - return -ENOMEM; - } + if (!loop_sizes) + goto out_sizes; - loop_blksizes = kmalloc (max_loop * sizeof(int), GFP_KERNEL); - if (!loop_blksizes) { - printk (KERN_ERR "loop: Unable to create loop_blksizes\n"); - kfree (loop_dev); - kfree (loop_sizes); - return -ENOMEM; - } + loop_blksizes = kmalloc(max_loop * sizeof(int), GFP_KERNEL); + if (!loop_blksizes) + goto out_blksizes; - blk_init_queue(BLK_DEFAULT_QUEUE(MAJOR_NR), DEVICE_REQUEST); - blk_queue_pluggable(BLK_DEFAULT_QUEUE(MAJOR_NR), no_plug_device); - blk_queue_headactive(BLK_DEFAULT_QUEUE(MAJOR_NR), 0); - for (i=0; i < max_loop; i++) { + blk_queue_make_request(BLK_DEFAULT_QUEUE(MAJOR_NR), loop_make_request); + + for (i = 0; i < max_loop; i++) { memset(&loop_dev[i], 0, sizeof(struct loop_device)); loop_dev[i].lo_number = i; } @@ -825,30 +969,41 @@ memset(loop_blksizes, 0, max_loop * sizeof(int)); blk_size[MAJOR_NR] = loop_sizes; blksize_size[MAJOR_NR] = loop_blksizes; - for (i=0; i < max_loop; i++) - register_disk(NULL, MKDEV(MAJOR_NR,i), 1, &lo_fops, 0); + for (i = 0; i < max_loop; i++) + register_disk(NULL, MKDEV(MAJOR_NR, i), 1, &lo_fops, 0); + printk(KERN_INFO "loop: loaded (max %d devices)\n", max_loop); return 0; + +out_dev: + kmem_cache_destroy(loop_bhp); +out_sizes: + kfree(loop_dev); +out_blksizes: + kfree(loop_sizes); + printk(KERN_ERR "loop: ran out of memory\n"); + return -ENOMEM; } -#ifdef MODULE -void cleanup_module(void) +void loop_exit(void) { - devfs_unregister (devfs_handle); - if (devfs_unregister_blkdev(MAJOR_NR, "loop") != 0) + devfs_unregister(devfs_handle); + if (devfs_unregister_blkdev(MAJOR_NR, "loop")) printk(KERN_WARNING "loop: cannot unregister blkdev\n"); - blk_cleanup_queue(BLK_DEFAULT_QUEUE(MAJOR_NR)); - kfree (loop_dev); - kfree (loop_sizes); - kfree (loop_blksizes); + kmem_cache_destroy(loop_bhp); + kfree(loop_dev); + kfree(loop_sizes); + kfree(loop_blksizes); } -#endif + +module_init(loop_init); +module_exit(loop_exit); #ifndef MODULE static int __init max_loop_setup(char *str) { - max_loop = simple_strtol(str,NULL,0); + max_loop = simple_strtol(str, NULL, 0); return 1; } diff -ur --exclude-from /home/axboe/exclude /opt/kernel/linux-2.4.2-pre1/fs/inode.c linux/fs/inode.c --- /opt/kernel/linux-2.4.2-pre1/fs/inode.c Mon Feb 5 16:41:36 2001 +++ linux/fs/inode.c Mon Feb 5 13:20:33 2001 @@ -617,6 +617,7 @@ inode->i_bdev = NULL; inode->i_data.a_ops = &empty_aops; inode->i_data.host = inode; + inode->i_data.gfp_mask = GFP_HIGHUSER; inode->i_mapping = &inode->i_data; } diff -ur --exclude-from /home/axboe/exclude /opt/kernel/linux-2.4.2-pre1/fs/nfs/dir.c linux/fs/nfs/dir.c --- /opt/kernel/linux-2.4.2-pre1/fs/nfs/dir.c Sun Dec 10 18:55:48 2000 +++ linux/fs/nfs/dir.c Sat Feb 3 17:14:34 2001 @@ -321,7 +321,7 @@ desc->page = NULL; } - page = page_cache_alloc(); + page = page_cache_alloc(NULL); if (!page) { status = -ENOMEM; goto out; diff -ur --exclude-from /home/axboe/exclude /opt/kernel/linux-2.4.2-pre1/include/linux/fs.h linux/include/linux/fs.h --- /opt/kernel/linux-2.4.2-pre1/include/linux/fs.h Tue Jan 30 08:24:56 2001 +++ linux/include/linux/fs.h Mon Feb 5 13:28:22 2001 @@ -373,6 +373,7 @@ struct vm_area_struct *i_mmap; /* list of private mappings */ struct vm_area_struct *i_mmap_shared; /* list of shared mappings */ spinlock_t i_shared_lock; /* and spinlock protecting it */ + int gfp_mask; /* how to allocate the pages */ }; struct block_device { diff -ur --exclude-from /home/axboe/exclude /opt/kernel/linux-2.4.2-pre1/include/linux/loop.h linux/include/linux/loop.h --- /opt/kernel/linux-2.4.2-pre1/include/linux/loop.h Mon Dec 11 21:50:30 2000 +++ linux/include/linux/loop.h Wed Jan 31 04:13:11 2001 @@ -16,7 +16,7 @@ #define LO_KEY_SIZE 32 #ifdef __KERNEL__ - + struct loop_device { int lo_number; struct dentry *lo_dentry; @@ -39,12 +39,32 @@ struct file * lo_backing_file; void *key_data; char key_reserved[48]; /* for use by the filter modules */ + + int lo_blksize; + int old_gfp_mask; + + pid_t lo_pid; + + struct task_struct *lo_tsk; + + spinlock_t lo_lock; + struct buffer_head *lo_bh; + struct buffer_head *lo_bhtail; }; typedef int (* transfer_proc_t)(struct loop_device *, int cmd, char *raw_buf, char *loop_buf, int size, int real_block); +extern inline int lo_do_transfer(struct loop_device *lo, int cmd, char *rbuf, + char *lbuf, int size, int rblock) +{ + if (!lo->transfer) + return 0; + + return lo->transfer(lo, cmd, rbuf, lbuf, size, rblock); +} + #endif /* __KERNEL__ */ /* @@ -102,9 +122,8 @@ /* Support for loadable transfer modules */ struct loop_func_table { int number; /* filter type */ - int (*transfer)(struct loop_device *lo, int cmd, - char *raw_buf, char *loop_buf, int size, - int real_block); + int (*transfer)(struct loop_device *lo, int cmd, char *raw_buf, + char *loop_buf, int size, int real_block); int (*init)(struct loop_device *, struct loop_info *); /* release is called from loop_unregister_transfer or clr_fd */ int (*release)(struct loop_device *); diff -ur --exclude-from /home/axboe/exclude /opt/kernel/linux-2.4.2-pre1/include/linux/pagemap.h linux/include/linux/pagemap.h --- /opt/kernel/linux-2.4.2-pre1/include/linux/pagemap.h Tue Jan 30 08:24:56 2001 +++ linux/include/linux/pagemap.h Mon Feb 5 13:28:29 2001 @@ -29,9 +29,13 @@ #define PAGE_CACHE_ALIGN(addr) (((addr)+PAGE_CACHE_SIZE-1)&PAGE_CACHE_MASK) #define page_cache_get(x) get_page(x) -#define page_cache_alloc() alloc_pages(GFP_HIGHUSER, 0) #define page_cache_free(x) __free_page(x) #define page_cache_release(x) __free_page(x) + +static inline struct page *page_cache_alloc(struct address_space *x) +{ + return alloc_pages(x ? x->gfp_mask : GFP_HIGHUSER, 0); +} /* * From a kernel address, get the "struct page *" diff -ur --exclude-from /home/axboe/exclude /opt/kernel/linux-2.4.2-pre1/kernel/ksyms.c linux/kernel/ksyms.c --- /opt/kernel/linux-2.4.2-pre1/kernel/ksyms.c Mon Jan 29 01:11:20 2001 +++ linux/kernel/ksyms.c Sat Feb 3 16:14:01 2001 @@ -120,6 +120,7 @@ EXPORT_SYMBOL(kmap_high); EXPORT_SYMBOL(kunmap_high); EXPORT_SYMBOL(highmem_start_page); +EXPORT_SYMBOL(create_bounce); #endif /* filesystem internal functions */ diff -ur --exclude-from /home/axboe/exclude /opt/kernel/linux-2.4.2-pre1/mm/filemap.c linux/mm/filemap.c --- /opt/kernel/linux-2.4.2-pre1/mm/filemap.c Tue Jan 16 02:14:41 2001 +++ linux/mm/filemap.c Sat Feb 3 17:14:34 2001 @@ -560,7 +560,7 @@ if (page) return 0; - page = page_cache_alloc(); + page = page_cache_alloc(mapping); if (!page) return -ENOMEM; @@ -1175,7 +1175,7 @@ */ if (!cached_page) { spin_unlock(&pagecache_lock); - cached_page = page_cache_alloc(); + cached_page = page_cache_alloc(mapping); if (!cached_page) { desc->error = -ENOMEM; break; @@ -1475,7 +1475,7 @@ */ old_page = page; if (no_share) { - struct page *new_page = page_cache_alloc(); + struct page *new_page = page_cache_alloc(NULL); if (new_page) { copy_user_highpage(new_page, old_page, address); @@ -2320,7 +2320,7 @@ page = __find_get_page(mapping, index, hash); if (!page) { if (!cached_page) { - cached_page = page_cache_alloc(); + cached_page = page_cache_alloc(mapping); if (!cached_page) return ERR_PTR(-ENOMEM); } @@ -2383,7 +2383,7 @@ page = __find_lock_page(mapping, index, hash); if (!page) { if (!*cached_page) { - *cached_page = page_cache_alloc(); + *cached_page = page_cache_alloc(mapping); if (!*cached_page) return NULL; } diff -ur --exclude-from /home/axboe/exclude /opt/kernel/linux-2.4.2-pre1/mm/memory.c linux/mm/memory.c --- /opt/kernel/linux-2.4.2-pre1/mm/memory.c Mon Feb 5 16:41:36 2001 +++ linux/mm/memory.c Mon Feb 5 13:20:33 2001 @@ -868,7 +868,7 @@ * Ok, we need to copy. Oh, well.. */ spin_unlock(&mm->page_table_lock); - new_page = page_cache_alloc(); + new_page = page_cache_alloc(NULL); if (!new_page) return -1; spin_lock(&mm->page_table_lock); diff -ur --exclude-from /home/axboe/exclude /opt/kernel/linux-2.4.2-pre1/mm/shmem.c linux/mm/shmem.c --- /opt/kernel/linux-2.4.2-pre1/mm/shmem.c Sun Jan 28 04:50:08 2001 +++ linux/mm/shmem.c Sat Feb 3 17:14:34 2001 @@ -321,7 +321,7 @@ inode->i_sb->u.shmem_sb.free_blocks--; spin_unlock (&inode->i_sb->u.shmem_sb.stat_lock); /* Ok, get a new page */ - page = page_cache_alloc(); + page = page_cache_alloc(mapping); if (!page) goto oom; clear_user_highpage(page, address); @@ -338,7 +338,7 @@ up(&inode->i_sem); if (no_share) { - struct page *new_page = page_cache_alloc(); + struct page *new_page = page_cache_alloc(inode->i_mapping); if (new_page) { copy_user_highpage(new_page, page, address);