diff -ur /opt/kernel/linux-2.4.0-test3-pre2/drivers/block/elevator.c linux/drivers/block/elevator.c --- /opt/kernel/linux-2.4.0-test3-pre2/drivers/block/elevator.c Tue Jul 4 19:32:06 2000 +++ linux/drivers/block/elevator.c Tue Jul 4 19:35:53 2000 @@ -88,25 +88,25 @@ head = head->next; while ((entry = entry->prev) != head && !starving) { - *req = blkdev_entry_to_request(entry); - latency += (*req)->nr_segments; - if (elevator_sequence_before((*req)->elevator_sequence, sequence)) + struct request *__rq = *req = blkdev_entry_to_request(entry); + latency += __rq->nr_segments; + if (elevator_sequence_before(__rq->elevator_sequence, sequence)) starving = 1; if (latency < 0) continue; - if ((*req)->sem) + if (__rq->sem) continue; - if ((*req)->cmd != rw) + if (__rq->cmd != rw) continue; - if ((*req)->nr_sectors + count > *max_sectors) + if (__rq->nr_sectors + count > *max_sectors) continue; - if ((*req)->rq_dev != bh->b_rdev) + if (__rq->rq_dev != bh->b_rdev) continue; - if ((*req)->sector + (*req)->nr_sectors == bh->b_rsector) { - if (latency - (*req)->nr_segments < 0) + if (__rq->sector + __rq->nr_sectors == bh->b_rsector) { + if (latency - __rq->nr_segments < 0) break; action = ELEVATOR_BACK_MERGE; - } else if ((*req)->sector - count == bh->b_rsector) { + } else if (__rq->sector - count == bh->b_rsector) { if (starving) break; action = ELEVATOR_FRONT_MERGE; @@ -161,31 +161,45 @@ int *max_sectors, int *max_segments) { struct list_head *entry, *head = &q->queue_head; - unsigned int count = bh->b_size >> 9; + unsigned int count = bh->b_size >> 9, ret = ELEVATOR_NO_MERGE; entry = head; if (q->head_active && !q->plugged) head = head->next; while ((entry = entry->prev) != head) { - *req = blkdev_entry_to_request(entry); - if (!(*req)->elevator_sequence) + struct request *__rq = *req = blkdev_entry_to_request(entry); + if (!__rq->elevator_sequence) break; - if ((*req)->sem) + if (__rq->sem) continue; - if ((*req)->cmd != rw) + if (__rq->cmd != rw) continue; - if ((*req)->nr_sectors + count > *max_sectors) + if (__rq->nr_sectors + count > *max_sectors) continue; - if ((*req)->rq_dev != bh->b_rdev) + if (__rq->rq_dev != bh->b_rdev) continue; - if ((*req)->sector + (*req)->nr_sectors == bh->b_rsector) - return ELEVATOR_BACK_MERGE; - if ((*req)->sector - count == bh->b_rsector) - return ELEVATOR_FRONT_MERGE; - (*req)->elevator_sequence--; + if (__rq->sector + __rq->nr_sectors == bh->b_rsector) { + ret = ELEVATOR_BACK_MERGE; + break; + } + if (__rq->sector - count == bh->b_rsector) { + ret = ELEVATOR_FRONT_MERGE; + break; + } } - return ELEVATOR_NO_MERGE; + + /* + * second pass scan of requests that got passed over, if any + */ + if (ret != ELEVATOR_NO_MERGE && *req) { + while ((entry = entry->next) != &q->queue_head) { + struct request *tmp = blkdev_entry_to_request(entry); + tmp->elevator_sequence--; + } + } + + return ret; } /* @@ -213,18 +227,18 @@ entry = head; while ((entry = entry->prev) != head) { - *req = blkdev_entry_to_request(entry); - if ((*req)->sem) + struct request *__rq = *req = blkdev_entry_to_request(entry); + if (__rq->sem) continue; - if ((*req)->cmd != rw) + if (__rq->cmd != rw) continue; - if ((*req)->nr_sectors + count > *max_sectors) + if (__rq->nr_sectors + count > *max_sectors) continue; - if ((*req)->rq_dev != bh->b_rdev) + if (__rq->rq_dev != bh->b_rdev) continue; - if ((*req)->sector + (*req)->nr_sectors == bh->b_rsector) + if (__rq->sector + __rq->nr_sectors == bh->b_rsector) return ELEVATOR_BACK_MERGE; - if ((*req)->sector - count == bh->b_rsector) + if (__rq->sector - count == bh->b_rsector) return ELEVATOR_FRONT_MERGE; } return ELEVATOR_NO_MERGE; diff -ur /opt/kernel/linux-2.4.0-test3-pre2/drivers/block/ll_rw_blk.c linux/drivers/block/ll_rw_blk.c --- /opt/kernel/linux-2.4.0-test3-pre2/drivers/block/ll_rw_blk.c Tue Jul 4 19:32:06 2000 +++ linux/drivers/block/ll_rw_blk.c Tue Jul 4 19:35:53 2000 @@ -148,34 +148,40 @@ return ret; } -/* - * Hopefully the low level driver has finished any out standing requests - * first... - */ -void blk_cleanup_queue(request_queue_t * q) +static int __block_cleanup_queue(struct list_head *head) { struct list_head *entry; struct request *rq; - int i = QUEUE_NR_REQUESTS; + int i = 0; - if (list_empty(&q->request_freelist)) - return; - - if (q->queue_requests) - BUG(); + if (list_empty(head)) + return 0; - entry = &q->request_freelist; - entry = entry->next; + entry = head->next; do { rq = list_entry(entry, struct request, table); entry = entry->next; list_del(&rq->table); kmem_cache_free(request_cachep, rq); - i--; - } while (!list_empty(&q->request_freelist)); + i++; + } while (!list_empty(head)); + + return i; +} + +/* + * Hopefully the low level driver has finished any out standing requests + * first... + */ +void blk_cleanup_queue(request_queue_t * q) +{ + int count = QUEUE_NR_REQUESTS; - if (i) - printk("blk_cleanup_queue: leaked requests (%d)\n", i); + count -= __block_cleanup_queue(&q->request_freelist[READ]); + count -= __block_cleanup_queue(&q->request_freelist[WRITE]); + + if (count) + printk("blk_cleanup_queue: leaked requests (%d)\n", count); memset(q, 0, sizeof(*q)); } @@ -280,10 +286,9 @@ for (i = 0; i < QUEUE_NR_REQUESTS; i++) { rq = kmem_cache_alloc(request_cachep, SLAB_KERNEL); rq->rq_status = RQ_INACTIVE; - list_add(&rq->table, &q->request_freelist); + list_add(&rq->table, &q->request_freelist[i & 1]); } - q->queue_requests = 0; init_waitqueue_head(&q->wait_for_request); spin_lock_init(&q->request_lock); } @@ -291,7 +296,8 @@ void blk_init_queue(request_queue_t * q, request_fn_proc * rfn) { INIT_LIST_HEAD(&q->queue_head); - INIT_LIST_HEAD(&q->request_freelist); + INIT_LIST_HEAD(&q->request_freelist[READ]); + INIT_LIST_HEAD(&q->request_freelist[WRITE]); elevator_init(&q->elevator, ELEVATOR_LINUS); blk_init_free_list(q); q->request_fn = rfn; @@ -342,19 +348,37 @@ */ static inline struct request *get_request(request_queue_t *q, int rw) { - register struct request *rq = NULL; + struct list_head *list = &q->request_freelist[rw]; + struct request *rq; - if (!list_empty(&q->request_freelist)) { - if ((q->queue_requests > QUEUE_WRITES_MAX) && (rw == WRITE)) - return NULL; + /* + * Reads get preferential treatment and are allowed to steal + * from the write free list if necessary. + */ + if (!list_empty(list)) { + rq = blkdev_free_rq(list); + goto got_rq; + } - rq = blkdev_free_rq(&q->request_freelist); - list_del(&rq->table); - rq->rq_status = RQ_ACTIVE; - rq->special = NULL; - rq->q = q; - q->queue_requests++; + /* + * if the WRITE list is non-empty, we know that rw is READ + * and that the READ list is empty. allow reads to 'steal' + * from the WRITE list. + */ + if (!list_empty(&q->request_freelist[WRITE])) { + list = &q->request_freelist[WRITE]; + rq = blkdev_free_rq(list); + goto got_rq; } + + return NULL; + +got_rq: + list_del(&rq->table); + rq->free_list = list; + rq->rq_status = RQ_ACTIVE; + rq->special = NULL; + rq->q = q; return rq; } @@ -486,9 +510,9 @@ /* * Request may not have originated from ll_rw_blk */ - if (req->q) { - list_add(&req->table, &req->q->request_freelist); - req->q->queue_requests--; + if (req->free_list) { + list_add(&req->table, req->free_list); + req->free_list = NULL; wake_up(&req->q->wait_for_request); } } @@ -557,7 +581,7 @@ int max_segments = MAX_SEGMENTS; struct request * req = NULL; int rw_ahead, max_sectors, el_ret; - struct list_head *head = &q->queue_head; + struct list_head *head; int latency; elevator_t *elevator = &q->elevator; @@ -602,12 +626,6 @@ goto end_io; /* Hmmph! Nothing to write */ refile_buffer(bh); do_write: - /* - * We don't allow the write-requests to fill up the - * queue completely: we want some room for reads, - * as they take precedence. The last third of the - * requests are only for reads. - */ kstat.pgpgout++; break; default: @@ -646,11 +664,6 @@ spin_lock_irq(&io_request_lock); elevator_default_debug(q, bh->b_rdev); - if (list_empty(head)) { - q->plug_device_fn(q, bh->b_rdev); /* is atomic */ - goto get_rq; - } - /* * skip first entry, for devices with active queue head */ @@ -658,6 +671,11 @@ if (q->head_active && !q->plugged) head = head->next; + if (list_empty(head)) { + q->plug_device_fn(q, bh->b_rdev); /* is atomic */ + goto get_rq; + } + el_ret = elevator->elevator_merge_fn(q, &req, bh, rw, &max_sectors, &max_segments); switch (el_ret) { @@ -843,7 +861,6 @@ sorry: for (i = 0; i < nr; i++) buffer_IO_error(bhs[i]); - return; } void ll_rw_block(int rw, int nr, struct buffer_head * bh[]) diff -ur /opt/kernel/linux-2.4.0-test3-pre2/drivers/scsi/scsi_lib.c linux/drivers/scsi/scsi_lib.c --- /opt/kernel/linux-2.4.0-test3-pre2/drivers/scsi/scsi_lib.c Thu Jun 29 16:45:11 2000 +++ linux/drivers/scsi/scsi_lib.c Tue Jul 4 19:35:54 2000 @@ -87,6 +87,7 @@ SCpnt->request.cmd = SPECIAL; SCpnt->request.special = (void *) SCpnt; SCpnt->request.q = NULL; + SCpnt->request.free_list = NULL; SCpnt->request.nr_segments = 0; /* diff -ur /opt/kernel/linux-2.4.0-test3-pre2/include/linux/blkdev.h linux/include/linux/blkdev.h --- /opt/kernel/linux-2.4.0-test3-pre2/include/linux/blkdev.h Thu Jun 29 16:45:11 2000 +++ linux/include/linux/blkdev.h Tue Jul 4 19:37:19 2000 @@ -23,9 +23,8 @@ int elevator_sequence; struct list_head table; - /* - * queue free list belongs to - */ + struct list_head *free_list; + volatile int rq_status; /* should split this into a few status bits */ #define RQ_INACTIVE (-1) #define RQ_ACTIVE 1 @@ -70,16 +69,14 @@ /* * Default nr free requests per queue */ -#define QUEUE_NR_REQUESTS 512 -#define QUEUE_WRITES_MAX ((2 * QUEUE_NR_REQUESTS) / 3) +#define QUEUE_NR_REQUESTS 256 struct request_queue { /* * the queue request freelist, one for reads and one for writes */ - struct list_head request_freelist; - int queue_requests; + struct list_head request_freelist[2]; /* * Together with queue_head for cacheline sharing