Index: drivers/block/ll_rw_blk.c =================================================================== RCS file: /cvs/linux-2.4-xfs/linux/drivers/block/ll_rw_blk.c,v retrieving revision 1.53 diff -u -r1.53 ll_rw_blk.c --- drivers/block/ll_rw_blk.c 2000/12/01 18:04:38 1.53 +++ drivers/block/ll_rw_blk.c 2000/12/11 14:30:00 @@ -835,7 +835,7 @@ if (!(PAGE_SIZE - c_off > total_bytes)) nr_bytes = PAGE_SIZE - c_off; - rq->current_nr_sectors = nr_bytes >> 9; + rq->hard_cur_sectors = rq->current_nr_sectors = nr_bytes >> 9; /* Recalculate # segments; reuse "kioind" from above */ for (; kioind < kiobuf->nr_pages && nr_bytes != total_bytes; kioind++){ @@ -872,7 +872,7 @@ int max_segments = MAX_SEGMENTS; struct request * req = NULL, *freereq = NULL; int rw_ahead, max_sectors, el_ret; - struct list_head *head = &q->queue_head; + struct list_head *head; int latency; elevator_t *elevator = &q->elevator; @@ -920,6 +920,7 @@ * not to schedule or do something nonatomic */ again: + head = &q->queue_head; spin_lock_irq(&io_request_lock); /* @@ -958,6 +959,7 @@ req->bh = bh; req->buffer = bh->b_data; req->current_nr_sectors = count; + req->hard_cur_sectors = count; req->sector = req->hard_sector = sector; req->nr_sectors = req->hard_nr_sectors += count; req->e = elevator; @@ -1002,7 +1004,7 @@ req->errors = 0; req->hard_sector = req->sector = sector; req->hard_nr_sectors = req->nr_sectors = count; - req->current_nr_sectors = count; + req->hard_cur_sectors = req->current_nr_sectors = count; req->nr_segments = 1; /* Always 1 for a new request. */ req->nr_hw_segments = 1; /* Always 1 for a new request. */ blk_init_req(req, bh, kiobuf); @@ -1013,8 +1015,6 @@ req_new_io(req, 0, count); add_request(q, req, head, latency); out: - if (freereq) - blkdev_release_request(freereq); if (!q->plugged) (q->request_fn)(q); if (freereq) @@ -1233,7 +1233,7 @@ * should try ll_rw_block() * for non-SCSI (e.g. IDE) disks. */ - if (!SCSI_DISK_MAJOR(MAJOR(dev))) { + if (!SCSI_DISK_MAJOR(MAJOR(dev)) && !IDE_DISK_MAJOR(MAJOR(dev))) { *error = -ENOSYS; goto end_io; } @@ -1323,6 +1323,7 @@ req->nr_sectors = req->hard_nr_sectors; req->current_nr_sectors = bh->b_size >> 9; + req->hard_cur_sectors = bh->b_size >> 9; if (req->nr_sectors < req->current_nr_sectors) { req->nr_sectors = req->current_nr_sectors; printk("end_request: buffer-list destroyed\n"); Index: drivers/ide/ide-disk.c =================================================================== RCS file: /cvs/linux-2.4-xfs/linux/drivers/ide/ide-disk.c,v retrieving revision 1.9 diff -u -r1.9 ide-disk.c --- drivers/ide/ide-disk.c 2000/11/01 21:35:42 1.9 +++ drivers/ide/ide-disk.c 2000/12/11 14:30:01 @@ -131,6 +131,7 @@ return 0; /* lba_capacity value may be bad */ } + /* * read_intr() is the handler for disk read/multread interrupts */ @@ -139,7 +140,7 @@ byte stat; int i; unsigned int msect, nsect; - struct request *rq; + struct request *rq = HWGROUP(drive)->rq; /* new way for dealing with premature shared PCI interrupts */ if (!OK_STAT(stat=GET_STAT(),DATA_READY,BAD_R_STAT)) { @@ -153,7 +154,6 @@ msect = drive->mult_count; read_next: - rq = HWGROUP(drive)->rq; if (msect) { if ((nsect = rq->current_nr_sectors) > msect) nsect = msect; @@ -200,7 +200,7 @@ rq->nr_sectors-1); #endif if ((rq->nr_sectors == 1) ^ ((stat & DRQ_STAT) != 0)) { - rq->sector++; + //rq->sector++; rq->buffer += 512; rq->errors = 0; i = --rq->nr_sectors; @@ -214,6 +214,7 @@ } return ide_stopped; } + return ide_stopped; /* the original code did this here (?) */ } return ide_error(drive, "write_intr", stat); @@ -228,21 +229,9 @@ int ide_multwrite (ide_drive_t *drive, unsigned int mcount) { ide_hwgroup_t *hwgroup= HWGROUP(drive); - - /* - * This may look a bit odd, but remember wrq is a copy of the - * request not the original. The pointers are real however so the - * bh's are not copies. Remember that or bad stuff will happen - * - * At the point we are called the drive has asked us for the - * data, and its our job to feed it, walking across bh boundaries - * if need be. - */ - - struct request *rq = &hwgroup->wrq; + struct request *rq = hwgroup->rq; do { - unsigned long flags; unsigned int nsect = rq->current_nr_sectors; if (nsect > mcount) nsect = mcount; @@ -254,7 +243,6 @@ drive->name, rq->sector, (unsigned long) rq->buffer, nsect, rq->nr_sectors - nsect); #endif - spin_lock_irqsave(&io_request_lock, flags); /* Is this really necessary? */ #ifdef CONFIG_BLK_DEV_PDC4030 rq->sector += nsect; #endif @@ -263,26 +251,12 @@ printk("%s: multwrite: count=%d, current=%ld\n", drive->name, nsect, rq->nr_sectors); #endif - spin_unlock_irqrestore(&io_request_lock, flags); break; } - if ((rq->current_nr_sectors -= nsect) == 0) { - if ((rq->bh = rq->bh->b_reqnext) != NULL) { - rq->current_nr_sectors = rq->bh->b_size>>9; - rq->buffer = rq->bh->b_data; - } else { - spin_unlock_irqrestore(&io_request_lock, flags); - printk("%s: buffer list corrupted (%ld, %ld, %d)\n", - drive->name, rq->current_nr_sectors, - rq->nr_sectors, nsect); - ide_end_request(0, hwgroup); - return 1; - } - } else { - /* Fix the pointer.. we ate data */ + if ((rq->current_nr_sectors -= nsect) == 0) + ide_end_request(1, hwgroup); + else rq->buffer += nsect << 9; - } - spin_unlock_irqrestore(&io_request_lock, flags); } while (mcount); return 0; } @@ -293,9 +267,8 @@ static ide_startstop_t multwrite_intr (ide_drive_t *drive) { byte stat; - int i; ide_hwgroup_t *hwgroup = HWGROUP(drive); - struct request *rq = &hwgroup->wrq; + struct request *rq = hwgroup->rq; if (OK_STAT(stat=GET_STAT(),DRIVE_READY,drive->bad_wstat)) { if (stat & DRQ_STAT) { @@ -310,20 +283,10 @@ return ide_started; } } else { - /* - * If the copy has all the blocks completed then - * we can end the original request. - */ - if (!rq->nr_sectors) { /* all done? */ - rq = hwgroup->rq; - for (i = rq->nr_sectors; i > 0;){ - i -= rq->current_nr_sectors; - ide_end_request(1, hwgroup); - } - return ide_stopped; - } + if (!rq->nr_sectors) + ide_end_request(1, hwgroup); } - return ide_stopped; /* the original code did this here (?) */ + return ide_stopped; } return ide_error(drive, "multwrite_intr", stat); } @@ -383,7 +346,13 @@ { if (IDE_CONTROL_REG) OUT_BYTE(drive->ctl,IDE_CONTROL_REG); - OUT_BYTE(rq->nr_sectors,IDE_NSECTOR_REG); + + /* + * for clustered kio requests, nr_sectors can be much bigger than + * what ide hw can handle. do those in chunks of 256 + */ + OUT_BYTE(rq->nr_sectors < 256 ? rq->nr_sectors : 0, IDE_NSECTOR_REG); + #ifdef CONFIG_BLK_DEV_PDC4030 if (drive->select.b.lba || IS_PDC4030_DRIVE) { #else /* !CONFIG_BLK_DEV_PDC4030 */ @@ -454,7 +423,6 @@ * * Except when you get an error it seems... */ - hwgroup->wrq = *rq; /* scratchpad */ ide_set_handler (drive, &multwrite_intr, WAIT_CMD, NULL); if (ide_multwrite(drive, drive->mult_count)) { unsigned long flags; Index: drivers/ide/ide-dma.c =================================================================== RCS file: /cvs/linux-2.4-xfs/linux/drivers/ide/ide-dma.c,v retrieving revision 1.6 diff -u -r1.6 ide-dma.c --- drivers/ide/ide-dma.c 2000/10/09 21:21:05 1.6 +++ drivers/ide/ide-dma.c 2000/12/11 14:30:01 @@ -197,9 +197,10 @@ if (OK_STAT(stat,DRIVE_READY,drive->bad_wstat|DRQ_STAT)) { if (!dma_stat) { struct request *rq = HWGROUP(drive)->rq; - rq = HWGROUP(drive)->rq; - for (i = rq->nr_sectors; i > 0;) { - i -= rq->current_nr_sectors; + if ((i = rq->nr_sectors) > 256) + i = 256; + while (i > 0) { + i -= rq->hard_cur_sectors; ide_end_request(1, HWGROUP(drive)); } return ide_stopped; @@ -209,16 +210,81 @@ return ide_error(drive, "dma_intr", stat); } -static int ide_build_sglist (ide_hwif_t *hwif, struct request *rq) +/* + * Build kiobuf based scatter-gather list. Return number of segments. + */ +static int ide_kio_sgl(struct request *rq, struct scatterlist *sg) +{ + unsigned int total_bytes, nr_bytes, nr_seg, sg_size, segs; + struct kiobuf *kiobuf = rq->kiobuf; + unsigned int c_off, nr_pgs = 0, nr_sects = 0; + unsigned char *va; + + /* + * For leftover requests, only rq->nr_sectors gets adjusted. + */ + total_bytes = rq->nr_sectors << 9; + + /* + * find offset + */ + c_off = kiobuf->offset + (((kiobuf->length >> 9) - rq->nr_sectors) << 9); + if (c_off >= PAGE_SIZE) { + nr_pgs = (c_off >> PAGE_SHIFT); + c_off &= ~PAGE_MASK; + } + + /* + * limit to max ide hw size, if need be + */ + if ((segs = rq->nr_segments) > PRD_ENTRIES) + segs = PRD_ENTRIES; + + /* + * now build sg list + */ + for (sg_size = 0, nr_seg = 0, nr_bytes = 0; nr_seg < segs && nr_sects <= 256; nr_seg++, nr_sects += sg_size >> 9) { + va = c_off + (char *) page_address(kiobuf->maplist[nr_pgs]); + nr_pgs++; + if (c_off) { + if ((sg_size = PAGE_SIZE - c_off) > total_bytes) + sg_size = total_bytes; + + c_off = 0; + } else if (nr_bytes + PAGE_SIZE > total_bytes) { + sg_size = total_bytes - nr_bytes; + } else { + sg_size = PAGE_SIZE; + } + + nr_bytes += sg_size; + memset(&sg[nr_seg], 0, sizeof(struct scatterlist)); + sg[nr_seg].address = va; + sg[nr_seg].length = sg_size; + } + + /* + * your plain paranoia + */ + if ((nr_bytes > total_bytes) || (nr_pgs > rq->kiobuf->nr_pages)) { + printk(KERN_ERR + "ide_kio_sgl: sgl bytes[%d], request bytes[%d]\n" + "ide_kio_sgl: pgcnt[%d], kiobuf->pgcnt[%d]!\n", + nr_bytes, total_bytes, nr_pgs, kiobuf->nr_pages); + BUG(); + } + + return nr_seg; +} + +/* + * Build bh based scatter-gather list. Return number of segments. + */ +static int ide_bh_sgl(struct request *rq, struct scatterlist *sg) { struct buffer_head *bh; - struct scatterlist *sg = hwif->sg_table; int nents = 0; - if (rq->cmd == READ) - hwif->sg_dma_direction = PCI_DMA_FROMDEVICE; - else - hwif->sg_dma_direction = PCI_DMA_TODEVICE; bh = rq->bh; do { unsigned char *virt_addr = bh->b_data; @@ -229,12 +295,32 @@ break; size += bh->b_size; } - memset(&sg[nents], 0, sizeof(*sg)); + memset(&sg[nents], 0, sizeof(struct scatterlist)); sg[nents].address = virt_addr; sg[nents].length = size; nents++; } while (bh != NULL); + return nents; +} + +static int ide_build_sglist (ide_hwif_t *hwif, struct request *rq) +{ + struct scatterlist *sg = hwif->sg_table; + int nents; + + if (rq->cmd == READ) + hwif->sg_dma_direction = PCI_DMA_FROMDEVICE; + else + hwif->sg_dma_direction = PCI_DMA_TODEVICE; + + if (rq->bh) + nents = ide_bh_sgl(rq, sg); + else if (rq->kiobuf) + nents = ide_kio_sgl(rq, sg); + else + panic("neither rq->bh nor rq->kiobuf in place\n"); + return pci_map_sg(hwif->pci_dev, sg, nents, hwif->sg_dma_direction); } @@ -257,6 +343,9 @@ HWIF(drive)->sg_nents = i = ide_build_sglist(HWIF(drive), HWGROUP(drive)->rq); + if (!i) + return 0; + sg = HWIF(drive)->sg_table; while (i && sg_dma_len(sg)) { u32 cur_addr; @@ -266,7 +355,7 @@ cur_len = sg_dma_len(sg); while (cur_len) { - if (++count >= PRD_ENTRIES) { + if (count++ >= PRD_ENTRIES) { printk("%s: DMA table too small\n", drive->name); pci_unmap_sg(HWIF(drive)->pci_dev, HWIF(drive)->sg_table, @@ -292,8 +381,11 @@ i--; } - if (!count) + if (!count) { + struct request *rq = HWGROUP(drive)->rq; printk("%s: empty DMA table?\n", drive->name); + printk("sector %lu, hard sector %lu, nr_sectors %lu, hard nr_sectors %lu, cur nr_sectors %lu\n", rq->sector, rq->hard_sector, rq->nr_sectors, rq->hard_nr_sectors, rq->current_nr_sectors); + } else if (!is_trm290_chipset) *--table |= cpu_to_le32(0x80000000); Index: drivers/ide/ide-probe.c =================================================================== RCS file: /cvs/linux-2.4-xfs/linux/drivers/ide/ide-probe.c,v retrieving revision 1.10 diff -u -r1.10 ide-probe.c --- drivers/ide/ide-probe.c 2000/10/09 21:21:05 1.10 +++ drivers/ide/ide-probe.c 2000/12/11 14:30:01 @@ -763,7 +763,7 @@ #ifdef CONFIG_BLK_DEV_PDC4030 *max_sect++ = ((hwif->chipset == ide_pdc4030) ? 127 : MAX_SECTORS); #else - *max_sect++ = MAX_SECTORS; + *max_sect++ = 256; #endif *max_ra++ = MAX_READAHEAD; } Index: drivers/ide/ide.c =================================================================== RCS file: /cvs/linux-2.4-xfs/linux/drivers/ide/ide.c,v retrieving revision 1.14 diff -u -r1.14 ide.c --- drivers/ide/ide.c 2000/11/01 21:35:42 1.14 +++ drivers/ide/ide.c 2000/12/11 14:30:02 @@ -502,6 +502,90 @@ return 1; /* drive ready: *might* be interrupting */ } +int ide_end_kio_request(struct request *rq, int uptodate, int sectors) +{ + int pgcnt = 0, nr_pages; + size_t curr_offset; + unsigned int nr_bytes, total_bytes, page_sectors, nr_sectors; + char *va = NULL; + + if ((nr_sectors = rq->hard_nr_sectors) > 256) + nr_sectors = 256; + + nr_pages = rq->kiobuf->nr_pages; + total_bytes = nr_sectors << 9; + curr_offset = rq->kiobuf->offset; + + /* + * In the case of leftover requests, the kiobuf->length + * remains the same, but rq->nr_sectors would be smaller. + * Adjust curr_offset in this case. If not a leftover, + * the following makes no difference. + */ + curr_offset += (((rq->kiobuf->length >> 9) - nr_sectors) << 9); + + /* How far into the kiobuf is the offset? */ + if (curr_offset > PAGE_SIZE) { + pgcnt = curr_offset >> PAGE_SHIFT; + curr_offset &= ~PAGE_MASK; + } + + /* + * Reusing the pgcnt and va value from above: + * Harvest pages to account for number of sectors + * passed into function. + */ + for (nr_bytes = 0; pgcnt < nr_pages && nr_bytes != total_bytes; pgcnt++) { + va = page_address(rq->kiobuf->maplist[pgcnt]) + curr_offset; + /* First page or final page? Partial page? */ + if (curr_offset) { + if (PAGE_SIZE - curr_offset > total_bytes) + page_sectors = total_bytes >> 9; + else + page_sectors = (PAGE_SIZE - curr_offset) >> 9; + curr_offset = 0; + } else if (nr_bytes + PAGE_SIZE > total_bytes) { + page_sectors = (total_bytes - nr_bytes) >> 9; + } else { + page_sectors = PAGE_SIZE >> 9; + } + nr_bytes += (page_sectors << 9); + /* Leftover sectors in this page (onward)? */ + if (sectors < page_sectors) { + rq->hard_nr_sectors -= sectors; + rq->sector += sectors; + rq->hard_cur_sectors = page_sectors - sectors; + va += (sectors << 9); /* Update for rq->buffer */ + sectors = 0; + break; + } + + /* Mark this page as done */ + rq->nr_segments--; + rq->hard_nr_sectors -= page_sectors; + rq->sector += page_sectors; + if (!uptodate && rq->kiobuf->errno) + rq->kiobuf->errno = -EIO; + sectors -= page_sectors; + } + + rq->current_nr_sectors = rq->hard_cur_sectors; + rq->nr_sectors = rq->hard_nr_sectors; + + /* Check for leftovers */ + if (rq->hard_nr_sectors) { + rq->buffer = va; + if (!rq->nr_segments) + rq->nr_segments = 1; + return 1; + } + + if (rq->kiobuf->end_io) + rq->kiobuf->end_io(rq->kiobuf); + + return 0; +} + /* * This is our end_request replacement function. */ @@ -509,11 +593,24 @@ { struct request *rq; unsigned long flags; + int r; spin_lock_irqsave(&io_request_lock, flags); rq = hwgroup->rq; + + /* + * kiovec request + */ + if (rq->kiobuf) + r = ide_end_kio_request(rq, uptodate, rq->hard_cur_sectors); + else + r = end_that_request_first(rq, uptodate, hwgroup->drive->name); - if (!end_that_request_first(rq, uptodate, hwgroup->drive->name)) { + /* + * finish request if either kiovec request or bh based request + * is done + */ + if (!r) { add_blkdev_randomness(MAJOR(rq->rq_dev)); blkdev_dequeue_request(rq); hwgroup->rq = NULL; Index: drivers/ide/pdc4030.c =================================================================== RCS file: /cvs/linux-2.4-xfs/linux/drivers/ide/pdc4030.c,v retrieving revision 1.2 diff -u -r1.2 pdc4030.c --- drivers/ide/pdc4030.c 2000/05/02 21:36:51 1.2 +++ drivers/ide/pdc4030.c 2000/12/11 14:30:02 @@ -453,7 +453,7 @@ static ide_startstop_t promise_write (ide_drive_t *drive) { ide_hwgroup_t *hwgroup = HWGROUP(drive); - struct request *rq = &hwgroup->wrq; + struct request *rq = hwgroup->rq; #ifdef DEBUG_WRITE printk(KERN_DEBUG "%s: promise_write: sectors(%ld-%ld), " @@ -541,7 +541,6 @@ } if (!drive->unmask) __cli(); /* local CPU only */ - HWGROUP(drive)->wrq = *rq; /* scratchpad */ return promise_write(drive); } else { Index: fs/pagebuf/page_buf_io.c =================================================================== RCS file: /cvs/linux-2.4-xfs/linux/fs/pagebuf/page_buf_io.c,v retrieving revision 1.36 diff -u -r1.36 page_buf_io.c --- fs/pagebuf/page_buf_io.c 2000/11/21 10:24:08 1.36 +++ fs/pagebuf/page_buf_io.c 2000/12/11 14:30:04 @@ -1016,8 +1016,8 @@ /* arr is sized for worst case */ struct buffer_head *bh, *head, *arr[PAGE_CACHE_SIZE / 512]; int blocksize, bbits = inode->i_sb->s_blocksize_bits; - unsigned long kaddr = 0; long nr, i, status = 0; + char *kaddr = NULL; if (!PageLocked(page) || (inode->i_op->pagebuf_bmap == NULL)) PAGE_BUG(page); Index: include/linux/blkdev.h =================================================================== RCS file: /cvs/linux-2.4-xfs/linux/include/linux/blkdev.h,v retrieving revision 1.26 diff -u -r1.26 blkdev.h --- include/linux/blkdev.h 2000/10/11 20:41:07 1.26 +++ include/linux/blkdev.h 2000/12/11 14:30:06 @@ -39,7 +39,7 @@ unsigned long start_time; unsigned long sector; unsigned long nr_sectors; - unsigned long hard_sector, hard_nr_sectors; + unsigned long hard_sector, hard_nr_sectors, hard_cur_sectors; unsigned int nr_segments; unsigned int nr_hw_segments; unsigned long current_nr_sectors; Index: include/linux/ide.h =================================================================== RCS file: /cvs/linux-2.4-xfs/linux/include/linux/ide.h,v retrieving revision 1.25 diff -u -r1.25 ide.h --- include/linux/ide.h 2000/11/01 21:35:42 1.25 +++ include/linux/ide.h 2000/12/11 14:30:08 @@ -497,7 +497,6 @@ ide_hwif_t *hwif; /* ptr to current hwif in linked-list */ struct request *rq; /* current request */ struct timer_list timer; /* failsafe timer */ - struct request wrq; /* local copy of current write rq */ unsigned long poll_timeout; /* timeout value during long polls */ ide_expiry_t *expiry; /* queried upon timeouts */ } ide_hwgroup_t; @@ -657,6 +656,7 @@ #include void ide_end_request(byte uptodate, ide_hwgroup_t *hwgroup); +int ide_end_kio_request(struct request *rq, int uptodate, int sectors); /* * This is used for (nearly) all data transfers from/to the IDE interface Index: include/linux/major.h =================================================================== RCS file: /cvs/linux-2.4-xfs/linux/include/linux/major.h,v retrieving revision 1.22 diff -u -r1.22 major.h --- include/linux/major.h 2000/10/23 18:56:35 1.22 +++ include/linux/major.h 2000/12/11 14:30:11 @@ -165,8 +165,23 @@ (SCSI_DISK_MAJOR(M) \ || (M) == SCSI_CDROM_MAJOR) -static __inline__ int scsi_blk_major(int m) { +static inline int scsi_blk_major(int m) +{ return SCSI_BLK_MAJOR(m); +} + +/* + * Tests for IDE devices + */ +#define IDE_DISK_MAJOR(M) ((M) == IDE0_MAJOR || (M) == IDE1_MAJOR || \ + (M) == IDE2_MAJOR || (M) == IDE3_MAJOR || \ + (M) == IDE4_MAJOR || (M) == IDE5_MAJOR || \ + (M) == IDE6_MAJOR || (M) == IDE7_MAJOR || \ + (M) == IDE8_MAJOR || (M) == IDE9_MAJOR) + +static inline int ide_blk_major(int m) +{ + return IDE_DISK_MAJOR(m); } #endif