diff -urN --exclude-from /home/axboe/exclude /opt/kernel/linux-2.4.5/drivers/block/ll_rw_blk.c linux/drivers/block/ll_rw_blk.c --- /opt/kernel/linux-2.4.5/drivers/block/ll_rw_blk.c Thu Apr 12 21:15:52 2001 +++ linux/drivers/block/ll_rw_blk.c Mon May 28 01:31:08 2001 @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -129,6 +130,7 @@ static int high_queued_sectors, low_queued_sectors; static int batch_requests, queue_nr_requests; static DECLARE_WAIT_QUEUE_HEAD(blk_buffers_wait); +unsigned long blk_max_low_pfn; static inline int get_max_sectors(kdev_t dev) { @@ -267,6 +269,32 @@ q->make_request_fn = mfn; } +/** + * blk_queue_bounce_limit - set bounce buffer limit for queue + * @q: the request queue for the device + * @bus_addr: bus address limit + * + * Description: + * Different hardware can have different requirements as to what pages + * it can do I/O directly to. A low level driver can call + * blk_queue_bounce_limit to have lower memory pages allocated as bounce + * buffers for doing I/O to pages residing above @page. By default + * the block layer sets this to the highest numbered "low" memory page, ie + * one the driver can still call bio_page() and get a valid address on. + **/ +void blk_queue_bounce_limit(request_queue_t *q, unsigned long dma_addr) +{ + q->bounce_limit = mem_map + (dma_addr >> PAGE_SHIFT); + + /* + * set page alloc gfp mask for bounce pages + */ + q->bounce_gfp = GFP_BUFFER; + + if (dma_addr > BLK_BOUNCE_HIGH) + q->bounce_gfp |= __GFP_DMA32; +} + static inline int ll_new_segment(request_queue_t *q, struct request *req, int max_segments) { if (req->nr_segments < max_segments) { @@ -279,7 +307,7 @@ static int ll_back_merge_fn(request_queue_t *q, struct request *req, struct buffer_head *bh, int max_segments) { - if (req->bhtail->b_data + req->bhtail->b_size == bh->b_data) + if (bh_bus(req->bhtail) + req->bhtail->b_size == bh_bus(bh)) return 1; return ll_new_segment(q, req, max_segments); } @@ -287,7 +315,7 @@ static int ll_front_merge_fn(request_queue_t *q, struct request *req, struct buffer_head *bh, int max_segments) { - if (bh->b_data + bh->b_size == req->bh->b_data) + if (bh_bus(bh) + bh->b_size == bh_bus(req->bh)) return 1; return ll_new_segment(q, req, max_segments); } @@ -297,7 +325,7 @@ { int total_segments = req->nr_segments + next->nr_segments; - if (req->bhtail->b_data + req->bhtail->b_size == next->bh->b_data) + if (bh_bus(req->bhtail) + req->bhtail->b_size == bh_bus(next->bh)) total_segments--; if (total_segments > max_segments) @@ -431,6 +459,8 @@ */ q->plug_device_fn = generic_plug_device; q->head_active = 1; + + blk_queue_bounce_limit(q, BLK_BOUNCE_HIGH); } #define blkdev_free_rq(list) list_entry((list)->next, struct request, table); @@ -704,9 +734,7 @@ * driver. Create a bounce buffer if the buffer data points into * high memory - keep the original buffer otherwise. */ -#if CONFIG_HIGHMEM - bh = create_bounce(rw, bh); -#endif + bh = blk_queue_bounce(q, rw, bh); /* look for a free request. */ /* @@ -751,8 +779,13 @@ elevator->elevator_merge_cleanup_fn(q, req, count); bh->b_reqnext = req->bh; req->bh = bh; + /* + * may not be valid, but queues not having bounce + * enabled for highmem pages must not look at + * ->buffer anyway + */ req->buffer = bh->b_data; - req->current_nr_sectors = count; + req->current_nr_sectors = req->hard_cur_sectors = count; req->sector = req->hard_sector = sector; req->nr_sectors = req->hard_nr_sectors += count; blk_started_io(count); @@ -802,7 +835,7 @@ req->errors = 0; req->hard_sector = req->sector = sector; req->hard_nr_sectors = req->nr_sectors = count; - req->current_nr_sectors = count; + req->current_nr_sectors = req->hard_cur_sectors = count; req->nr_segments = 1; /* Always 1 for a new request. */ req->nr_hw_segments = 1; /* Always 1 for a new request. */ req->buffer = bh->b_data; @@ -1125,6 +1158,7 @@ req->nr_sectors = req->hard_nr_sectors; req->current_nr_sectors = bh->b_size >> 9; + req->hard_cur_sectors = req->current_nr_sectors; if (req->nr_sectors < req->current_nr_sectors) { req->nr_sectors = req->current_nr_sectors; printk("end_request: buffer-list destroyed\n"); @@ -1202,6 +1236,8 @@ low_queued_sectors / 2, queue_nr_requests); + blk_max_low_pfn = max_low_pfn; + #ifdef CONFIG_AMIGA_Z2RAM z2_init(); #endif @@ -1322,3 +1358,5 @@ EXPORT_SYMBOL(blkdev_release_request); EXPORT_SYMBOL(generic_unplug_device); EXPORT_SYMBOL(queued_sectors); +EXPORT_SYMBOL(blk_queue_bounce_limit); +EXPORT_SYMBOL(blk_max_low_pfn); diff -urN --exclude-from /home/axboe/exclude /opt/kernel/linux-2.4.5/drivers/block/loop.c linux/drivers/block/loop.c --- /opt/kernel/linux-2.4.5/drivers/block/loop.c Thu Apr 12 04:05:14 2001 +++ linux/drivers/block/loop.c Sun May 27 18:26:41 2001 @@ -453,9 +453,7 @@ goto err; } -#if CONFIG_HIGHMEM - rbh = create_bounce(rw, rbh); -#endif + rbh = blk_queue_bounce(q, rw, rbh); /* * file backed, queue for loop_thread to handle diff -urN --exclude-from /home/axboe/exclude /opt/kernel/linux-2.4.5/fs/buffer.c linux/fs/buffer.c --- /opt/kernel/linux-2.4.5/fs/buffer.c Sat May 26 13:30:49 2001 +++ linux/fs/buffer.c Mon May 28 14:41:50 2001 @@ -1267,13 +1267,11 @@ bh->b_page = page; if (offset >= PAGE_SIZE) BUG(); - if (PageHighMem(page)) - /* - * This catches illegal uses and preserves the offset: - */ - bh->b_data = (char *)(0 + offset); - else - bh->b_data = page_address(page) + offset; + /* + * ->virtual is NULL on highmem pages, so we can catch the + * offset even though using page_address on it + */ + bh->b_data = page_address(page) + offset; } /* diff -urN --exclude-from /home/axboe/exclude /opt/kernel/linux-2.4.5/include/asm-i386/kmap_types.h linux/include/asm-i386/kmap_types.h --- /opt/kernel/linux-2.4.5/include/asm-i386/kmap_types.h Thu Apr 12 21:11:39 2001 +++ linux/include/asm-i386/kmap_types.h Mon May 28 01:28:29 2001 @@ -6,6 +6,7 @@ KM_BOUNCE_WRITE, KM_SKB_DATA, KM_SKB_DATA_SOFTIRQ, + KM_BH_IRQ, KM_TYPE_NR }; diff -urN --exclude-from /home/axboe/exclude /opt/kernel/linux-2.4.5/include/asm-i386/page.h linux/include/asm-i386/page.h --- /opt/kernel/linux-2.4.5/include/asm-i386/page.h Sat Apr 28 00:48:20 2001 +++ linux/include/asm-i386/page.h Sun May 27 18:26:59 2001 @@ -116,7 +116,8 @@ #define __va(x) ((void *)((unsigned long)(x)+PAGE_OFFSET)) #define virt_to_page(kaddr) (mem_map + (__pa(kaddr) >> PAGE_SHIFT)) #define VALID_PAGE(page) ((page - mem_map) < max_mapnr) - +#define page_to_phys(page) (((page) - mem_map) * PAGE_SIZE) +#define page_to_bus(page) page_to_phys((page)) #endif /* __KERNEL__ */ diff -urN --exclude-from /home/axboe/exclude /opt/kernel/linux-2.4.5/include/asm-i386/pci.h linux/include/asm-i386/pci.h --- /opt/kernel/linux-2.4.5/include/asm-i386/pci.h Sat May 26 13:30:49 2001 +++ linux/include/asm-i386/pci.h Mon May 28 02:07:54 2001 @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -78,6 +79,27 @@ /* Nothing to do */ } +/* + * pci_{map,unmap}_single_page maps a kernel page to a dma_addr_t. identical + * to pci_map_single, but takes a struct page instead of a virtual address + */ +extern inline dma_addr_t pci_map_page(struct pci_dev *hwdev, struct page *page, + size_t size, int offset, int direction) +{ + if (direction == PCI_DMA_NONE) + BUG(); + + return (page - mem_map) * PAGE_SIZE + offset; +} + +extern inline void pci_unmap_page(struct pci_dev *hwdev, dma_addr_t dma_address, + size_t size, int direction) +{ + if (direction == PCI_DMA_NONE) + BUG(); + /* Nothing to do */ +} + /* Map a set of buffers described by scatterlist in streaming * mode for DMA. This is the scather-gather version of the * above pci_map_single interface. Here the scatter gather list @@ -96,8 +118,26 @@ extern inline int pci_map_sg(struct pci_dev *hwdev, struct scatterlist *sg, int nents, int direction) { + int i; + if (direction == PCI_DMA_NONE) BUG(); + + /* + * temporary 2.4 hack + */ + for (i = 0; i < nents; i++ ) { + if (sg[i].address && sg[i].page) + BUG(); + else if (!sg[i].address && !sg[i].page) + BUG(); + + if (sg[i].page) + sg[i].dma_address = page_to_bus(sg[i].page) + sg[i].offset; + else + sg[i].dma_address = virt_to_bus(sg[i].address); + } + return nents; } @@ -167,10 +207,9 @@ /* These macros should be used after a pci_map_sg call has been done * to get bus addresses of each of the SG entries and their lengths. * You should only work with the number of sg entries pci_map_sg - * returns, or alternatively stop on the first sg_dma_len(sg) which - * is 0. + * returns. */ -#define sg_dma_address(sg) (virt_to_bus((sg)->address)) +#define sg_dma_address(sg) ((sg)->dma_address) #define sg_dma_len(sg) ((sg)->length) /* Return the index of the PCI controller for device. */ diff -urN --exclude-from /home/axboe/exclude /opt/kernel/linux-2.4.5/include/asm-i386/scatterlist.h linux/include/asm-i386/scatterlist.h --- /opt/kernel/linux-2.4.5/include/asm-i386/scatterlist.h Mon Dec 30 12:01:10 1996 +++ linux/include/asm-i386/scatterlist.h Mon May 28 02:02:36 2001 @@ -1,12 +1,34 @@ #ifndef _I386_SCATTERLIST_H #define _I386_SCATTERLIST_H +/* + * temporary measure, include a page and offset. + */ struct scatterlist { - char * address; /* Location data is to be transferred to */ + struct page * page; /* Location for highmem page, if any */ + char * address; /* Location data is to be transferred to, NULL for + * highmem page */ char * alt_address; /* Location of actual if address is a * dma indirect buffer. NULL otherwise */ + dma_addr_t dma_address; unsigned int length; + unsigned int offset;/* for highmem, page offset */ }; + +extern inline void set_bh_sg(struct scatterlist *sg, struct buffer_head *bh) +{ + if (PageHighMem(bh->b_page)) { + sg->page = bh->b_page; + sg->offset = bh_offset(bh); + sg->address = NULL; + } else { + sg->page = NULL; + sg->offset = 0; + sg->address = bh->b_data; + } + + sg->length = bh->b_size; +} #define ISA_DMA_THRESHOLD (0x00ffffff) diff -urN --exclude-from /home/axboe/exclude /opt/kernel/linux-2.4.5/include/linux/blkdev.h linux/include/linux/blkdev.h --- /opt/kernel/linux-2.4.5/include/linux/blkdev.h Sat Apr 28 00:48:49 2001 +++ linux/include/linux/blkdev.h Mon May 28 02:07:38 2001 @@ -38,7 +38,7 @@ unsigned long hard_sector, hard_nr_sectors; unsigned int nr_segments; unsigned int nr_hw_segments; - unsigned long current_nr_sectors; + unsigned long current_nr_sectors, hard_cur_sectors; void * special; char * buffer; struct semaphore * sem; @@ -112,6 +112,9 @@ */ char head_active; + struct page *bounce_limit; + int bounce_gfp; + /* * Is meant to protect the queue in the future instead of * io_request_lock @@ -123,6 +126,27 @@ */ wait_queue_head_t wait_for_request; }; + +extern unsigned long blk_max_low_pfn; + +#define BLK_BOUNCE_HIGH (blk_max_low_pfn * PAGE_SIZE) +#define BLK_BOUNCE_4G PCI_MAX_DMA32 + +extern void blk_queue_bounce_limit(request_queue_t *, unsigned long); + +#ifdef CONFIG_HIGHMEM +extern struct buffer_head *create_bounce(int, struct buffer_head *, int); +extern inline struct buffer_head *blk_queue_bounce(request_queue_t *q, int rw, + struct buffer_head *bh) +{ + if (bh->b_page <= q->bounce_limit) + return bh; + + return create_bounce(rw, bh, q->bounce_gfp); +} +#else +#define blk_queue_bounce(q, rw, bh) (bh) +#endif struct blk_dev_struct { /* diff -urN --exclude-from /home/axboe/exclude /opt/kernel/linux-2.4.5/include/linux/fs.h linux/include/linux/fs.h --- /opt/kernel/linux-2.4.5/include/linux/fs.h Sat May 26 13:30:50 2001 +++ linux/include/linux/fs.h Sun May 27 18:41:23 2001 @@ -271,6 +271,8 @@ #define bh_offset(bh) ((unsigned long)(bh)->b_data & ~PAGE_MASK) +#define bh_bus(bh) (page_to_bus((bh)->b_page) + bh_offset((bh))) + extern void set_bh_page(struct buffer_head *bh, struct page *page, unsigned long offset); #define touch_buffer(bh) SetPageReferenced(bh->b_page) diff -urN --exclude-from /home/axboe/exclude /opt/kernel/linux-2.4.5/include/linux/highmem.h linux/include/linux/highmem.h --- /opt/kernel/linux-2.4.5/include/linux/highmem.h Sat Apr 28 00:48:31 2001 +++ linux/include/linux/highmem.h Mon May 28 02:07:43 2001 @@ -13,7 +13,8 @@ /* declarations for linux/mm/highmem.c */ FASTCALL(unsigned int nr_free_highpages(void)); -extern struct buffer_head * create_bounce(int rw, struct buffer_head * bh_orig); +extern struct buffer_head *create_bounce(int rw, struct buffer_head * bh_orig, + int gfp_mask); static inline char *bh_kmap(struct buffer_head *bh) @@ -26,6 +27,26 @@ kunmap(bh->b_page); } +/* + * remember to add offset! + */ +static inline char *bh_kmap_irq(struct buffer_head *bh) +{ + unsigned long addr = (unsigned long) kmap_atomic(bh->b_page, KM_BH_IRQ); + + if (addr & ~PAGE_MASK) + BUG(); + + return (char *) addr + bh_offset(bh); +} + +static inline void bh_kunmap_irq(char *buffer) +{ + char *ptr = buffer - (((unsigned long) buffer) & ~PAGE_MASK); + + kunmap_atomic(ptr, KM_BH_IRQ); +} + #else /* CONFIG_HIGHMEM */ static inline unsigned int nr_free_highpages(void) { return 0; } @@ -39,6 +60,8 @@ #define bh_kmap(bh) ((bh)->b_data) #define bh_kunmap(bh) do { } while (0) +#define bh_kmap_irq(bh) ((bh)->b_data) +#define bh_kunmap_irq(bh) do { } while (0) #endif /* CONFIG_HIGHMEM */ diff -urN --exclude-from /home/axboe/exclude /opt/kernel/linux-2.4.5/include/linux/pci.h linux/include/linux/pci.h --- /opt/kernel/linux-2.4.5/include/linux/pci.h Sat May 26 13:30:50 2001 +++ linux/include/linux/pci.h Mon May 28 02:11:10 2001 @@ -314,6 +314,8 @@ #define PCI_DMA_FROMDEVICE 2 #define PCI_DMA_NONE 3 +#define PCI_MAX_DMA32 (0xffffffff) + #define DEVICE_COUNT_COMPATIBLE 4 #define DEVICE_COUNT_IRQ 2 #define DEVICE_COUNT_DMA 2 --- /opt/kernel/linux-2.4.5/drivers/block/elevator.c Fri Feb 16 01:58:34 2001 +++ linux/drivers/block/elevator.c Mon May 28 17:56:24 2001 @@ -110,7 +110,6 @@ break; } else if (__rq->sector - count == bh->b_rsector) { ret = ELEVATOR_FRONT_MERGE; - __rq->elevator_sequence -= count; *req = __rq; break; }