diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/Documentation/DMA-mapping.txt linux/Documentation/DMA-mapping.txt --- /opt/kernel/linux-2.4.9/Documentation/DMA-mapping.txt Wed Aug 15 10:54:44 2001 +++ linux/Documentation/DMA-mapping.txt Tue Aug 21 11:03:52 2001 @@ -6,14 +6,15 @@ Jakub Jelinek Most of the 64bit platforms have special hardware that translates bus -addresses (DMA addresses) to physical addresses similarly to how page -tables and/or TLB translate virtual addresses to physical addresses. -This is needed so that e.g. PCI devices can access with a Single Address -Cycle (32bit DMA address) any page in the 64bit physical address space. -Previously in Linux those 64bit platforms had to set artificial limits on -the maximum RAM size in the system, so that the virt_to_bus() static scheme -works (the DMA address translation tables were simply filled on bootup -to map each bus address to the physical page __pa(bus_to_virt())). +addresses (DMA addresses) to physical addresses. This is similar to +how page tables and/or a TLB translates virtual addresses to physical +addresses on a cpu. This is needed so that e.g. PCI devices can +access with a Single Address Cycle (32bit DMA address) any page in the +64bit physical address space. Previously in Linux those 64bit +platforms had to set artificial limits on the maximum RAM size in the +system, so that the virt_to_bus() static scheme works (the DMA address +translation tables were simply filled on bootup to map each bus +address to the physical page __pa(bus_to_virt())). So that Linux can use the dynamic DMA mapping, it needs some help from the drivers, namely it has to take into account that DMA addresses should be @@ -28,9 +29,11 @@ #include -is in your driver. This file will obtain for you the definition of -the dma_addr_t type which should be used everywhere you hold a DMA -(bus) address returned from the DMA mapping functions. +is in your driver. This file will obtain for you the definition of the +dma_addr_t (which can hold only SAC addresses) and dma64_addr_t (which +can hold both SAC and DAC addresses) types which should be used +everywhere you hold a DMA (bus) address returned from the DMA mapping +functions. What memory is DMA'able? @@ -237,6 +240,12 @@ driver needs regions sized smaller than a page, you may prefer using the pci_pool interface, described below. +The consistent DMA mapping interfaces, for non-NULL dev, will always +return a DMA address which is SAC (Single Address Cycle) addressible. +Even if the device indicates (via PCI dma mask) that it may address +the upper 32-bits and thus perform DAC cycles, consistent allocation +will still only return 32-bit PCI addresses for DMA. + It returns two values: the virtual address which you can use to access it from the CPU and dma_handle which you pass to the card. @@ -492,6 +501,249 @@ supports dynamic DMA mapping in hardware) in your driver structures and/or in the card registers. + 64-bit DMA and DAC cycle support + +In order to understand how to use the 64-bit portion of Linux's DMA +support interfaces, some background is necessary. + +PCI system implementations (from Linux's perspective) generally fall +into 2 categories: + +1) The PCI address space equals the physical memory address space. + Only the lower 4GB of physical memory may be addressed by + single-address cycles on PCI. All upper memory can only be + accessed via PCI dual-address cycles. + + The x86 and current ia64 systems are examples. + +2) All of physical memory can be accessed from PCI single-address + cycles via a remapping mechanism. 32-bit PCI addresses within + a certain range are "translated" into a full physical memory + address. These "translations" are setup by pci_map_{single,sg}() + and torn down by pci_unmap_{single,sg}() + + These platforms also provide a way to get at all of physical memory + using dual-address cycles. The "translation" is not used in these + cases, but often performance is lower when using this scheme. This + issue is central to certain aspects of Linux's 64-bit DMA APIs. + + Sparc64, Alpha, and ppc64 are examples of such systems. + + The author's belief is that any sane 64-bit platform will do + something along these lines until 64-bit PCI is so ubiquitous that + 32-bit PCI cards need not be considered anymore (ie. a few years + from now at best). + + Therefore, the author also believes that future ia64 systems are + likely to have PCI 32-bit "translation" mechanisms in hardware. + +On platforms in category #1 the situation is pretty straight forward. + +The category #2 platforms present an issue because they provide two +mechanisms to get at physical memory. Which is best? + +There are two situation where you want to always get DAC (Dual +Address Cycle) addresses from the 64-bit PCI dma APIs: + +1) Your device can hold onto an enormous number of concurrent + DMA transactions at once. This is bad for "translation" schemes + because the translations are a finite resource. + + A great and often mentioned example are compute cluster cards. + These devices can require DMA mappings to several gigabytes of + main memory at once. In such a case, DAC addresses are the only + way to go about doing this. + +2) Your device can _only_ generic DAC cycles. + +This is a device attribute, and we provide a way for the driver +to tell the kernel about them: + + pci_change_dma_flag(struct pci_dev *dev, + unsigned int on, + unsigned int off); + +This turns off and on various DMA attribute flags for a device. Here +are the first two attributes, corresponding to situations #1 and #2 +(respectively) above: + + PCI_DMA_FLAG_HUGE_MAPS + + Setting this bit informs the kernel that your device may need + to hold onto an enormous number of DMA mappings at once, and + thus DAC addressing should be used for everything. + + Do not set this bit unless you absolutely _need_ it. This can + degrade performance of the DMA transfers on some platforms. + If you do not set this bit, the platform specific PCI support + layer will decide if you receive SAC (Single Address Cycle) or + DAC addresses. + +and + + PCI_DMA_FLAG_DAC_ONLY + + Setting this bit tells the kernel that your device generates + only DAC addressing cycles. + +There is another flag used to describe DAC addressing limitations +the hardware may have: + + PCI_DMA_FLAG_NO_4GB_CROSS + + The device cannot handle properly an address/len pair + which would cross a 4GB boundary. When set, the PCI + platform and generic layers will work together to make + sure that such address/length pairs are broken up before + they reach your driver. You need never worry about this + issue once you have set this flag. + +Next, we have an interface so the driver can see if DAC operation +should or can be used: + + int pci_dac_cycles_ok(struct pci_dev *pdev); + +Returns a boolean indicating whether the driver should use DAC +addressing or not. If this returns false, then the driver MUST +reconfigure the attributes of pci_dev and try again with some +SAC configuration. + +So let us show how the driver for a device supporting DAC might begin +probing: + + #define MYDEV_PCI_ADDRESS_BITS64 ((u64)0xffffffffffffffff) + #define MYDEV_PCI_ADDRESS_BITS32 ((u64)0x00000000ffffffff) + + int using_dac; + + /* Try turning on DAC support in the PCI layer first, + * if this fails we simply fall back to SAC operation. + */ + pci_set_dma_mask(mp->pdev, MYDEV_PCI_ADDRESS_BITS64); + + /* We are just a normal device and do not hold onto huge + * numbers of mappings at once. We do support SAC addressing. + * + * This call is superfluous since all device DMA flags default + * to off. + */ + pci_change_dma_flag(mp->pdev, 0, PCI_DMA_FLAG_HUGE_MAPS); + + if (!pci_dac_cycles_ok(mp->pdev)) { + if (!pci_dma_supported(mp->pdev, MYDEV_PCI_ADDRESS_BITS32)) + return -ENODEV; + using_dac = 0; + } else { + using_dac = 1; + } + + if (using_dac) + writel(PCI_MODE_DAC, mp->regs + PCI_MODE_REG); + else + writel(0, mp->regs + PCI_MODE_REG); + +Note that pci_dac_cycles_ok can fail not only because DAC addressing +is not possible. It may also fail because using DAC addressing is +undesirable. For example, this would happen if performance of DMA +transfers will be faster using SAC addressing. + +If your device is PCI_DMA_FLAG_DAC_ONLY and pci_dac_cycles_ok fails +then you cannot use DMA with this device. Usually this will cause +the device probe to fail. + +The rest of the driver work is relatively easy. There is a 64-bit +equivalent of each DMA mapping routine mentioned in the earlier +sections of this document. + +If you are using the pci64_*() routines below, and have enabled DAC +via the mask and flag bits, but you absolutely must use a SAC address +in a certain situation: simply use the non-64bit mapping interfaces +above. + +Let us check them out one by one: + + void *pci64_alloc_consistent(struct pci_dev *hwdev, size_t size, + dma64_addr_t *dma_handle); + void pci64_free_consistent(struct pci_dev *hwdev, size_t size, + void *vaddr, dma64_addr_t dma_handle); + +These two routines act the same as pci_{alloc,free}_consistent() +except that they can return DAC addresses. Next: + +dma64_addr_t pci64_map_page(struct pci_dev *hwdev, + struct page *page, unsigned long offset, + size_t size, int direction); +void pci64_unmap_page(struct pci_dev *hwdev, dma64_addr_t dma_addr, + size_t size, int direction); + +Same as pci_{map,unmap}_page(), except that they can return DAC +addresses. Note that there are no pci64_{map,unmap}_single(), +because on HIGHMEM systems the page+offset pair is the only way +to express each and every physical memory address in the machine. +Next: + +int pci64_map_sg(struct pci_dev *hwdev, struct scatterlist *sg, + int nents, int direction); +void pci64_unmap_sg(struct pci_dev *hwdev, struct scatterlist *sg, + int nhwents, int direction); + +Same as pci_{map,unmap}_sg(), except that they can return DAC +addresses. Next: + +void pci64_dma_sync_single(struct pci_dev *hwdev, dma64_addr_t dma_handle, + size_t size, int direction); +void pci64_dma_sync_sg(struct pci_dev *hwdev, struct scatterlist *sg, + int nelems, int direction); + +Same as pci_dma_sync_{single,sg}(), except that these routines know +how to deal with DAC addresses. + +That's it. Note that the above routines can and will return SAC +addresses unless you've told them otherwise. + +It is believed that the above allows the driver author to deal with +any reasonable combination of device and platform. At the same time +it allows drivers for SAC only devices to be coded more efficiently +by only using the 32-bit dma_addr_t type in the interfaces. + +There is an important assumption built into how a platform may +implement all of these interfaces, PLEASE READ THIS CAREFULLY: + + The kernel assumes that, for a device supporting DAC and SAC + addressing, that it will behave in a certain way. Specifically, + it assumes that if the device is given a DMA address with the top + 32-bits cleared to zero, it will _ALWAYS_ use a SAC cycle. + + Nearly all devices work this way, for performance reasons, since it + require one less PCI cycle for a DMA transaction. However, as always, + it is possible that there are a few broken devices out there which can + only operate in an "all DAC" or "all SAC" mode. So, we've provided + a way in which even devices like these can be handled correctly (this + is described later on). + + Why does the kernel make this assumption? The reason is that many + platforms in category #2 above interpret addresses seen in SAC cycles + different from those seen in DAC cycles. Specifically, only SAC cycle + addresses can go through the "translation" mechanism. Thus, things + would not work if the kernel gave you a SAC address and your device + used a DAC cycle to access it. + + If your device does not behave this way, you have two choices + of how to deal with this: + + a) Tell the kernel that your device is PCI_DMA_FLAG_DAC_ONLY. + This selection is UNDESIREABLE, because performance may + suffer on some platforms when you do this. + + b) The more desirable solution: configure the device to only + generate SAC addresses. + +If your device driver uses the 64-bit APIs and is not working properly +on some platforms, the reason may be that the device violates the +above assumptions built into the Linux kernel. + + Closing + This document, and the API itself, would not be in it's current form without the feedback and suggestions from numerous individuals. We would like to specifically mention, in no particular order, the @@ -503,3 +755,5 @@ Grant Grundler Jay Estabrook Thomas Sailer + Andrea Arcangeli + Jens Axboe diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/arch/alpha/kernel/pci_iommu.c linux/arch/alpha/kernel/pci_iommu.c --- /opt/kernel/linux-2.4.9/arch/alpha/kernel/pci_iommu.c Sun Aug 12 19:51:41 2001 +++ linux/arch/alpha/kernel/pci_iommu.c Tue Aug 21 11:03:52 2001 @@ -636,7 +636,7 @@ supported properly. */ int -pci_dma_supported(struct pci_dev *pdev, dma_addr_t mask) +pci_dma_supported(struct pci_dev *pdev, u64 mask) { struct pci_controller *hose; struct pci_iommu_arena *arena; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/arch/ia64/sn/io/pci_dma.c linux/arch/ia64/sn/io/pci_dma.c --- /opt/kernel/linux-2.4.9/arch/ia64/sn/io/pci_dma.c Thu Apr 12 21:16:35 2001 +++ linux/arch/ia64/sn/io/pci_dma.c Tue Aug 21 11:03:52 2001 @@ -182,7 +182,7 @@ } /* - * On sn1 we use the alt_address entry of the scatterlist to store + * On sn1 we use the orig_address entry of the scatterlist to store * the physical address corresponding to the given virtual address */ int diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/arch/parisc/kernel/ccio-dma.c linux/arch/parisc/kernel/ccio-dma.c --- /opt/kernel/linux-2.4.9/arch/parisc/kernel/ccio-dma.c Fri Feb 9 20:29:44 2001 +++ linux/arch/parisc/kernel/ccio-dma.c Tue Aug 21 11:03:52 2001 @@ -638,7 +638,7 @@ } -static int ccio_dma_supported( struct pci_dev *dev, dma_addr_t mask) +static int ccio_dma_supported( struct pci_dev *dev, u64 mask) { if (dev == NULL) { printk(MODULE_NAME ": EISA/ISA/et al not supported\n"); diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/arch/parisc/kernel/ccio-rm-dma.c linux/arch/parisc/kernel/ccio-rm-dma.c --- /opt/kernel/linux-2.4.9/arch/parisc/kernel/ccio-rm-dma.c Tue Dec 5 21:29:39 2000 +++ linux/arch/parisc/kernel/ccio-rm-dma.c Tue Aug 21 11:03:52 2001 @@ -93,7 +93,7 @@ } -static int ccio_dma_supported( struct pci_dev *dev, dma_addr_t mask) +static int ccio_dma_supported( struct pci_dev *dev, u64 mask) { if (dev == NULL) { printk(MODULE_NAME ": EISA/ISA/et al not supported\n"); diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/arch/parisc/kernel/pci-dma.c linux/arch/parisc/kernel/pci-dma.c --- /opt/kernel/linux-2.4.9/arch/parisc/kernel/pci-dma.c Fri Feb 9 20:29:44 2001 +++ linux/arch/parisc/kernel/pci-dma.c Tue Aug 21 11:03:52 2001 @@ -77,7 +77,7 @@ static inline void dump_resmap(void) {;} #endif -static int pa11_dma_supported( struct pci_dev *dev, dma_addr_t mask) +static int pa11_dma_supported( struct pci_dev *dev, u64 mask) { return 1; } diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/arch/parisc/kernel/sba_iommu.c linux/arch/parisc/kernel/sba_iommu.c --- /opt/kernel/linux-2.4.9/arch/parisc/kernel/sba_iommu.c Fri Feb 9 20:29:44 2001 +++ linux/arch/parisc/kernel/sba_iommu.c Tue Aug 21 11:03:52 2001 @@ -779,7 +779,7 @@ } static int -sba_dma_supported( struct pci_dev *dev, dma_addr_t mask) +sba_dma_supported( struct pci_dev *dev, u64 mask) { if (dev == NULL) { printk(MODULE_NAME ": EISA/ISA/et al not supported\n"); diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/arch/sparc64/kernel/iommu_common.c linux/arch/sparc64/kernel/iommu_common.c --- /opt/kernel/linux-2.4.9/arch/sparc64/kernel/iommu_common.c Tue Jun 20 02:59:38 2000 +++ linux/arch/sparc64/kernel/iommu_common.c Tue Aug 21 11:03:52 2001 @@ -12,7 +12,7 @@ */ #ifdef VERIFY_SG -int verify_lengths(struct scatterlist *sg, int nents, int npages) +static int verify_lengths(struct scatterlist *sg, int nents, int npages) { int sg_len, dma_len; int i, pgcount; @@ -22,8 +22,8 @@ sg_len += sg[i].length; dma_len = 0; - for (i = 0; i < nents && sg[i].dvma_length; i++) - dma_len += sg[i].dvma_length; + for (i = 0; i < nents && sg[i].dma_length; i++) + dma_len += sg[i].dma_length; if (sg_len != dma_len) { printk("verify_lengths: Error, different, sg[%d] dma[%d]\n", @@ -32,13 +32,13 @@ } pgcount = 0; - for (i = 0; i < nents && sg[i].dvma_length; i++) { + for (i = 0; i < nents && sg[i].dma_length; i++) { unsigned long start, end; - start = sg[i].dvma_address; + start = sg[i].dma_address; start = start & PAGE_MASK; - end = sg[i].dvma_address + sg[i].dvma_length; + end = sg[i].dma_address + sg[i].dma_length; end = (end + (PAGE_SIZE - 1)) & PAGE_MASK; pgcount += ((end - start) >> PAGE_SHIFT); @@ -55,15 +55,16 @@ return 0; } -int verify_one_map(struct scatterlist *dma_sg, struct scatterlist **__sg, int nents, iopte_t **__iopte) +static int verify_one_map(struct scatterlist *dma_sg, struct scatterlist **__sg, int nents, iopte_t **__iopte) { struct scatterlist *sg = *__sg; iopte_t *iopte = *__iopte; - u32 dlen = dma_sg->dvma_length; - u32 daddr = dma_sg->dvma_address; + u32 dlen = dma_sg->dma_length; + u32 daddr; unsigned int sglen; unsigned long sgaddr; + daddr = dma_sg->dma_address; sglen = sg->length; sgaddr = (unsigned long) sg->address; while (dlen > 0) { @@ -136,7 +137,7 @@ return nents; } -int verify_maps(struct scatterlist *sg, int nents, iopte_t *iopte) +static int verify_maps(struct scatterlist *sg, int nents, iopte_t *iopte) { struct scatterlist *dma_sg = sg; struct scatterlist *orig_dma_sg = dma_sg; @@ -147,7 +148,7 @@ if (nents <= 0) break; dma_sg++; - if (dma_sg->dvma_length == 0) + if (dma_sg->dma_length == 0) break; } @@ -174,14 +175,15 @@ verify_maps(sg, nents, iopte) < 0) { int i; - printk("verify_sglist: Crap, messed up mappings, dumping, iodma at %08x.\n", - (u32) (sg->dvma_address & PAGE_MASK)); + printk("verify_sglist: Crap, messed up mappings, dumping, iodma at "); + printk("%016lx.\n", sg->dma_address & PAGE_MASK); + for (i = 0; i < nents; i++) { printk("sg(%d): address(%p) length(%x) " - "dma_address[%08x] dma_length[%08x]\n", + "dma_address[%016lx] dma_length[%08x]\n", i, sg[i].address, sg[i].length, - sg[i].dvma_address, sg[i].dvma_length); + sg[i].dma_address, sg[i].dma_length); } } @@ -189,30 +191,23 @@ } #endif -/* Two addresses are "virtually contiguous" if and only if: - * 1) They are equal, or... - * 2) They are both on a page boundry - */ -#define VCONTIG(__X, __Y) (((__X) == (__Y)) || \ - (((__X) | (__Y)) << (64UL - PAGE_SHIFT)) == 0UL) - unsigned long prepare_sg(struct scatterlist *sg, int nents) { struct scatterlist *dma_sg = sg; unsigned long prev; - u32 dent_addr, dent_len; + u64 dent_addr, dent_len; prev = (unsigned long) sg->address; prev += (unsigned long) (dent_len = sg->length); - dent_addr = (u32) ((unsigned long)sg->address & (PAGE_SIZE - 1UL)); + dent_addr = (u64) ((unsigned long)sg->address & (PAGE_SIZE - 1UL)); while (--nents) { unsigned long addr; sg++; addr = (unsigned long) sg->address; if (! VCONTIG(prev, addr)) { - dma_sg->dvma_address = dent_addr; - dma_sg->dvma_length = dent_len; + dma_sg->dma_address = dent_addr; + dma_sg->dma_length = dent_len; dma_sg++; dent_addr = ((dent_addr + @@ -225,8 +220,8 @@ dent_len += sg->length; prev = addr + sg->length; } - dma_sg->dvma_address = dent_addr; - dma_sg->dvma_length = dent_len; + dma_sg->dma_address = dent_addr; + dma_sg->dma_length = dent_len; return ((unsigned long) dent_addr + (unsigned long) dent_len + diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/arch/sparc64/kernel/iommu_common.h linux/arch/sparc64/kernel/iommu_common.h --- /opt/kernel/linux-2.4.9/arch/sparc64/kernel/iommu_common.h Tue Dec 21 07:05:52 1999 +++ linux/arch/sparc64/kernel/iommu_common.h Tue Aug 21 11:03:52 2001 @@ -18,10 +18,7 @@ #undef VERIFY_SG #ifdef VERIFY_SG -int verify_lengths(struct scatterlist *sg, int nents, int npages); -int verify_one_map(struct scatterlist *dma_sg, struct scatterlist **__sg, int nents, iopte_t **__iopte); -int verify_maps(struct scatterlist *sg, int nents, iopte_t *iopte); -void verify_sglist(struct scatterlist *sg, int nents, iopte_t *iopte, int npages); +extern void verify_sglist(struct scatterlist *sg, int nents, iopte_t *iopte, int npages); #endif /* Two addresses are "virtually contiguous" if and only if: @@ -31,4 +28,4 @@ #define VCONTIG(__X, __Y) (((__X) == (__Y)) || \ (((__X) | (__Y)) << (64UL - PAGE_SHIFT)) == 0UL) -unsigned long prepare_sg(struct scatterlist *sg, int nents); +extern unsigned long prepare_sg(struct scatterlist *sg, int nents); diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/arch/sparc64/kernel/pci_iommu.c linux/arch/sparc64/kernel/pci_iommu.c --- /opt/kernel/linux-2.4.9/arch/sparc64/kernel/pci_iommu.c Fri May 25 00:00:58 2001 +++ linux/arch/sparc64/kernel/pci_iommu.c Tue Aug 21 11:03:52 2001 @@ -237,6 +237,36 @@ return ret; } +void *pci64_alloc_consistent(struct pci_dev *pdev, size_t size, dma64_addr_t *dma_addrp) +{ + if (!(pdev->dma_flags & __PCI_DMA_FLAG_MUST_DAC)) { + dma_addr_t tmp; + void *ret; + + ret = pci_alloc_consistent(pdev, size, &tmp); + if (ret != NULL) + *dma_addrp = (dma64_addr_t) tmp; + + return ret; + } else { + unsigned long order, first_page; + + size = PAGE_ALIGN(size); + order = get_order(size); + if (order >= 10) + return NULL; + + first_page = __get_free_pages(GFP_ATOMIC, order); + if (first_page == 0UL) + return NULL; + + memset((char *)first_page, 0, PAGE_SIZE << order); + *dma_addrp = PCI64_ADDR_BASE + __pa(first_page); + + return (void *) first_page; + } +} + /* Free and unmap a consistent DMA translation. */ void pci_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, dma_addr_t dvma) { @@ -299,6 +329,16 @@ free_pages((unsigned long)cpu, order); } +void pci64_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, dma64_addr_t dvma) +{ + if (!(pdev->dma_flags & __PCI_DMA_FLAG_MUST_DAC)) { + pci_free_consistent(pdev, size, cpu, + (dma_addr_t) dvma); + } + + /* Else, nothing to do. */ +} + /* Map a single buffer at PTR of SZ bytes for PCI DMA * in streaming mode. */ @@ -356,6 +396,20 @@ return 0; } +dma64_addr_t pci64_map_page(struct pci_dev *pdev, + struct page *page, unsigned long offset, + size_t sz, int direction) +{ + if (!(pdev->dma_flags & __PCI_DMA_FLAG_MUST_DAC)) { + return (dma64_addr_t) + pci_map_single(pdev, + page_address(page) + offset, + sz, direction); + } + + return PCI64_ADDR_BASE + (__pa(page_address(page)) + offset); +} + /* Unmap a single streaming mode DMA translation. */ void pci_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int direction) { @@ -378,7 +432,8 @@ ((bus_addr - iommu->page_table_map_base) >> PAGE_SHIFT); #ifdef DEBUG_PCI_IOMMU if (iopte_val(*base) == IOPTE_INVALID) - printk("pci_unmap_single called on non-mapped region %08x,%08x from %016lx\n", bus_addr, sz, __builtin_return_address(0)); + printk("pci_unmap_single called on non-mapped region %08x,%08x from %016lx\n", + bus_addr, sz, __builtin_return_address(0)); #endif bus_addr &= PAGE_MASK; @@ -423,18 +478,39 @@ spin_unlock_irqrestore(&iommu->lock, flags); } -static inline void fill_sg(iopte_t *iopte, struct scatterlist *sg, int nused, unsigned long iopte_protection) +void pci64_unmap_page(struct pci_dev *pdev, dma64_addr_t bus_addr, + size_t sz, int direction) +{ + if (!(pdev->dma_flags & __PCI_DMA_FLAG_MUST_DAC)) { + if ((bus_addr >> 32) != (dma64_addr_t) 0) + BUG(); + + return pci_unmap_single(pdev, (dma_addr_t) bus_addr, + sz, direction); + } + + /* If doing real DAC, there is nothing to do. */ +} + +#define SG_ENT_PHYS_ADDRESS(SG) \ + ((SG)->address ? \ + __pa((SG)->address) : \ + (__pa(page_address((SG)->page)) + (SG)->offset)) + +static inline void fill_sg(iopte_t *iopte, struct scatterlist *sg, + int nused, int nelems, unsigned long iopte_protection) { struct scatterlist *dma_sg = sg; + struct scatterlist *sg_end = sg + nelems; int i; for (i = 0; i < nused; i++) { unsigned long pteval = ~0UL; - u32 dma_npages; + u64 dma_npages; - dma_npages = ((dma_sg->dvma_address & (PAGE_SIZE - 1UL)) + - dma_sg->dvma_length + - ((u32)(PAGE_SIZE - 1UL))) >> PAGE_SHIFT; + dma_npages = ((dma_sg->dma_address & (PAGE_SIZE - 1UL)) + + dma_sg->dma_length + + ((PAGE_SIZE - 1UL))) >> PAGE_SHIFT; do { unsigned long offset; signed int len; @@ -447,7 +523,7 @@ for (;;) { unsigned long tmp; - tmp = (unsigned long) __pa(sg->address); + tmp = SG_ENT_PHYS_ADDRESS(sg); len = sg->length; if (((tmp ^ pteval) >> PAGE_SHIFT) != 0UL) { pteval = tmp & PAGE_MASK; @@ -479,10 +555,11 @@ * adjusting pteval along the way. Stop when we * detect a page crossing event. */ - while ((pteval << (64 - PAGE_SHIFT)) != 0UL && - pteval == __pa(sg->address) && + while (sg < sg_end && + (pteval << (64 - PAGE_SHIFT)) != 0UL && + (pteval == SG_ENT_PHYS_ADDRESS(sg)) && ((pteval ^ - (__pa(sg->address) + sg->length - 1UL)) >> PAGE_SHIFT) == 0UL) { + (SG_ENT_PHYS_ADDRESS(sg) + sg->length - 1UL)) >> PAGE_SHIFT) == 0UL) { pteval += sg->length; sg++; } @@ -505,14 +582,19 @@ struct pci_strbuf *strbuf; unsigned long flags, ctx, npages, iopte_protection; iopte_t *base; - u32 dma_base; + u64 dma_base; struct scatterlist *sgtmp; int used; /* Fast path single entry scatterlists. */ if (nelems == 1) { - sglist->dvma_address = pci_map_single(pdev, sglist->address, sglist->length, direction); - sglist->dvma_length = sglist->length; + sglist->dma_address = (dma64_addr_t) + pci_map_single(pdev, + (sglist->address ? + sglist->address : + (page_address(sglist->page) + sglist->offset)), + sglist->length, direction); + sglist->dma_length = sglist->length; return 1; } @@ -540,8 +622,8 @@ used = nelems; sgtmp = sglist; - while (used && sgtmp->dvma_length) { - sgtmp->dvma_address += dma_base; + while (used && sgtmp->dma_length) { + sgtmp->dma_address += dma_base; sgtmp++; used--; } @@ -559,7 +641,7 @@ iopte_protection = IOPTE_CONSISTENT(ctx); if (direction != PCI_DMA_TODEVICE) iopte_protection |= IOPTE_WRITE; - fill_sg (base, sglist, used, iopte_protection); + fill_sg (base, sglist, used, nelems, iopte_protection); #ifdef VERIFY_SG verify_sglist(sglist, nelems, base, npages); #endif @@ -574,6 +656,23 @@ return 0; } +int pci64_map_sg(struct pci_dev *pdev, struct scatterlist *sg, + int nelems, int direction) +{ + if ((pdev->dma_flags & __PCI_DMA_FLAG_MUST_DAC) != 0) { + int i; + + for (i = 0; i < nelems; i++) { + sg[i].dma_address = + PCI64_ADDR_BASE + SG_ENT_PHYS_ADDRESS(&sg[i]); + sg[i].dma_length = sg[i].length; + } + return nelems; + } + + return pci_map_sg(pdev, sg, nelems, direction); +} + /* Unmap a set of streaming mode DMA translations. */ void pci_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int direction) { @@ -582,7 +681,7 @@ struct pci_strbuf *strbuf; iopte_t *base; unsigned long flags, ctx, i, npages; - u32 bus_addr; + u64 bus_addr; if (direction == PCI_DMA_NONE) BUG(); @@ -591,20 +690,21 @@ iommu = pcp->pbm->iommu; strbuf = &pcp->pbm->stc; - bus_addr = sglist->dvma_address & PAGE_MASK; + bus_addr = sglist->dma_address & PAGE_MASK; for (i = 1; i < nelems; i++) - if (sglist[i].dvma_length == 0) + if (sglist[i].dma_length == 0) break; i--; - npages = (PAGE_ALIGN(sglist[i].dvma_address + sglist[i].dvma_length) - bus_addr) >> PAGE_SHIFT; + npages = (PAGE_ALIGN(sglist[i].dma_address + sglist[i].dma_length) - bus_addr) >> PAGE_SHIFT; base = iommu->page_table + ((bus_addr - iommu->page_table_map_base) >> PAGE_SHIFT); #ifdef DEBUG_PCI_IOMMU if (iopte_val(*base) == IOPTE_INVALID) - printk("pci_unmap_sg called on non-mapped region %08x,%d from %016lx\n", sglist->dvma_address, nelems, __builtin_return_address(0)); + printk("pci_unmap_sg called on non-mapped region %016lx,%d from %016lx\n", + sglist->dma_address, nelems, __builtin_return_address(0)); #endif spin_lock_irqsave(&iommu->lock, flags); @@ -616,7 +716,7 @@ /* Step 1: Kick data out of streaming buffers if necessary. */ if (strbuf->strbuf_enabled) { - u32 vaddr = bus_addr; + u32 vaddr = (u32) bus_addr; PCI_STC_FLUSHFLAG_INIT(strbuf); if (strbuf->strbuf_ctxflush && @@ -648,6 +748,15 @@ spin_unlock_irqrestore(&iommu->lock, flags); } +void pci64_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, + int nelems, int direction) +{ + if (!(pdev->dma_flags & __PCI_DMA_FLAG_MUST_DAC)) + return pci_unmap_sg(pdev, sglist, nelems, direction); + + /* If doing real DAC, there is nothing to do. */ +} + /* Make physical memory consistent for a single * streaming mode DMA translation after a transfer. */ @@ -709,6 +818,20 @@ spin_unlock_irqrestore(&iommu->lock, flags); } +void pci64_dma_sync_single(struct pci_dev *pdev, dma64_addr_t bus_addr, + size_t sz, int direction) +{ + if (!(pdev->dma_flags & __PCI_DMA_FLAG_MUST_DAC)) { + if ((bus_addr >> 32) != (dma64_addr_t) 0) + BUG(); + + return pci_dma_sync_single(pdev, (dma_addr_t) bus_addr, + sz, direction); + } + + /* If doing real DAC, there is nothing to do. */ +} + /* Make physical memory consistent for a set of streaming * mode DMA translations after a transfer. */ @@ -735,7 +858,7 @@ iopte_t *iopte; iopte = iommu->page_table + - ((sglist[0].dvma_address - iommu->page_table_map_base) >> PAGE_SHIFT); + ((sglist[0].dma_address - iommu->page_table_map_base) >> PAGE_SHIFT); ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL; } @@ -752,15 +875,15 @@ } while (((long)pci_iommu_read(matchreg)) < 0L); } else { unsigned long i, npages; - u32 bus_addr; + u64 bus_addr; - bus_addr = sglist[0].dvma_address & PAGE_MASK; + bus_addr = sglist[0].dma_address & PAGE_MASK; for(i = 1; i < nelems; i++) - if (!sglist[i].dvma_length) + if (!sglist[i].dma_length) break; i--; - npages = (PAGE_ALIGN(sglist[i].dvma_address + sglist[i].dvma_length) - bus_addr) >> PAGE_SHIFT; + npages = (PAGE_ALIGN(sglist[i].dma_address + sglist[i].dma_length) - bus_addr) >> PAGE_SHIFT; for (i = 0; i < npages; i++, bus_addr += PAGE_SIZE) pci_iommu_write(strbuf->strbuf_pflush, bus_addr); } @@ -774,10 +897,19 @@ spin_unlock_irqrestore(&iommu->lock, flags); } -int pci_dma_supported(struct pci_dev *pdev, dma_addr_t device_mask) +void pci64_dma_sync_sg(struct pci_dev *pdev, struct scatterlist *sglist, + int nelems, int direction) +{ + if (!(pdev->dma_flags & __PCI_DMA_FLAG_MUST_DAC)) + return pci_dma_sync_sg(pdev, sglist, nelems, direction); + + /* If doing real DAC, there is nothing to do. */ +} + +int pci_dma_supported(struct pci_dev *pdev, u64 device_mask) { struct pcidev_cookie *pcp = pdev->sysdata; - u32 dma_addr_mask; + u64 dma_addr_mask; if (pdev == NULL) { dma_addr_mask = 0xffffffff; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/arch/sparc64/kernel/sbus.c linux/arch/sparc64/kernel/sbus.c --- /opt/kernel/linux-2.4.9/arch/sparc64/kernel/sbus.c Fri May 25 00:00:58 2001 +++ linux/arch/sparc64/kernel/sbus.c Tue Aug 21 11:03:52 2001 @@ -376,18 +376,24 @@ spin_unlock_irqrestore(&iommu->lock, flags); } -static inline void fill_sg(iopte_t *iopte, struct scatterlist *sg, int nused, unsigned long iopte_bits) +#define SG_ENT_PHYS_ADDRESS(SG) \ + ((SG)->address ? \ + __pa((SG)->address) : \ + (__pa(page_address((SG)->page)) + (SG)->offset)) + +static inline void fill_sg(iopte_t *iopte, struct scatterlist *sg, int nused, int nelems, unsigned long iopte_bits) { struct scatterlist *dma_sg = sg; + struct scatterlist *sg_end = sg + nelems; int i; for (i = 0; i < nused; i++) { unsigned long pteval = ~0UL; - u32 dma_npages; + u64 dma_npages; - dma_npages = ((dma_sg->dvma_address & (PAGE_SIZE - 1UL)) + - dma_sg->dvma_length + - ((u32)(PAGE_SIZE - 1UL))) >> PAGE_SHIFT; + dma_npages = ((dma_sg->dma_address & (PAGE_SIZE - 1UL)) + + dma_sg->dma_length + + ((PAGE_SIZE - 1UL))) >> PAGE_SHIFT; do { unsigned long offset; signed int len; @@ -400,7 +406,7 @@ for (;;) { unsigned long tmp; - tmp = (unsigned long) __pa(sg->address); + tmp = (unsigned long) SG_ENT_PHYS_ADDRESS(sg); len = sg->length; if (((tmp ^ pteval) >> PAGE_SHIFT) != 0UL) { pteval = tmp & PAGE_MASK; @@ -432,10 +438,11 @@ * adjusting pteval along the way. Stop when we * detect a page crossing event. */ - while ((pteval << (64 - PAGE_SHIFT)) != 0UL && - pteval == __pa(sg->address) && + while (sg < sg_end && + (pteval << (64 - PAGE_SHIFT)) != 0UL && + (pteval == SG_ENT_PHYS_ADDRESS(sg)) && ((pteval ^ - (__pa(sg->address) + sg->length - 1UL)) >> PAGE_SHIFT) == 0UL) { + (SG_ENT_PHYS_ADDRESS(sg) + sg->length - 1UL)) >> PAGE_SHIFT) == 0UL) { pteval += sg->length; sg++; } @@ -461,8 +468,13 @@ /* Fast path single entry scatterlists. */ if (nents == 1) { - sg->dvma_address = sbus_map_single(sdev, sg->address, sg->length, dir); - sg->dvma_length = sg->length; + sg->dma_address = (dma64_addr_t) + sbus_map_single(sdev, + (sg->address ? + sg->address : + (page_address(sg->page) + sg->offset)), + sg->length, dir); + sg->dma_length = sg->length; return 1; } @@ -478,8 +490,8 @@ sgtmp = sg; used = nents; - while (used && sgtmp->dvma_length) { - sgtmp->dvma_address += dma_base; + while (used && sgtmp->dma_length) { + sgtmp->dma_address += dma_base; sgtmp++; used--; } @@ -489,7 +501,7 @@ if (dir != SBUS_DMA_TODEVICE) iopte_bits |= IOPTE_WRITE; - fill_sg(iopte, sg, used, iopte_bits); + fill_sg(iopte, sg, used, nents, iopte_bits); #ifdef VERIFY_SG verify_sglist(sg, nents, iopte, npages); #endif @@ -507,22 +519,22 @@ { unsigned long size, flags; struct sbus_iommu *iommu; - u32 dvma_base; + u64 dvma_base; int i; /* Fast path single entry scatterlists. */ if (nents == 1) { - sbus_unmap_single(sdev, sg->dvma_address, sg->dvma_length, direction); + sbus_unmap_single(sdev, sg->dma_address, sg->dma_length, direction); return; } - dvma_base = sg[0].dvma_address & PAGE_MASK; + dvma_base = sg[0].dma_address & PAGE_MASK; for (i = 0; i < nents; i++) { - if (sg[i].dvma_length == 0) + if (sg[i].dma_length == 0) break; } i--; - size = PAGE_ALIGN(sg[i].dvma_address + sg[i].dvma_length) - dvma_base; + size = PAGE_ALIGN(sg[i].dma_address + sg[i].dma_length) - dvma_base; iommu = sdev->bus->iommu; spin_lock_irqsave(&iommu->lock, flags); @@ -547,16 +559,16 @@ { struct sbus_iommu *iommu = sdev->bus->iommu; unsigned long flags, size; - u32 base; + u64 base; int i; - base = sg[0].dvma_address & PAGE_MASK; + base = sg[0].dma_address & PAGE_MASK; for (i = 0; i < nents; i++) { - if (sg[i].dvma_length == 0) + if (sg[i].dma_length == 0) break; } i--; - size = PAGE_ALIGN(sg[i].dvma_address + sg[i].dvma_length) - base; + size = PAGE_ALIGN(sg[i].dma_address + sg[i].dma_length) - base; spin_lock_irqsave(&iommu->lock, flags); strbuf_flush(iommu, base, size >> PAGE_SHIFT); diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/arch/sparc64/kernel/sparc64_ksyms.c linux/arch/sparc64/kernel/sparc64_ksyms.c --- /opt/kernel/linux-2.4.9/arch/sparc64/kernel/sparc64_ksyms.c Thu Aug 16 18:39:37 2001 +++ linux/arch/sparc64/kernel/sparc64_ksyms.c Tue Aug 21 11:03:52 2001 @@ -215,12 +215,20 @@ EXPORT_SYMBOL(insl); EXPORT_SYMBOL(pci_alloc_consistent); EXPORT_SYMBOL(pci_free_consistent); +EXPORT_SYMBOL(pci64_alloc_consistent); +EXPORT_SYMBOL(pci64_free_consistent); EXPORT_SYMBOL(pci_map_single); EXPORT_SYMBOL(pci_unmap_single); +EXPORT_SYMBOL(pci64_map_page); +EXPORT_SYMBOL(pci64_unmap_page); EXPORT_SYMBOL(pci_map_sg); EXPORT_SYMBOL(pci_unmap_sg); +EXPORT_SYMBOL(pci64_map_sg); +EXPORT_SYMBOL(pci64_unmap_sg); EXPORT_SYMBOL(pci_dma_sync_single); +EXPORT_SYMBOL(pci64_dma_sync_single); EXPORT_SYMBOL(pci_dma_sync_sg); +EXPORT_SYMBOL(pci64_dma_sync_sg); EXPORT_SYMBOL(pci_dma_supported); #endif diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/block/cciss.c linux/drivers/block/cciss.c --- /opt/kernel/linux-2.4.9/drivers/block/cciss.c Mon Jul 2 22:56:40 2001 +++ linux/drivers/block/cciss.c Tue Aug 21 11:17:37 2001 @@ -1129,7 +1129,7 @@ { temp64.val32.lower = cmd->SG[i].Addr.lower; temp64.val32.upper = cmd->SG[i].Addr.upper; - pci_unmap_single(hba[cmd->ctlr]->pdev, + pci_unmap_page(hba[cmd->ctlr]->pdev, temp64.val, cmd->SG[i].Len, (cmd->Request.Type.Direction == XFER_READ) ? PCI_DMA_FROMDEVICE : PCI_DMA_TODEVICE); @@ -1225,7 +1225,7 @@ static int cpq_back_merge_fn(request_queue_t *q, struct request *rq, struct buffer_head *bh, int max_segments) { - if (rq->bhtail->b_data + rq->bhtail->b_size == bh->b_data) + if (bh_bus(rq->bhtail) + rq->bhtail->b_size == bh_bus(bh)) return 1; return cpq_new_segment(q, rq, max_segments); } @@ -1233,7 +1233,7 @@ static int cpq_front_merge_fn(request_queue_t *q, struct request *rq, struct buffer_head *bh, int max_segments) { - if (bh->b_data + bh->b_size == rq->bh->b_data) + if (bh_bus(bh) + bh->b_size == bh_bus(rq->bh)) return 1; return cpq_new_segment(q, rq, max_segments); } @@ -1243,7 +1243,7 @@ { int total_segments = rq->nr_segments + nxt->nr_segments; - if (rq->bhtail->b_data + rq->bhtail->b_size == nxt->bh->b_data) + if (bh_bus(rq->bhtail) + rq->bhtail->b_size == bh_bus(nxt->bh)) total_segments--; if (total_segments > MAXSGENTRIES) @@ -1264,7 +1264,7 @@ ctlr_info_t *h= q->queuedata; CommandList_struct *c; int log_unit, start_blk, seg, sect; - char *lastdataend; + unsigned long lastdataend; struct buffer_head *bh; struct list_head *queue_head = &q->queue_head; struct request *creq; @@ -1272,10 +1272,15 @@ struct my_sg tmp_sg[MAXSGENTRIES]; int i; - // Loop till the queue is empty if or it is plugged + if (q->plugged) { + start_io(h); + return; + } + + // Loop till the queue is empty while (1) { - if (q->plugged || list_empty(queue_head)) { + if (list_empty(queue_head)) { start_io(h); return; } @@ -1323,12 +1328,12 @@ (int) creq->nr_sectors); #endif /* CCISS_DEBUG */ seg = 0; - lastdataend = NULL; + lastdataend = 0; sect = 0; while(bh) { sect += bh->b_size/512; - if (bh->b_data == lastdataend) + if (bh_bus(bh) == lastdataend) { // tack it on to the last segment tmp_sg[seg-1].len +=bh->b_size; lastdataend += bh->b_size; @@ -1336,9 +1341,10 @@ { if (seg == MAXSGENTRIES) BUG(); + tmp_sg[seg].page = bh->b_page; tmp_sg[seg].len = bh->b_size; - tmp_sg[seg].start_addr = bh->b_data; - lastdataend = bh->b_data + bh->b_size; + tmp_sg[seg].offset = bh_offset(bh); + lastdataend = bh_bus(bh) + bh->b_size; seg++; } bh = bh->b_reqnext; @@ -1347,9 +1353,8 @@ for (i=0; iSG[i].Len = tmp_sg[i].len; - temp64.val = (__u64) pci_map_single( h->pdev, - tmp_sg[i].start_addr, - tmp_sg[i].len, + temp64.val = (__u64) pci_map_page( h->pdev, + tmp_sg[i].page, tmp_sg[i].len, tmp_sg[i].offset, (c->Request.Type.Direction == XFER_READ) ? PCI_DMA_FROMDEVICE : PCI_DMA_TODEVICE); c->SG[i].Addr.lower = temp64.val32.lower; @@ -1940,6 +1945,7 @@ q = BLK_DEFAULT_QUEUE(MAJOR_NR + i); q->queuedata = hba[i]; blk_init_queue(q, do_cciss_request); + blk_queue_bounce_limit(q, hba[i]->pdev->dma_mask); blk_queue_headactive(q, 0); /* fill in the other Kernel structs */ diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/block/cciss.h linux/drivers/block/cciss.h --- /opt/kernel/linux-2.4.9/drivers/block/cciss.h Tue May 22 19:23:16 2001 +++ linux/drivers/block/cciss.h Tue Aug 21 11:11:56 2001 @@ -16,8 +16,9 @@ #define MAJOR_NR COMPAQ_CISS_MAJOR struct my_sg { - int len; - char *start_addr; + struct page *page; + unsigned short len; + unsigned short offset; }; struct ctlr_info; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/block/cpqarray.c linux/drivers/block/cpqarray.c --- /opt/kernel/linux-2.4.9/drivers/block/cpqarray.c Wed Jul 25 23:12:01 2001 +++ linux/drivers/block/cpqarray.c Tue Aug 21 11:17:16 2001 @@ -367,7 +367,7 @@ static int cpq_back_merge_fn(request_queue_t *q, struct request *rq, struct buffer_head *bh, int max_segments) { - if (rq->bhtail->b_data + rq->bhtail->b_size == bh->b_data) + if (bh_bus(rq->bhtail) + rq->bhtail->b_size == bh_bus(bh)) return 1; return cpq_new_segment(q, rq, max_segments); } @@ -375,7 +375,7 @@ static int cpq_front_merge_fn(request_queue_t *q, struct request *rq, struct buffer_head *bh, int max_segments) { - if (bh->b_data + bh->b_size == rq->bh->b_data) + if (bh_bus(bh) + bh->b_size == bh_bus(rq->bh)) return 1; return cpq_new_segment(q, rq, max_segments); } @@ -385,7 +385,7 @@ { int total_segments = rq->nr_segments + nxt->nr_segments; - if (rq->bhtail->b_data + rq->bhtail->b_size == nxt->bh->b_data) + if (bh_bus(rq->bhtail) + rq->bhtail->b_size == bh_bus(nxt->bh)) total_segments--; if (total_segments > SG_MAX) @@ -532,6 +532,7 @@ q = BLK_DEFAULT_QUEUE(MAJOR_NR + i); q->queuedata = hba[i]; blk_init_queue(q, do_ida_request); + blk_queue_bounce_limit(q, hba[i]->pci_dev->dma_mask); blk_queue_headactive(q, 0); blksize_size[MAJOR_NR+i] = ida_blocksizes + (i*256); hardsect_size[MAJOR_NR+i] = ida_hardsizes + (i*256); @@ -923,17 +924,22 @@ ctlr_info_t *h = q->queuedata; cmdlist_t *c; int seg, sect; - char *lastdataend; + unsigned long lastdataend; struct list_head * queue_head = &q->queue_head; struct buffer_head *bh; struct request *creq; struct my_sg tmp_sg[SG_MAX]; int i; -// Loop till the queue is empty if or it is plugged + if (q->plugged) { + start_io(h); + return; + } + +// Loop till the queue is empty while (1) { - if (q->plugged || list_empty(queue_head)) { + if (list_empty(queue_head)) { start_io(h); return; } @@ -973,19 +979,20 @@ printk("sector=%d, nr_sectors=%d\n", creq->sector, creq->nr_sectors); ); - seg = 0; lastdataend = NULL; + seg = lastdataend = 0; sect = 0; while(bh) { sect += bh->b_size/512; - if (bh->b_data == lastdataend) { + if (bh_bus(bh) == lastdataend) { tmp_sg[seg-1].size += bh->b_size; lastdataend += bh->b_size; } else { if (seg == SG_MAX) BUG(); + tmp_sg[seg].page = bh->b_page; tmp_sg[seg].size = bh->b_size; - tmp_sg[seg].start_addr = bh->b_data; - lastdataend = bh->b_data + bh->b_size; + tmp_sg[seg].offset = bh_offset(bh); + lastdataend = bh_bus(bh) + bh->b_size; seg++; } bh = bh->b_reqnext; @@ -994,9 +1001,9 @@ for( i=0; i < seg; i++) { c->req.sg[i].size = tmp_sg[i].size; - c->req.sg[i].addr = (__u32) pci_map_single( - h->pci_dev, tmp_sg[i].start_addr, - tmp_sg[i].size, + c->req.sg[i].addr = (__u32) pci_map_page( + h->pci_dev, tmp_sg[i].page, tmp_sg[i].size, + tmp_sg[i].offset, (creq->cmd == READ) ? PCI_DMA_FROMDEVICE : PCI_DMA_TODEVICE); } @@ -1103,7 +1110,7 @@ /* unmap the DMA mapping for all the scatter gather elements */ for(i=0; ireq.hdr.sg_cnt; i++) { - pci_unmap_single(hba[cmd->ctlr]->pci_dev, + pci_unmap_page(hba[cmd->ctlr]->pci_dev, cmd->req.sg[i].addr, cmd->req.sg[i].size, (cmd->req.hdr.cmd == IDA_READ) ? PCI_DMA_FROMDEVICE : PCI_DMA_TODEVICE); } diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/block/cpqarray.h linux/drivers/block/cpqarray.h --- /opt/kernel/linux-2.4.9/drivers/block/cpqarray.h Tue May 22 19:23:16 2001 +++ linux/drivers/block/cpqarray.h Tue Aug 21 11:11:56 2001 @@ -57,8 +57,9 @@ #ifdef __KERNEL__ struct my_sg { - int size; - char *start_addr; + struct page *page; + unsigned short size; + unsigned short offset; }; struct ctlr_info; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/block/elevator.c linux/drivers/block/elevator.c --- /opt/kernel/linux-2.4.9/drivers/block/elevator.c Fri Jul 20 05:59:41 2001 +++ linux/drivers/block/elevator.c Tue Aug 21 11:11:56 2001 @@ -110,7 +110,6 @@ break; } else if (__rq->sector - count == bh->b_rsector) { ret = ELEVATOR_FRONT_MERGE; - __rq->elevator_sequence -= count; *req = __rq; break; } diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/block/ll_rw_blk.c linux/drivers/block/ll_rw_blk.c --- /opt/kernel/linux-2.4.9/drivers/block/ll_rw_blk.c Sat Aug 4 07:37:09 2001 +++ linux/drivers/block/ll_rw_blk.c Tue Aug 21 13:18:43 2001 @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -124,6 +125,8 @@ */ static int queue_nr_requests, batch_requests; +unsigned long blk_max_low_pfn; + static inline int get_max_sectors(kdev_t dev) { if (!max_sectors[MAJOR(dev)]) @@ -131,7 +134,7 @@ return max_sectors[MAJOR(dev)][MINOR(dev)]; } -inline request_queue_t *__blk_get_queue(kdev_t dev) +inline request_queue_t *blk_get_queue(kdev_t dev) { struct blk_dev_struct *bdev = blk_dev + MAJOR(dev); @@ -141,22 +144,6 @@ return &blk_dev[MAJOR(dev)].request_queue; } -/* - * NOTE: the device-specific queue() functions - * have to be atomic! - */ -request_queue_t *blk_get_queue(kdev_t dev) -{ - request_queue_t *ret; - unsigned long flags; - - spin_lock_irqsave(&io_request_lock,flags); - ret = __blk_get_queue(dev); - spin_unlock_irqrestore(&io_request_lock,flags); - - return ret; -} - static int __blk_cleanup_queue(struct list_head *head) { struct request *rq; @@ -261,6 +248,57 @@ q->make_request_fn = mfn; } +/** + * blk_queue_bounce_limit - set bounce buffer limit for queue + * @q: the request queue for the device + * @dma_addr: bus address limit + * + * Description: + * Different hardware can have different requirements as to what pages + * it can do I/O directly to. A low level driver can call + * blk_queue_bounce_limit to have lower memory pages allocated as bounce + * buffers for doing I/O to pages residing above @page. By default + * the block layer sets this to the highest numbered "low" memory page. + **/ +void blk_queue_bounce_limit(request_queue_t *q, unsigned long dma_addr) +{ + unsigned long mb = dma_addr >> 20; + + /* + * keep this for debugging for now... + */ + if (dma_addr != BLK_BOUNCE_HIGH) + printk("blk: queue %p, I/O limit %luMb (mask %lx)\n", q, mb, dma_addr); + q->bounce_limit = mem_map + (dma_addr >> PAGE_SHIFT); + + /* + * just default to _not_ crossing a 4GB boundary, it's the easiest. + * drivers are free to define change this themselves of course. + */ + q->seg_bound_mask = 0xffffffff; +} + +/* + * can we merge the two segments, or do we need to start a new one? + */ +inline int blk_seg_merge_ok(request_queue_t *q, struct buffer_head *bh, + struct buffer_head *nxt) +{ + dma_addr_t b_bus, n_bus; + + b_bus = bh_bus(bh); + n_bus = bh_bus(nxt); + + if (b_bus + bh->b_size != n_bus) + return 0; + + if ((b_bus | q->seg_bound_mask) == + ((n_bus + nxt->b_size - 1) | q->seg_bound_mask)) + return 1; + + return 0; +} + static inline int ll_new_segment(request_queue_t *q, struct request *req, int max_segments) { if (req->nr_segments < max_segments) { @@ -273,16 +311,18 @@ static int ll_back_merge_fn(request_queue_t *q, struct request *req, struct buffer_head *bh, int max_segments) { - if (req->bhtail->b_data + req->bhtail->b_size == bh->b_data) + if (blk_seg_merge_ok(q, req->bhtail, bh)) return 1; + return ll_new_segment(q, req, max_segments); } static int ll_front_merge_fn(request_queue_t *q, struct request *req, struct buffer_head *bh, int max_segments) { - if (bh->b_data + bh->b_size == req->bh->b_data) + if (blk_seg_merge_ok(q, bh, req->bh)) return 1; + return ll_new_segment(q, req, max_segments); } @@ -291,7 +331,7 @@ { int total_segments = req->nr_segments + next->nr_segments; - if (req->bhtail->b_data + req->bhtail->b_size == next->bh->b_data) + if (blk_seg_merge_ok(q, req->bhtail, next->bh)) total_segments--; if (total_segments > max_segments) @@ -430,6 +470,8 @@ */ q->plug_device_fn = generic_plug_device; q->head_active = 1; + + blk_queue_bounce_limit(q, BLK_BOUNCE_HIGH); } #define blkdev_free_rq(list) list_entry((list)->next, struct request, table); @@ -696,9 +738,7 @@ * driver. Create a bounce buffer if the buffer data points into * high memory - keep the original buffer otherwise. */ -#if CONFIG_HIGHMEM - bh = create_bounce(rw, bh); -#endif + bh = blk_queue_bounce(q, rw, bh); /* look for a free request. */ /* @@ -743,8 +783,13 @@ elevator->elevator_merge_cleanup_fn(q, req, count); bh->b_reqnext = req->bh; req->bh = bh; + /* + * may not be valid, but queues not having bounce + * enabled for highmem pages must not look at + * ->buffer anyway + */ req->buffer = bh->b_data; - req->current_nr_sectors = count; + req->current_nr_sectors = req->hard_cur_sectors = count; req->sector = req->hard_sector = sector; req->nr_sectors = req->hard_nr_sectors += count; blk_started_io(count); @@ -794,7 +839,7 @@ req->errors = 0; req->hard_sector = req->sector = sector; req->hard_nr_sectors = req->nr_sectors = count; - req->current_nr_sectors = count; + req->current_nr_sectors = req->hard_cur_sectors = count; req->nr_segments = 1; /* Always 1 for a new request. */ req->nr_hw_segments = 1; /* Always 1 for a new request. */ req->buffer = bh->b_data; @@ -1104,6 +1149,7 @@ req->nr_sectors = req->hard_nr_sectors; req->current_nr_sectors = bh->b_size >> 9; + req->hard_cur_sectors = req->current_nr_sectors; if (req->nr_sectors < req->current_nr_sectors) { req->nr_sectors = req->current_nr_sectors; printk("end_request: buffer-list destroyed\n"); @@ -1152,7 +1198,7 @@ */ queue_nr_requests = 64; if (total_ram > MB(32)) - queue_nr_requests = 128; + queue_nr_requests = 256; /* * Batch frees according to queue length @@ -1160,6 +1206,8 @@ batch_requests = queue_nr_requests >> 3; printk("block: %d slots per queue, batch=%d\n", queue_nr_requests, batch_requests); + blk_max_low_pfn = max_low_pfn; + #ifdef CONFIG_AMIGA_Z2RAM z2_init(); #endif @@ -1272,10 +1320,12 @@ EXPORT_SYMBOL(end_that_request_last); EXPORT_SYMBOL(blk_init_queue); EXPORT_SYMBOL(blk_get_queue); -EXPORT_SYMBOL(__blk_get_queue); EXPORT_SYMBOL(blk_cleanup_queue); EXPORT_SYMBOL(blk_queue_headactive); EXPORT_SYMBOL(blk_queue_make_request); EXPORT_SYMBOL(generic_make_request); EXPORT_SYMBOL(blkdev_release_request); EXPORT_SYMBOL(generic_unplug_device); +EXPORT_SYMBOL(blk_queue_bounce_limit); +EXPORT_SYMBOL(blk_max_low_pfn); +EXPORT_SYMBOL(blk_seg_merge_ok); diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/block/loop.c linux/drivers/block/loop.c --- /opt/kernel/linux-2.4.9/drivers/block/loop.c Sat Jun 30 01:16:56 2001 +++ linux/drivers/block/loop.c Tue Aug 21 11:11:56 2001 @@ -453,9 +453,7 @@ goto err; } -#if CONFIG_HIGHMEM - rbh = create_bounce(rw, rbh); -#endif + rbh = blk_queue_bounce(q, rw, rbh); /* * file backed, queue for loop_thread to handle diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/ide/hpt34x.c linux/drivers/ide/hpt34x.c --- /opt/kernel/linux-2.4.9/drivers/ide/hpt34x.c Sun May 20 02:43:06 2001 +++ linux/drivers/ide/hpt34x.c Tue Aug 21 11:11:56 2001 @@ -425,6 +425,7 @@ hwif->autodma = 0; hwif->dmaproc = &hpt34x_dmaproc; + hwif->highmem = 1; } else { hwif->drives[0].autotune = 1; hwif->drives[1].autotune = 1; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/ide/hpt366.c linux/drivers/ide/hpt366.c --- /opt/kernel/linux-2.4.9/drivers/ide/hpt366.c Wed Aug 15 05:01:07 2001 +++ linux/drivers/ide/hpt366.c Tue Aug 21 11:11:56 2001 @@ -730,6 +730,7 @@ hwif->autodma = 1; else hwif->autodma = 0; + hwif->highmem = 1; } else { hwif->autodma = 0; hwif->drives[0].autotune = 1; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/ide/ide-disk.c linux/drivers/ide/ide-disk.c --- /opt/kernel/linux-2.4.9/drivers/ide/ide-disk.c Mon Aug 13 23:56:19 2001 +++ linux/drivers/ide/ide-disk.c Tue Aug 21 11:11:56 2001 @@ -27,9 +27,10 @@ * Version 1.09 added increment of rq->sector in ide_multwrite * added UDMA 3/4 reporting * Version 1.10 request queue changes, Ultra DMA 100 + * Version 1.11 Highmem I/O support, Jens Axboe */ -#define IDEDISK_VERSION "1.10" +#define IDEDISK_VERSION "1.11" #undef REALLY_SLOW_IO /* most systems can safely undef this */ @@ -139,7 +140,9 @@ byte stat; int i; unsigned int msect, nsect; + unsigned long flags; struct request *rq; + char *to; /* new way for dealing with premature shared PCI interrupts */ if (!OK_STAT(stat=GET_STAT(),DATA_READY,BAD_R_STAT)) { @@ -150,8 +153,8 @@ ide_set_handler(drive, &read_intr, WAIT_CMD, NULL); return ide_started; } + msect = drive->mult_count; - read_next: rq = HWGROUP(drive)->rq; if (msect) { @@ -160,14 +163,15 @@ msect -= nsect; } else nsect = 1; - idedisk_input_data(drive, rq->buffer, nsect * SECTOR_WORDS); + to = ide_map_buffer(rq, &flags); + idedisk_input_data(drive, to, nsect * SECTOR_WORDS); #ifdef DEBUG printk("%s: read: sectors(%ld-%ld), buffer=0x%08lx, remaining=%ld\n", drive->name, rq->sector, rq->sector+nsect-1, (unsigned long) rq->buffer+(nsect<<9), rq->nr_sectors-nsect); #endif + ide_unmap_buffer(to, &flags); rq->sector += nsect; - rq->buffer += nsect<<9; rq->errors = 0; i = (rq->nr_sectors -= nsect); if (((long)(rq->current_nr_sectors -= nsect)) <= 0) @@ -201,14 +205,16 @@ #endif if ((rq->nr_sectors == 1) ^ ((stat & DRQ_STAT) != 0)) { rq->sector++; - rq->buffer += 512; rq->errors = 0; i = --rq->nr_sectors; --rq->current_nr_sectors; if (((long)rq->current_nr_sectors) <= 0) ide_end_request(1, hwgroup); if (i > 0) { - idedisk_output_data (drive, rq->buffer, SECTOR_WORDS); + unsigned long flags; + char *to = ide_map_buffer(rq, &flags); + idedisk_output_data (drive, to, SECTOR_WORDS); + ide_unmap_buffer(to, &flags); ide_set_handler (drive, &write_intr, WAIT_CMD, NULL); return ide_started; } @@ -238,14 +244,14 @@ do { char *buffer; int nsect = rq->current_nr_sectors; - + unsigned long flags; + if (nsect > mcount) nsect = mcount; mcount -= nsect; - buffer = rq->buffer; + buffer = ide_map_buffer(rq, &flags); rq->sector += nsect; - rq->buffer += nsect << 9; rq->nr_sectors -= nsect; rq->current_nr_sectors -= nsect; @@ -259,7 +265,7 @@ } else { rq->bh = bh; rq->current_nr_sectors = bh->b_size >> 9; - rq->buffer = bh->b_data; + rq->hard_cur_sectors = rq->current_nr_sectors; } } @@ -268,6 +274,7 @@ * re-entering us on the last transfer. */ idedisk_output_data(drive, buffer, nsect<<7); + ide_unmap_buffer(buffer, &flags); } while (mcount); return 0; @@ -452,8 +459,11 @@ return ide_stopped; } } else { + unsigned long flags; + char *buffer = ide_map_buffer(rq, &flags); ide_set_handler (drive, &write_intr, WAIT_CMD, NULL); - idedisk_output_data(drive, rq->buffer, SECTOR_WORDS); + idedisk_output_data(drive, buffer, SECTOR_WORDS); + ide_unmap_buffer(buffer, &flags); } return ide_started; } diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/ide/ide-dma.c linux/drivers/ide/ide-dma.c --- /opt/kernel/linux-2.4.9/drivers/ide/ide-dma.c Mon Aug 13 23:56:19 2001 +++ linux/drivers/ide/ide-dma.c Tue Aug 21 11:20:49 2001 @@ -231,33 +231,42 @@ { struct buffer_head *bh; struct scatterlist *sg = hwif->sg_table; + unsigned long lastdataend; int nents = 0; if (hwif->sg_dma_active) BUG(); - + if (rq->cmd == READ) hwif->sg_dma_direction = PCI_DMA_FROMDEVICE; else hwif->sg_dma_direction = PCI_DMA_TODEVICE; + bh = rq->bh; + lastdataend = 0; do { - unsigned char *virt_addr = bh->b_data; - unsigned int size = bh->b_size; - - if (nents >= PRD_ENTRIES) - return 0; - - while ((bh = bh->b_reqnext) != NULL) { - if ((virt_addr + size) != (unsigned char *) bh->b_data) - break; - size += bh->b_size; + /* + * continue segment from before? + */ + if (bh_bus(bh) == lastdataend) { + sg[nents - 1].length += bh->b_size; + lastdataend += bh->b_size; + } else { + struct scatterlist *sge; + /* + * start new segment + */ + if (nents >= PRD_ENTRIES) + return 0; + + sge = &sg[nents]; + sge->page = bh->b_page; + sge->length = bh->b_size; + sge->offset = bh_offset(bh); + lastdataend = bh_bus(bh) + bh->b_size; + nents++; } - memset(&sg[nents], 0, sizeof(*sg)); - sg[nents].address = virt_addr; - sg[nents].length = size; - nents++; - } while (bh != NULL); + } while ((bh = bh->b_reqnext) != NULL); return pci_map_sg(hwif->pci_dev, sg, nents, hwif->sg_dma_direction); } @@ -285,7 +294,7 @@ return 0; sg = HWIF(drive)->sg_table; - while (i && sg_dma_len(sg)) { + while (i) { u32 cur_addr; u32 cur_len; @@ -299,36 +308,35 @@ */ while (cur_len) { - if (count++ >= PRD_ENTRIES) { - printk("%s: DMA table too small\n", drive->name); - goto use_pio_instead; - } else { - u32 xcount, bcount = 0x10000 - (cur_addr & 0xffff); - - if (bcount > cur_len) - bcount = cur_len; - *table++ = cpu_to_le32(cur_addr); - xcount = bcount & 0xffff; - if (is_trm290_chipset) - xcount = ((xcount >> 2) - 1) << 16; - if (xcount == 0x0000) { - /* - * Most chipsets correctly interpret a length of 0x0000 as 64KB, - * but at least one (e.g. CS5530) misinterprets it as zero (!). - * So here we break the 64KB entry into two 32KB entries instead. - */ - if (count++ >= PRD_ENTRIES) { - printk("%s: DMA table too small\n", drive->name); - goto use_pio_instead; - } - *table++ = cpu_to_le32(0x8000); - *table++ = cpu_to_le32(cur_addr + 0x8000); - xcount = 0x8000; - } - *table++ = cpu_to_le32(xcount); - cur_addr += bcount; - cur_len -= bcount; + u32 xcount, bcount = 0x10000 - (cur_addr & 0xffff); + + if (count++ >= PRD_ENTRIES) + BUG(); + + if (bcount > cur_len) + bcount = cur_len; + *table++ = cpu_to_le32(cur_addr); + xcount = bcount & 0xffff; + if (is_trm290_chipset) + xcount = ((xcount >> 2) - 1) << 16; + if (xcount == 0x0000) { + /* + * Most chipsets correctly interpret a length + * of 0x0000 as 64KB, but at least one + * (e.g. CS5530) misinterprets it as zero (!). + * So here we break the 64KB entry into two + * 32KB entries instead. + */ + if (count++ >= PRD_ENTRIES) + goto use_pio_instead; + + *table++ = cpu_to_le32(0x8000); + *table++ = cpu_to_le32(cur_addr + 0x8000); + xcount = 0x8000; } + *table++ = cpu_to_le32(xcount); + cur_addr += bcount; + cur_len -= bcount; } sg++; @@ -512,6 +520,20 @@ } #endif /* CONFIG_BLK_DEV_IDEDMA_TIMEOUT */ +#ifdef CONFIG_HIGHMEM +static inline void ide_toggle_bounce(ide_drive_t *drive, int on) +{ + unsigned long addr = BLK_BOUNCE_HIGH; + + if (on && drive->media == ide_disk && HWIF(drive)->highmem) + addr = HWIF(drive)->pci_dev->dma_mask; + + blk_queue_bounce_limit(&drive->queue, addr); +} +#else +#define ide_toggle_bounce(drive, on) +#endif + /* * ide_dmaproc() initiates/aborts DMA read/write operations on a drive. * @@ -534,18 +556,20 @@ ide_hwif_t *hwif = HWIF(drive); unsigned long dma_base = hwif->dma_base; byte unit = (drive->select.b.unit & 0x01); - unsigned int count, reading = 0; + unsigned int count, reading = 0, set_high = 1; byte dma_stat; switch (func) { case ide_dma_off: printk("%s: DMA disabled\n", drive->name); + set_high = 0; case ide_dma_off_quietly: outb(inb(dma_base+2) & ~(1<<(5+unit)), dma_base+2); case ide_dma_on: drive->using_dma = (func == ide_dma_on); if (drive->using_dma) outb(inb(dma_base+2)|(1<<(5+unit)), dma_base+2); + ide_toggle_bounce(drive, set_high); return 0; case ide_dma_check: return config_drive_for_dma (drive); diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/ide/pdc202xx.c linux/drivers/ide/pdc202xx.c --- /opt/kernel/linux-2.4.9/drivers/ide/pdc202xx.c Mon Aug 13 23:56:19 2001 +++ linux/drivers/ide/pdc202xx.c Tue Aug 21 11:11:56 2001 @@ -891,6 +891,7 @@ #ifdef CONFIG_BLK_DEV_IDEDMA if (hwif->dma_base) { hwif->dmaproc = &pdc202xx_dmaproc; + hwif->highmem = 1; if (!noautodma) hwif->autodma = 1; } else { diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/ide/piix.c linux/drivers/ide/piix.c --- /opt/kernel/linux-2.4.9/drivers/ide/piix.c Mon Aug 13 23:56:19 2001 +++ linux/drivers/ide/piix.c Tue Aug 21 11:11:56 2001 @@ -521,6 +521,7 @@ if (!hwif->dma_base) return; + hwif->highmem = 1; #ifndef CONFIG_BLK_DEV_IDEDMA hwif->autodma = 0; #else /* CONFIG_BLK_DEV_IDEDMA */ diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/net/acenic.c linux/drivers/net/acenic.c --- /opt/kernel/linux-2.4.9/drivers/net/acenic.c Wed Aug 15 10:22:15 2001 +++ linux/drivers/net/acenic.c Tue Aug 21 11:03:52 2001 @@ -202,6 +202,7 @@ #define pci_free_consistent(cookie, size, ptr, dma_ptr) kfree(ptr) #define pci_map_single(cookie, address, size, dir) virt_to_bus(address) #define pci_unmap_single(cookie, address, size, dir) +#define pci_set_dma_mask(dev, mask) do { } while (0) #endif #if (LINUX_VERSION_CODE < 0x02032b) @@ -258,11 +259,6 @@ #define ace_mark_net_bh() {do{} while(0);} #define ace_if_down(dev) {do{} while(0);} #endif - -#ifndef pci_set_dma_mask -#define pci_set_dma_mask(dev, mask) dev->dma_mask = mask; -#endif - #if (LINUX_VERSION_CODE >= 0x02031b) #define NEW_NETINIT diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/pci/pci.c linux/drivers/pci/pci.c --- /opt/kernel/linux-2.4.9/drivers/pci/pci.c Mon Aug 13 23:49:21 2001 +++ linux/drivers/pci/pci.c Tue Aug 21 11:03:52 2001 @@ -832,7 +832,7 @@ } int -pci_set_dma_mask(struct pci_dev *dev, dma_addr_t mask) +pci_set_dma_mask(struct pci_dev *dev, u64 mask) { if(! pci_dma_supported(dev, mask)) return -EIO; @@ -842,6 +842,12 @@ return 0; } +void +pci_change_dma_flag(struct pci_dev *dev, unsigned int on, unsigned int off) +{ + dev->dma_flags |= on; + dev->dma_flags &= ~off; +} /* * Translate the low bits of the PCI base @@ -1954,6 +1960,7 @@ EXPORT_SYMBOL(pci_find_subsys); EXPORT_SYMBOL(pci_set_master); EXPORT_SYMBOL(pci_set_dma_mask); +EXPORT_SYMBOL(pci_change_dma_flag); EXPORT_SYMBOL(pci_assign_resource); EXPORT_SYMBOL(pci_register_driver); EXPORT_SYMBOL(pci_unregister_driver); diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/scsi/aha1542.c linux/drivers/scsi/aha1542.c --- /opt/kernel/linux-2.4.9/drivers/scsi/aha1542.c Wed May 2 01:05:00 2001 +++ linux/drivers/scsi/aha1542.c Tue Aug 21 11:03:52 2001 @@ -67,12 +67,10 @@ int nseg, int badseg) { - printk(KERN_CRIT "sgpnt[%d:%d] addr %p/0x%lx alt %p/0x%lx length %d\n", + printk(KERN_CRIT "sgpnt[%d:%d] addr %p/0x%lx length %d\n", badseg, nseg, sgpnt[badseg].address, SCSI_PA(sgpnt[badseg].address), - sgpnt[badseg].alt_address, - sgpnt[badseg].alt_address ? SCSI_PA(sgpnt[badseg].alt_address) : 0, sgpnt[badseg].length); /* @@ -716,7 +714,7 @@ unsigned char *ptr; printk(KERN_CRIT "Bad segment list supplied to aha1542.c (%d, %d)\n", SCpnt->use_sg, i); for (i = 0; i < SCpnt->use_sg; i++) { - printk(KERN_CRIT "%d: %x %x %d\n", i, (unsigned int) sgpnt[i].address, (unsigned int) sgpnt[i].alt_address, + printk(KERN_CRIT "%d: %p %d\n", i, sgpnt[i].address, sgpnt[i].length); }; printk(KERN_CRIT "cptr %x: ", (unsigned int) cptr); diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/scsi/aic7xxx/aic7xxx_linux_host.h linux/drivers/scsi/aic7xxx/aic7xxx_linux_host.h --- /opt/kernel/linux-2.4.9/drivers/scsi/aic7xxx/aic7xxx_linux_host.h Sat May 5 00:16:28 2001 +++ linux/drivers/scsi/aic7xxx/aic7xxx_linux_host.h Tue Aug 21 11:11:56 2001 @@ -81,7 +81,8 @@ present: 0, /* number of 7xxx's present */\ unchecked_isa_dma: 0, /* no memory DMA restrictions */\ use_clustering: ENABLE_CLUSTERING, \ - use_new_eh_code: 1 \ + use_new_eh_code: 1, \ + can_dma_32: 1 \ } #endif /* _AIC7XXX_LINUX_HOST_H_ */ diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/scsi/hosts.c linux/drivers/scsi/hosts.c --- /opt/kernel/linux-2.4.9/drivers/scsi/hosts.c Thu Jul 5 20:28:17 2001 +++ linux/drivers/scsi/hosts.c Tue Aug 21 11:11:56 2001 @@ -235,6 +235,7 @@ retval->cmd_per_lun = tpnt->cmd_per_lun; retval->unchecked_isa_dma = tpnt->unchecked_isa_dma; retval->use_clustering = tpnt->use_clustering; + retval->can_dma_32 = tpnt->can_dma_32; retval->select_queue_depths = tpnt->select_queue_depths; retval->max_sectors = tpnt->max_sectors; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/scsi/hosts.h linux/drivers/scsi/hosts.h --- /opt/kernel/linux-2.4.9/drivers/scsi/hosts.h Wed Aug 15 23:23:11 2001 +++ linux/drivers/scsi/hosts.h Tue Aug 21 12:19:26 2001 @@ -291,6 +291,8 @@ */ unsigned emulated:1; + unsigned can_dma_32:1; + /* * Name of proc directory */ @@ -390,6 +392,7 @@ unsigned in_recovery:1; unsigned unchecked_isa_dma:1; unsigned use_clustering:1; + unsigned can_dma_32:1; /* * True if this host was loaded as a loadable module */ diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/scsi/ips.c linux/drivers/scsi/ips.c --- /opt/kernel/linux-2.4.9/drivers/scsi/ips.c Thu Aug 16 18:49:49 2001 +++ linux/drivers/scsi/ips.c Tue Aug 21 12:00:50 2001 @@ -3743,6 +3743,8 @@ int i; sg = SC->request_buffer; + scb->org_sg_list = sg; + (void) pci_map_sg(ha->pcidev, sg, SC->use_sg, ips_command_direction[scb->scsi_cmd->cmnd[0]]); if (SC->use_sg == 1) { if (sg[0].length > ha->max_xfer) { @@ -3752,12 +3754,12 @@ scb->data_len = sg[0].length; scb->dcdb.transfer_length = scb->data_len; - scb->data_busaddr = VIRT_TO_BUS(sg[0].address); + scb->data_busaddr = sg_dma_address(&sg[0]); scb->sg_len = 0; } else { /* Check for the first Element being bigger than MAX_XFER */ if (sg[0].length > ha->max_xfer) { - scb->sg_list[0].address = VIRT_TO_BUS(sg[0].address); + scb->sg_list[0].address = sg_dma_address(&sg[0]); scb->sg_list[0].length = ha->max_xfer; scb->data_len = ha->max_xfer; scb->breakup = 0; @@ -3766,7 +3768,7 @@ } else { for (i = 0; i < SC->use_sg; i++) { - scb->sg_list[i].address = VIRT_TO_BUS(sg[i].address); + scb->sg_list[i].address = sg_dma_address(&sg[i]); scb->sg_list[i].length = sg[i].length; if (scb->data_len + sg[i].length > ha->max_xfer) { @@ -4441,11 +4443,11 @@ if (sg[0].length - (bk_save * ha->max_xfer)) { /* Further breakup required */ scb->data_len = ha->max_xfer; - scb->data_busaddr = VIRT_TO_BUS(sg[0].address + (bk_save * ha->max_xfer)); + scb->data_busaddr = sg_dma_address(&sg[0] + (bk_save * ha->max_xfer)); scb->breakup = bk_save + 1; } else { scb->data_len = sg[0].length - (bk_save * ha->max_xfer); - scb->data_busaddr = VIRT_TO_BUS(sg[0].address + (bk_save * ha->max_xfer)); + scb->data_busaddr = sg_dma_address(&sg[0] + (bk_save * ha->max_xfer)); } scb->dcdb.transfer_length = scb->data_len; @@ -4462,7 +4464,7 @@ /* pointed to by bk_save */ if (scb->sg_break) { scb->sg_len = 1; - scb->sg_list[0].address = VIRT_TO_BUS(sg[bk_save].address+ha->max_xfer*scb->sg_break); + scb->sg_list[0].address = sg_dma_address(&sg[bk_save] + ha->max_xfer*scb->sg_break); if (ha->max_xfer > sg[bk_save].length-ha->max_xfer * scb->sg_break) scb->sg_list[0].length = sg[bk_save].length-ha->max_xfer * scb->sg_break; else @@ -4480,7 +4482,7 @@ } else { /* ( sg_break == 0 ), so this is our first look at a new sg piece */ if (sg[bk_save].length > ha->max_xfer) { - scb->sg_list[0].address = VIRT_TO_BUS(sg[bk_save].address); + scb->sg_list[0].address = sg_dma_address(&sg[bk_save]); scb->sg_list[0].length = ha->max_xfer; scb->breakup = bk_save; scb->sg_break = 1; @@ -4493,7 +4495,7 @@ scb->sg_break = 0; /* We're only doing full units here */ for (i = bk_save; i < scb->scsi_cmd->use_sg; i++) { - scb->sg_list[i - bk_save].address = VIRT_TO_BUS(sg[i].address); + scb->sg_list[i - bk_save].address = sg_dma_address(&sg[i]); scb->sg_list[i - bk_save].length = sg[i].length; if (scb->data_len + sg[i].length > ha->max_xfer) { scb->breakup = i; /* sneaky, if not more work, than breakup is 0 */ @@ -4560,6 +4562,7 @@ break; } /* end case */ + pci_unmap_sg(ha->pcidev, scb->org_sg_list, scb->scsi_cmd->use_sg, ips_command_direction[scb->scsi_cmd->cmnd[0]]); return ; } #ifndef NO_IPS_CMDLINE diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/scsi/ips.h linux/drivers/scsi/ips.h --- /opt/kernel/linux-2.4.9/drivers/scsi/ips.h Fri Jul 20 06:08:13 2001 +++ linux/drivers/scsi/ips.h Tue Aug 21 11:54:06 2001 @@ -419,7 +419,8 @@ present : 0, \ unchecked_isa_dma : 0, \ use_clustering : ENABLE_CLUSTERING, \ - use_new_eh_code : 1 \ + use_new_eh_code : 1, \ + can_dma_32 : 1 \ } #endif @@ -1026,6 +1027,7 @@ u32 flags; u32 op_code; IPS_SG_LIST *sg_list; + struct scatterlist *org_sg_list; Scsi_Cmnd *scsi_cmd; struct ips_scb *q_next; ips_scb_callback callback; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/scsi/osst.c linux/drivers/scsi/osst.c --- /opt/kernel/linux-2.4.9/drivers/scsi/osst.c Fri Jul 20 06:18:15 2001 +++ linux/drivers/scsi/osst.c Tue Aug 21 11:03:52 2001 @@ -4933,7 +4933,6 @@ tb->sg[0].address = (unsigned char *)__get_free_pages(priority, order); if (tb->sg[0].address != NULL) { - tb->sg[0].alt_address = NULL; tb->sg[0].length = b_size; break; } @@ -4969,7 +4968,6 @@ tb = NULL; break; } - tb->sg[segs].alt_address = NULL; tb->sg[segs].length = b_size; got += b_size; segs++; @@ -5043,7 +5041,6 @@ normalize_buffer(STbuffer); return FALSE; } - STbuffer->sg[segs].alt_address = NULL; STbuffer->sg[segs].length = b_size; STbuffer->sg_segs += 1; got += b_size; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/scsi/qlogicfc.c linux/drivers/scsi/qlogicfc.c --- /opt/kernel/linux-2.4.9/drivers/scsi/qlogicfc.c Sun Aug 12 19:51:41 2001 +++ linux/drivers/scsi/qlogicfc.c Tue Aug 21 11:03:52 2001 @@ -21,6 +21,9 @@ * * Big endian support and dynamic DMA mapping added * by Jakub Jelinek . + * + * Conversion to final pci64 DMA interfaces + * by David S. Miller . */ /* @@ -63,31 +66,10 @@ #include "sd.h" #include "hosts.h" -#if 1 -/* Once pci64_ DMA mapping interface is in, kill this. */ -typedef dma_addr_t dma64_addr_t; -#define pci64_alloc_consistent(d,s,p) pci_alloc_consistent((d),(s),(p)) -#define pci64_free_consistent(d,s,c,a) pci_free_consistent((d),(s),(c),(a)) -#define pci64_map_single(d,c,s,dir) pci_map_single((d),(c),(s),(dir)) -#define pci64_map_sg(d,s,n,dir) pci_map_sg((d),(s),(n),(dir)) -#define pci64_unmap_single(d,a,s,dir) pci_unmap_single((d),(a),(s),(dir)) -#define pci64_unmap_sg(d,s,n,dir) pci_unmap_sg((d),(s),(n),(dir)) -#if BITS_PER_LONG > 32 #define pci64_dma_hi32(a) ((u32) (0xffffffff & (((u64)(a))>>32))) #define pci64_dma_lo32(a) ((u32) (0xffffffff & (((u64)(a))))) -#else -#define pci64_dma_hi32(a) 0 -#define pci64_dma_lo32(a) (a) -#endif /* BITS_PER_LONG */ -#define pci64_dma_build(hi,lo) (lo) -#define sg_dma64_address(s) sg_dma_address(s) -#define sg_dma64_len(s) sg_dma_len(s) -#if BITS_PER_LONG > 32 -#define PCI64_DMA_BITS 64 -#else -#define PCI64_DMA_BITS 32 -#endif /* BITS_PER_LONG */ -#endif +#define pci64_dma_build(hi,lo) \ + ((dma64_addr_t)(((u64)(lo))|(((u64)(hi))<<32))) #include "qlogicfc.h" @@ -245,13 +227,8 @@ }; /* entry header type commands */ -#if PCI64_DMA_BITS > 32 #define ENTRY_COMMAND 0x19 #define ENTRY_CONTINUATION 0x0a -#else -#define ENTRY_COMMAND 0x11 -#define ENTRY_CONTINUATION 0x02 -#endif #define ENTRY_STATUS 0x03 #define ENTRY_MARKER 0x04 @@ -262,23 +239,12 @@ #define EFLAG_BAD_HEADER 4 #define EFLAG_BAD_PAYLOAD 8 -#if PCI64_DMA_BITS > 32 - struct dataseg { u_int d_base; u_int d_base_hi; u_int d_count; }; -#else - -struct dataseg { - u_int d_base; - u_int d_count; -}; - -#endif - struct Command_Entry { struct Entry_header hdr; u_int handle; @@ -303,18 +269,10 @@ #define CFLAG_READ 0x20 #define CFLAG_WRITE 0x40 -#if PCI64_DMA_BITS > 32 struct Continuation_Entry { struct Entry_header hdr; struct dataseg dataseg[DATASEGS_PER_CONT]; }; -#else -struct Continuation_Entry { - struct Entry_header hdr; - u32 rsvd; - struct dataseg dataseg[DATASEGS_PER_CONT]; -}; -#endif struct Marker_Entry { struct Entry_header hdr; @@ -746,7 +704,7 @@ tmpt->proc_name = "isp2x00"; if (pci_present() == 0) { - printk("qlogicfc : PCI not present\n"); + printk(KERN_INFO "qlogicfc : PCI not present\n"); return 0; } @@ -756,6 +714,13 @@ if (pci_enable_device(pdev)) continue; + /* Try to configure DMA attributes. */ + pci_set_dma_mask(pdev, (u64)0xffffffffffffffff); + if (!pci_dac_cycles_ok(pdev)) { + if (!pci_dma_supported(pdev, (u64) 0xffffffff)) + continue; + } + host = scsi_register(tmpt, sizeof(struct isp2x00_hostdata)); if (!host) { printk("qlogicfc%d : could not register host.\n", hosts); @@ -1284,10 +1249,8 @@ for (i = 0; i < n; i++) { ds[i].d_base = cpu_to_le32(pci64_dma_lo32(sg_dma64_address(sg))); -#if PCI64_DMA_BITS > 32 ds[i].d_base_hi = cpu_to_le32(pci64_dma_hi32(sg_dma64_address(sg))); -#endif - ds[i].d_count = cpu_to_le32(sg_dma64_len(sg)); + ds[i].d_count = cpu_to_le32(sg_dma_len(sg)); ++sg; } sg_count -= DATASEGS_PER_COMMAND; @@ -1310,30 +1273,29 @@ n = DATASEGS_PER_CONT; for (i = 0; i < n; ++i) { ds[i].d_base = cpu_to_le32(pci64_dma_lo32(sg_dma64_address(sg))); -#if PCI64_DMA_BITS > 32 ds[i].d_base_hi = cpu_to_le32(pci64_dma_hi32(sg_dma64_address(sg))); -#endif - ds[i].d_count = cpu_to_le32(sg_dma64_len(sg)); + ds[i].d_count = cpu_to_le32(sg_dma_len(sg)); ++sg; } sg_count -= n; } } else if (Cmnd->request_bufflen && Cmnd->sc_data_direction != PCI_DMA_NONE) { - dma64_addr_t busaddr = pci64_map_single(hostdata->pci_dev, Cmnd->request_buffer, Cmnd->request_bufflen, - scsi_to_pci_dma_dir(Cmnd->sc_data_direction)); - + struct page *page = virt_to_page(Cmnd->request_buffer); + unsigned long offset = ((unsigned long)Cmnd->request_buffer & + ~PAGE_MASK); + dma64_addr_t busaddr = pci64_map_page(hostdata->pci_dev, + page, offset, + Cmnd->request_bufflen, + scsi_to_pci_dma_dir(Cmnd->sc_data_direction)); *(dma64_addr_t *)&Cmnd->SCp = busaddr; + cmd->dataseg[0].d_base = cpu_to_le32(pci64_dma_lo32(busaddr)); -#if PCI64_DMA_BITS > 32 cmd->dataseg[0].d_base_hi = cpu_to_le32(pci64_dma_hi32(busaddr)); -#endif cmd->dataseg[0].d_count = cpu_to_le32(Cmnd->request_bufflen); cmd->segment_cnt = cpu_to_le16(1); } else { cmd->dataseg[0].d_base = 0; -#if PCI64_DMA_BITS > 32 cmd->dataseg[0].d_base_hi = 0; -#endif cmd->segment_cnt = cpu_to_le16(1); /* Shouldn't this be 0? */ } @@ -1431,11 +1393,12 @@ Cmnd->use_sg, scsi_to_pci_dma_dir(Cmnd->sc_data_direction)); else if (Cmnd->request_bufflen && - Cmnd->sc_data_direction != PCI_DMA_NONE) - pci64_unmap_single(hostdata->pci_dev, - *(dma64_addr_t *)&Cmnd->SCp, - Cmnd->request_bufflen, - scsi_to_pci_dma_dir(Cmnd->sc_data_direction)); + Cmnd->sc_data_direction != PCI_DMA_NONE) { + pci64_unmap_page(hostdata->pci_dev, + *(dma64_addr_t *)&Cmnd->SCp, + Cmnd->request_bufflen, + scsi_to_pci_dma_dir(Cmnd->sc_data_direction)); + } hostdata->handle_ptrs[i]->result = DID_SOFT_ERROR << 16; @@ -1538,10 +1501,10 @@ scsi_to_pci_dma_dir(Cmnd->sc_data_direction)); else if (Cmnd->request_bufflen && Cmnd->sc_data_direction != PCI_DMA_NONE) - pci64_unmap_single(hostdata->pci_dev, - *(dma64_addr_t *)&Cmnd->SCp, - Cmnd->request_bufflen, - scsi_to_pci_dma_dir(Cmnd->sc_data_direction)); + pci64_unmap_page(hostdata->pci_dev, + *(dma64_addr_t *)&Cmnd->SCp, + Cmnd->request_bufflen, + scsi_to_pci_dma_dir(Cmnd->sc_data_direction)); Cmnd->result = 0x0; (*Cmnd->scsi_done) (Cmnd); } else @@ -1591,9 +1554,10 @@ (struct scatterlist *)Cmnd->buffer, Cmnd->use_sg, scsi_to_pci_dma_dir(Cmnd->sc_data_direction)); else if (Cmnd->request_bufflen && Cmnd->sc_data_direction != PCI_DMA_NONE) - pci64_unmap_single(hostdata->pci_dev, *(dma64_addr_t *)&Cmnd->SCp, - Cmnd->request_bufflen, - scsi_to_pci_dma_dir(Cmnd->sc_data_direction)); + pci64_unmap_page(hostdata->pci_dev, + *(dma64_addr_t *)&Cmnd->SCp, + Cmnd->request_bufflen, + scsi_to_pci_dma_dir(Cmnd->sc_data_direction)); /* * if any of the following are true we do not @@ -1964,9 +1928,15 @@ hostdata->wwn |= (u64) (cpu_to_le16(hostdata->control_block.node_name[3]) & 0x00ff) << 8; hostdata->wwn |= (u64) (cpu_to_le16(hostdata->control_block.node_name[3]) & 0xff00) >> 8; - /* FIXME: If the DMA transfer goes one way only, this should use PCI_DMA_TODEVICE and below as well. */ - busaddr = pci64_map_single(hostdata->pci_dev, &hostdata->control_block, sizeof(hostdata->control_block), - PCI_DMA_BIDIRECTIONAL); + /* FIXME: If the DMA transfer goes one way only, this should use + * PCI_DMA_TODEVICE and below as well. + */ + busaddr = pci64_map_page(hostdata->pci_dev, + virt_to_page(&hostdata->control_block), + ((unsigned long) &hostdata->control_block & + ~PAGE_MASK), + sizeof(hostdata->control_block), + PCI_DMA_BIDIRECTIONAL); param[0] = MBOX_INIT_FIRMWARE; param[2] = (u_short) (pci64_dma_lo32(busaddr) >> 16); @@ -1978,21 +1948,24 @@ isp2x00_mbox_command(host, param); if (param[0] != MBOX_COMMAND_COMPLETE) { printk("qlogicfc%d.c: Ouch 0x%04x\n", hostdata->host_id, param[0]); - pci64_unmap_single(hostdata->pci_dev, busaddr, sizeof(hostdata->control_block), - PCI_DMA_BIDIRECTIONAL); + pci64_unmap_page(hostdata->pci_dev, busaddr, + sizeof(hostdata->control_block), + PCI_DMA_BIDIRECTIONAL); return 1; } param[0] = MBOX_GET_FIRMWARE_STATE; isp2x00_mbox_command(host, param); if (param[0] != MBOX_COMMAND_COMPLETE) { printk("qlogicfc%d.c: 0x%04x\n", hostdata->host_id, param[0]); - pci64_unmap_single(hostdata->pci_dev, busaddr, sizeof(hostdata->control_block), - PCI_DMA_BIDIRECTIONAL); + pci64_unmap_page(hostdata->pci_dev, busaddr, + sizeof(hostdata->control_block), + PCI_DMA_BIDIRECTIONAL); return 1; } - pci64_unmap_single(hostdata->pci_dev, busaddr, sizeof(hostdata->control_block), - PCI_DMA_BIDIRECTIONAL); + pci64_unmap_page(hostdata->pci_dev, busaddr, + sizeof(hostdata->control_block), + PCI_DMA_BIDIRECTIONAL); LEAVE("isp2x00_reset_hardware"); return 0; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/scsi/qlogicfc.h linux/drivers/scsi/qlogicfc.h --- /opt/kernel/linux-2.4.9/drivers/scsi/qlogicfc.h Mon Jun 26 21:02:16 2000 +++ linux/drivers/scsi/qlogicfc.h Tue Aug 21 11:11:56 2001 @@ -100,7 +100,8 @@ cmd_per_lun: QLOGICFC_CMD_PER_LUN, \ present: 0, \ unchecked_isa_dma: 0, \ - use_clustering: ENABLE_CLUSTERING \ + use_clustering: ENABLE_CLUSTERING, \ + can_dma_32: 1 \ } #endif /* _QLOGICFC_H */ diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/scsi/scsi.c linux/drivers/scsi/scsi.c --- /opt/kernel/linux-2.4.9/drivers/scsi/scsi.c Fri Jul 20 06:07:04 2001 +++ linux/drivers/scsi/scsi.c Tue Aug 21 11:11:56 2001 @@ -178,10 +178,13 @@ * handler in the list - ultimately they call scsi_request_fn * to do the dirty deed. */ -void scsi_initialize_queue(Scsi_Device * SDpnt, struct Scsi_Host * SHpnt) { - blk_init_queue(&SDpnt->request_queue, scsi_request_fn); - blk_queue_headactive(&SDpnt->request_queue, 0); - SDpnt->request_queue.queuedata = (void *) SDpnt; +void scsi_initialize_queue(Scsi_Device * SDpnt, struct Scsi_Host * SHpnt) +{ + request_queue_t *q = &SDpnt->request_queue; + + blk_init_queue(q, scsi_request_fn); + blk_queue_headactive(q, 0); + q->queuedata = (void *) SDpnt; } #ifdef MODULE diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/scsi/scsi.h linux/drivers/scsi/scsi.h --- /opt/kernel/linux-2.4.9/drivers/scsi/scsi.h Wed Aug 15 23:23:11 2001 +++ linux/drivers/scsi/scsi.h Tue Aug 21 12:19:26 2001 @@ -391,7 +391,7 @@ #define CONTIGUOUS_BUFFERS(X,Y) \ (virt_to_phys((X)->b_data+(X)->b_size-1)+1==virt_to_phys((Y)->b_data)) #else -#define CONTIGUOUS_BUFFERS(X,Y) ((X->b_data+X->b_size) == Y->b_data) +#define CONTIGUOUS_BUFFERS(X,Y) (bh_bus((X)) + (X)->b_size == bh_bus((Y))) #endif @@ -745,7 +745,8 @@ unsigned request_bufflen; /* Actual request size */ struct timer_list eh_timeout; /* Used to time out the command. */ - void *request_buffer; /* Actual requested buffer */ + void *request_buffer; /* Actual requested buffer */ + void **bounce_buffers; /* Array of bounce buffers when using scatter-gather */ /* These elements define the operation we ultimately want to perform */ unsigned char data_cmnd[MAX_COMMAND_SIZE]; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/scsi/scsi_debug.c linux/drivers/scsi/scsi_debug.c --- /opt/kernel/linux-2.4.9/drivers/scsi/scsi_debug.c Sun Nov 12 04:01:11 2000 +++ linux/drivers/scsi/scsi_debug.c Tue Aug 21 11:03:52 2001 @@ -154,10 +154,7 @@ if (SCpnt->use_sg) { sgpnt = (struct scatterlist *) SCpnt->buffer; for (i = 0; i < SCpnt->use_sg; i++) { - lpnt = (int *) sgpnt[i].alt_address; - printk(":%p %p %d\n", sgpnt[i].alt_address, sgpnt[i].address, sgpnt[i].length); - if (lpnt) - printk(" (Alt %x) ", lpnt[15]); + printk(":%p %d\n", sgpnt[i].address, sgpnt[i].length); }; } else { printk("nosg: %p %p %d\n", SCpnt->request.buffer, SCpnt->buffer, @@ -175,12 +172,6 @@ printk("\n"); if (flag == 0) return; - lpnt = (unsigned int *) sgpnt[0].alt_address; - for (i = 0; i < sizeof(Scsi_Cmnd) / 4 + 1; i++) { - if ((i & 7) == 0) - printk("\n"); - printk("%x ", *lpnt++); - }; #if 0 printk("\n"); lpnt = (unsigned int *) sgpnt[0].address; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/scsi/scsi_lib.c linux/drivers/scsi/scsi_lib.c --- /opt/kernel/linux-2.4.9/drivers/scsi/scsi_lib.c Sun Aug 12 19:51:42 2001 +++ linux/drivers/scsi/scsi_lib.c Tue Aug 21 11:11:56 2001 @@ -388,6 +388,7 @@ req->nr_sectors -= nsect; req->current_nr_sectors = bh->b_size >> 9; + req->hard_cur_sectors = req->current_nr_sectors; if (req->nr_sectors < req->current_nr_sectors) { req->nr_sectors = req->current_nr_sectors; printk("scsi_end_request: buffer-list destroyed\n"); @@ -410,7 +411,6 @@ q = &SCpnt->device->request_queue; - req->buffer = bh->b_data; /* * Bleah. Leftovers again. Stick the leftovers in * the front of the queue, and goose the queue again. @@ -489,6 +489,8 @@ */ static void scsi_release_buffers(Scsi_Cmnd * SCpnt) { + struct request *req = &SCpnt->request; + ASSERT_LOCK(&io_request_lock, 0); /* @@ -496,20 +498,22 @@ */ if (SCpnt->use_sg) { struct scatterlist *sgpnt; + void **bbpnt; int i; sgpnt = (struct scatterlist *) SCpnt->request_buffer; + bbpnt = SCpnt->bounce_buffers; - for (i = 0; i < SCpnt->use_sg; i++) { - if (sgpnt[i].alt_address) { - scsi_free(sgpnt[i].address, sgpnt[i].length); + if (bbpnt) { + for (i = 0; i < SCpnt->use_sg; i++) { + if (bbpnt[i]) + scsi_free(sgpnt[i].address, sgpnt[i].length); } } scsi_free(SCpnt->request_buffer, SCpnt->sglist_len); } else { - if (SCpnt->request_buffer != SCpnt->request.buffer) { - scsi_free(SCpnt->request_buffer, SCpnt->request_bufflen); - } + if (SCpnt->request_buffer != req->buffer) + scsi_free(SCpnt->request_buffer,SCpnt->request_bufflen); } /* @@ -545,6 +549,7 @@ int result = SCpnt->result; int this_count = SCpnt->bufflen >> 9; request_queue_t *q = &SCpnt->device->request_queue; + struct request *req = &SCpnt->request; /* * We must do one of several things here: @@ -568,26 +573,32 @@ */ if (SCpnt->use_sg) { struct scatterlist *sgpnt; + void **bbpnt; int i; sgpnt = (struct scatterlist *) SCpnt->buffer; + bbpnt = SCpnt->bounce_buffers; - for (i = 0; i < SCpnt->use_sg; i++) { - if (sgpnt[i].alt_address) { - if (SCpnt->request.cmd == READ) { - memcpy(sgpnt[i].alt_address, - sgpnt[i].address, - sgpnt[i].length); + if (bbpnt) { + for (i = 0; i < SCpnt->use_sg; i++) { + if (bbpnt[i]) { + if (req->cmd == READ) { + memcpy(bbpnt[i], + sgpnt[i].address, + sgpnt[i].length); + } + scsi_free(sgpnt[i].address, sgpnt[i].length); } - scsi_free(sgpnt[i].address, sgpnt[i].length); } } scsi_free(SCpnt->buffer, SCpnt->sglist_len); } else { - if (SCpnt->buffer != SCpnt->request.buffer) { - if (SCpnt->request.cmd == READ) { - memcpy(SCpnt->request.buffer, SCpnt->buffer, - SCpnt->bufflen); + if (SCpnt->buffer != req->buffer) { + if (req->cmd == READ) { + unsigned long flags; + char *to = bh_kmap_irq(req->bh, &flags); + memcpy(to, SCpnt->buffer, SCpnt->bufflen); + bh_kunmap_irq(to, &flags); } scsi_free(SCpnt->buffer, SCpnt->bufflen); } diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/scsi/scsi_merge.c linux/drivers/scsi/scsi_merge.c --- /opt/kernel/linux-2.4.9/drivers/scsi/scsi_merge.c Thu Jul 5 20:28:17 2001 +++ linux/drivers/scsi/scsi_merge.c Tue Aug 21 12:54:33 2001 @@ -6,6 +6,7 @@ * Based upon conversations with large numbers * of people at Linux Expo. * Support for dynamic DMA mapping: Jakub Jelinek (jakub@redhat.com). + * Support for highmem I/O: Jens Axboe */ /* @@ -95,7 +96,7 @@ printk("Segment 0x%p, blocks %d, addr 0x%lx\n", bh, bh->b_size >> 9, - virt_to_phys(bh->b_data - 1)); + bh_bus(bh) - 1); } panic("Ththththaats all folks. Too dangerous to continue.\n"); } @@ -120,9 +121,11 @@ { int jj; struct scatterlist *sgpnt; + void **bbpnt; int consumed = 0; sgpnt = (struct scatterlist *) SCpnt->request_buffer; + bbpnt = SCpnt->bounce_buffers; /* * Now print out a bunch of stats. First, start with the request @@ -136,15 +139,13 @@ */ for(jj=0; jj < SCpnt->use_sg; jj++) { - printk("[%d]\tlen:%d\taddr:%p\talt:%p\n", + printk("[%d]\tlen:%d\taddr:%p\tbounce:%p\n", jj, sgpnt[jj].length, sgpnt[jj].address, - sgpnt[jj].alt_address); - if( sgpnt[jj].alt_address != NULL ) - { - consumed = (sgpnt[jj].length >> 9); - } + (bbpnt ? bbpnt[jj] : NULL)); + if (bbpnt && bbpnt[jj]) + consumed += sgpnt[jj].length; } printk("Total %d sectors consumed\n", consumed); panic("DMA pool exhausted"); @@ -223,8 +224,7 @@ * DMA capable host, make sure that a segment doesn't span * the DMA threshold boundary. */ - if (dma_host && - virt_to_phys(bhnext->b_data) - 1 == ISA_DMA_THRESHOLD) { + if (dma_host && bh_bus(bhnext) - 1 == ISA_DMA_THRESHOLD) { ret++; reqsize = bhnext->b_size; } else if (CONTIGUOUS_BUFFERS(bh, bhnext)) { @@ -241,8 +241,7 @@ * kind of screwed and we need to start * another segment. */ - if( dma_host - && virt_to_phys(bh->b_data) - 1 >= ISA_DMA_THRESHOLD + if( dma_host && bh_bus(bh) - 1 >= ISA_DMA_THRESHOLD && reqsize + bhnext->b_size > PAGE_SIZE ) { ret++; @@ -304,7 +303,7 @@ } #define MERGEABLE_BUFFERS(X,Y) \ -(((((long)(X)->b_data+(X)->b_size)|((long)(Y)->b_data)) & \ +(((((long)bh_bus((X))+(X)->b_size)|((long)bh_bus((Y)))) & \ (DMA_CHUNK_SIZE - 1)) == 0) #ifdef DMA_CHUNK_SIZE @@ -419,6 +418,8 @@ if ((req->nr_sectors + (bh->b_size >> 9)) > SHpnt->max_sectors) return 0; + else if (!blk_seg_merge_ok(q, req->bhtail, bh)) + return 0; if (use_clustering) { /* @@ -427,14 +428,11 @@ * DMA capable host, make sure that a segment doesn't span * the DMA threshold boundary. */ - if (dma_host && - virt_to_phys(req->bhtail->b_data) - 1 == ISA_DMA_THRESHOLD) { + if (dma_host && bh_bus(req->bhtail) - 1 == ISA_DMA_THRESHOLD) goto new_end_segment; - } if (CONTIGUOUS_BUFFERS(req->bhtail, bh)) { #ifdef DMA_SEGMENT_SIZE_LIMITED - if( dma_host - && virt_to_phys(bh->b_data) - 1 >= ISA_DMA_THRESHOLD ) { + if (dma_host && bh_bus(bh) - 1 >= ISA_DMA_THRESHOLD) { segment_size = 0; count = __count_segments(req, use_clustering, dma_host, &segment_size); if( segment_size + bh->b_size > PAGE_SIZE ) { @@ -478,6 +476,8 @@ if ((req->nr_sectors + (bh->b_size >> 9)) > SHpnt->max_sectors) return 0; + else if (!blk_seg_merge_ok(q, bh, req->bh)) + return 0; if (use_clustering) { /* @@ -486,14 +486,12 @@ * DMA capable host, make sure that a segment doesn't span * the DMA threshold boundary. */ - if (dma_host && - virt_to_phys(bh->b_data) - 1 == ISA_DMA_THRESHOLD) { + if (dma_host && bh_bus(bh) - 1 == ISA_DMA_THRESHOLD) { goto new_start_segment; } if (CONTIGUOUS_BUFFERS(bh, req->bh)) { #ifdef DMA_SEGMENT_SIZE_LIMITED - if( dma_host - && virt_to_phys(bh->b_data) - 1 >= ISA_DMA_THRESHOLD ) { + if (dma_host && bh_bus(bh) - 1 >= ISA_DMA_THRESHOLD) { segment_size = bh->b_size; count = __count_segments(req, use_clustering, dma_host, &segment_size); if( count != req->nr_segments ) { @@ -613,6 +611,9 @@ SDpnt = (Scsi_Device *) q->queuedata; SHpnt = SDpnt->host; + if (!blk_seg_merge_ok(q, req->bhtail, next->bh)) + return 0; + #ifdef DMA_CHUNK_SIZE if (max_segments > 64) max_segments = 64; @@ -652,10 +653,8 @@ * DMA capable host, make sure that a segment doesn't span * the DMA threshold boundary. */ - if (dma_host && - virt_to_phys(req->bhtail->b_data) - 1 == ISA_DMA_THRESHOLD) { + if (dma_host && bh_bus(req->bhtail) - 1 == ISA_DMA_THRESHOLD) goto dont_combine; - } #ifdef DMA_SEGMENT_SIZE_LIMITED /* * We currently can only allocate scatter-gather bounce @@ -663,7 +662,7 @@ */ if (dma_host && CONTIGUOUS_BUFFERS(req->bhtail, next->bh) - && virt_to_phys(req->bhtail->b_data) - 1 >= ISA_DMA_THRESHOLD ) + && bh_bus(req->bhtail) - 1 >= ISA_DMA_THRESHOLD ) { int segment_size = 0; int count = 0; @@ -807,30 +806,8 @@ int sectors; struct scatterlist * sgpnt; int this_count; + void ** bbpnt; - /* - * FIXME(eric) - don't inline this - it doesn't depend on the - * integer flags. Come to think of it, I don't think this is even - * needed any more. Need to play with it and see if we hit the - * panic. If not, then don't bother. - */ - if (!SCpnt->request.bh) { - /* - * Case of page request (i.e. raw device), or unlinked buffer - * Typically used for swapping, but this isn't how we do - * swapping any more. - */ - panic("I believe this is dead code. If we hit this, I was wrong"); -#if 0 - SCpnt->request_bufflen = SCpnt->request.nr_sectors << 9; - SCpnt->request_buffer = SCpnt->request.buffer; - SCpnt->use_sg = 0; - /* - * FIXME(eric) - need to handle DMA here. - */ -#endif - return 1; - } req = &SCpnt->request; /* * First we need to know how many scatter gather segments are needed. @@ -847,24 +824,26 @@ * buffer. */ if (dma_host && scsi_dma_free_sectors <= 10) { - this_count = SCpnt->request.current_nr_sectors; + this_count = req->current_nr_sectors; goto single_segment; } - /* - * Don't bother with scatter-gather if there is only one segment. - */ - if (count == 1) { - this_count = SCpnt->request.nr_sectors; - goto single_segment; - } - SCpnt->use_sg = count; /* * Allocate the actual scatter-gather table itself. - * scsi_malloc can only allocate in chunks of 512 bytes */ - SCpnt->sglist_len = (SCpnt->use_sg - * sizeof(struct scatterlist) + 511) & ~511; + SCpnt->use_sg = count; + SCpnt->sglist_len = (SCpnt->use_sg * sizeof(struct scatterlist)); + + /* If we could potentially require ISA bounce buffers, allocate + * space for this array here. + */ + if (dma_host) + SCpnt->sglist_len += (SCpnt->use_sg * sizeof(void *)); + + /* scsi_malloc can only allocate in chunks of 512 bytes so + * round it up. + */ + SCpnt->sglist_len = (SCpnt->sglist_len + 511) & ~511; sgpnt = (struct scatterlist *) scsi_malloc(SCpnt->sglist_len); @@ -877,7 +856,7 @@ * simply write the first buffer all by itself. */ printk("Warning - running *really* short on DMA buffers\n"); - this_count = SCpnt->request.current_nr_sectors; + this_count = req->current_nr_sectors; goto single_segment; } /* @@ -889,11 +868,17 @@ SCpnt->request_bufflen = 0; bhprev = NULL; - for (count = 0, bh = SCpnt->request.bh; - bh; bh = bh->b_reqnext) { + if (dma_host) + bbpnt = (void **) ((char *)sgpnt + + (SCpnt->use_sg * sizeof(struct scatterlist))); + else + bbpnt = NULL; + + SCpnt->bounce_buffers = bbpnt; + + for (count = 0, bh = req->bh; bh; bh = bh->b_reqnext) { if (use_clustering && bhprev != NULL) { - if (dma_host && - virt_to_phys(bhprev->b_data) - 1 == ISA_DMA_THRESHOLD) { + if (dma_host && bh_bus(bhprev) - 1 == ISA_DMA_THRESHOLD) { /* Nothing - fall through */ } else if (CONTIGUOUS_BUFFERS(bhprev, bh)) { /* @@ -904,7 +889,7 @@ */ if( dma_host ) { #ifdef DMA_SEGMENT_SIZE_LIMITED - if( virt_to_phys(bh->b_data) - 1 < ISA_DMA_THRESHOLD + if (bh_bus(bh) - 1 < ISA_DMA_THRESHOLD || sgpnt[count - 1].length + bh->b_size <= PAGE_SIZE ) { sgpnt[count - 1].length += bh->b_size; bhprev = bh; @@ -923,12 +908,15 @@ } } } - count++; - sgpnt[count - 1].address = bh->b_data; - sgpnt[count - 1].length += bh->b_size; - if (!dma_host) { + + sgpnt[count].page = bh->b_page; + sgpnt[count].offset = bh_offset(bh); + sgpnt[count].length = bh->b_size; + + if (!dma_host) SCpnt->request_bufflen += bh->b_size; - } + + count++; bhprev = bh; } @@ -951,12 +939,16 @@ for (i = 0; i < count; i++) { sectors = (sgpnt[i].length >> 9); SCpnt->request_bufflen += sgpnt[i].length; + /* + * only done for dma_host, in which case .page is not + * set since it's guarenteed to be a low memory page + */ if (virt_to_phys(sgpnt[i].address) + sgpnt[i].length - 1 > ISA_DMA_THRESHOLD) { if( scsi_dma_free_sectors - sectors <= 10 ) { /* * If this would nearly drain the DMA - * pool, mpty, then let's stop here. + * pool empty, then let's stop here. * Don't make this request any larger. * This is kind of a safety valve that * we use - we could get screwed later @@ -970,7 +962,7 @@ break; } - sgpnt[i].alt_address = sgpnt[i].address; + bbpnt[i] = sgpnt[i].address; sgpnt[i].address = (char *) scsi_malloc(sgpnt[i].length); /* @@ -986,8 +978,8 @@ } break; } - if (SCpnt->request.cmd == WRITE) { - memcpy(sgpnt[i].address, sgpnt[i].alt_address, + if (req->cmd == WRITE) { + memcpy(sgpnt[i].address, bbpnt[i], sgpnt[i].length); } } @@ -1031,8 +1023,7 @@ * single-block requests if we had hundreds of free sectors. */ if( scsi_dma_free_sectors > 30 ) { - for (this_count = 0, bh = SCpnt->request.bh; - bh; bh = bh->b_reqnext) { + for (this_count = 0, bh = req->bh; bh; bh = bh->b_reqnext) { if( scsi_dma_free_sectors - this_count < 30 || this_count == sectors ) { @@ -1045,7 +1036,7 @@ /* * Yow! Take the absolute minimum here. */ - this_count = SCpnt->request.current_nr_sectors; + this_count = req->current_nr_sectors; } /* @@ -1058,28 +1049,31 @@ * segment. Possibly the entire request, or possibly a small * chunk of the entire request. */ - bh = SCpnt->request.bh; - buff = SCpnt->request.buffer; + bh = req->bh; + buff = req->buffer = bh->b_data; - if (dma_host) { + if (dma_host || PageHighMem(bh->b_page)) { /* * Allocate a DMA bounce buffer. If the allocation fails, fall * back and allocate a really small one - enough to satisfy * the first buffer. */ - if (virt_to_phys(SCpnt->request.bh->b_data) - + (this_count << 9) - 1 > ISA_DMA_THRESHOLD) { + if (bh_bus(bh) + (this_count << 9) - 1 > ISA_DMA_THRESHOLD) { buff = (char *) scsi_malloc(this_count << 9); if (!buff) { printk("Warning - running low on DMA memory\n"); - this_count = SCpnt->request.current_nr_sectors; + this_count = req->current_nr_sectors; buff = (char *) scsi_malloc(this_count << 9); if (!buff) { dma_exhausted(SCpnt, 0); } } - if (SCpnt->request.cmd == WRITE) - memcpy(buff, (char *) SCpnt->request.buffer, this_count << 9); + if (req->cmd == WRITE) { + unsigned long flags; + char *buf = bh_kmap_irq(bh, &flags); + memcpy(buff, buf, this_count << 9); + bh_kunmap_irq(buf, &flags); + } } } SCpnt->request_bufflen = this_count << 9; @@ -1127,14 +1121,6 @@ q = &SDpnt->request_queue; /* - * If the host has already selected a merge manager, then don't - * pick a new one. - */ -#if 0 - if (q->back_merge_fn && q->front_merge_fn) - return; -#endif - /* * If this host has an unlimited tablesize, then don't bother with a * merge manager. The whole point of the operation is to make sure * that requests don't grow too large, and this host isn't picky. @@ -1166,4 +1152,14 @@ q->merge_requests_fn = scsi_merge_requests_fn_dc; SDpnt->scsi_init_io_fn = scsi_init_io_vdc; } + + /* + * now enable highmem I/O, if appropriate + */ +#ifdef CONFIG_HIGHMEM + if (SHpnt->can_dma_32 && (SDpnt->type == TYPE_DISK)) + blk_queue_bounce_limit(q, SHpnt->pci_dev->dma_mask); + else + blk_queue_bounce_limit(q, BLK_BOUNCE_HIGH); +#endif } diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/scsi/sr.c linux/drivers/scsi/sr.c --- /opt/kernel/linux-2.4.9/drivers/scsi/sr.c Thu Jul 5 20:28:17 2001 +++ linux/drivers/scsi/sr.c Tue Aug 21 11:03:52 2001 @@ -265,6 +265,7 @@ struct scatterlist *sg, *old_sg = NULL; int i, fsize, bsize, sg_ent, sg_count; char *front, *back; + void **bbpnt, **old_bbpnt = NULL; back = front = NULL; sg_ent = SCpnt->use_sg; @@ -292,17 +293,25 @@ * extend or allocate new scatter-gather table */ sg_count = SCpnt->use_sg; - if (sg_count) + if (sg_count) { old_sg = (struct scatterlist *) SCpnt->request_buffer; - else { + old_bbpnt = SCpnt->bounce_buffers; + } else { sg_count = 1; sg_ent++; } - i = ((sg_ent * sizeof(struct scatterlist)) + 511) & ~511; + /* Get space for scatterlist and bounce buffer array. */ + i = sg_ent * sizeof(struct scatterlist); + i += sg_ent * sizeof(void *); + i = (i + 511) & ~511; + if ((sg = scsi_malloc(i)) == NULL) goto no_mem; + bbpnt = (void **) + ((char *)sg + (sg_ent * sizeof(struct scatterlist))); + /* * no more failing memory allocs possible, we can safely assign * SCpnt values now @@ -313,13 +322,15 @@ i = 0; if (fsize) { - sg[0].address = sg[0].alt_address = front; + sg[0].address = bbpnt[0] = front; sg[0].length = fsize; i++; } if (old_sg) { memcpy(sg + i, old_sg, SCpnt->use_sg * sizeof(struct scatterlist)); - scsi_free(old_sg, ((SCpnt->use_sg * sizeof(struct scatterlist)) + 511) & ~511); + memcpy(bbpnt + i, old_bbpnt, SCpnt->use_sg * sizeof(void *)); + scsi_free(old_sg, (((SCpnt->use_sg * sizeof(struct scatterlist)) + + (SCpnt->use_sg * sizeof(void *))) + 511) & ~511); } else { sg[i].address = SCpnt->request_buffer; sg[i].length = SCpnt->request_bufflen; @@ -327,11 +338,12 @@ SCpnt->request_bufflen += (fsize + bsize); SCpnt->request_buffer = sg; + SCpnt->bounce_buffers = bbpnt; SCpnt->use_sg += i; if (bsize) { sg[SCpnt->use_sg].address = back; - sg[SCpnt->use_sg].alt_address = back; + bbpnt[SCpnt->use_sg] = back; sg[SCpnt->use_sg].length = bsize; SCpnt->use_sg++; } diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/scsi/st.c linux/drivers/scsi/st.c --- /opt/kernel/linux-2.4.9/drivers/scsi/st.c Sun Aug 12 20:21:47 2001 +++ linux/drivers/scsi/st.c Tue Aug 21 11:03:52 2001 @@ -3222,7 +3222,6 @@ tb->sg[0].address = (unsigned char *) __get_free_pages(priority, order); if (tb->sg[0].address != NULL) { - tb->sg[0].alt_address = NULL; tb->sg[0].length = b_size; break; } @@ -3258,7 +3257,6 @@ tb = NULL; break; } - tb->sg[segs].alt_address = NULL; tb->sg[segs].length = b_size; got += b_size; segs++; @@ -3332,7 +3330,6 @@ normalize_buffer(STbuffer); return FALSE; } - STbuffer->sg[segs].alt_address = NULL; STbuffer->sg[segs].length = b_size; STbuffer->sg_segs += 1; got += b_size; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/scsi/sym53c8xx.c linux/drivers/scsi/sym53c8xx.c --- /opt/kernel/linux-2.4.9/drivers/scsi/sym53c8xx.c Thu Jul 5 20:28:16 2001 +++ linux/drivers/scsi/sym53c8xx.c Tue Aug 21 11:03:52 2001 @@ -13101,7 +13101,7 @@ (int) (PciDeviceFn(pdev) & 7)); #ifdef SCSI_NCR_DYNAMIC_DMA_MAPPING - if (pci_set_dma_mask(pdev, (dma_addr_t) (0xffffffffUL))) { + if (pci_set_dma_mask(pdev, 0xffffffff)) { printk(KERN_WARNING NAME53C8XX "32 BIT PCI BUS DMA ADDRESSING NOT SUPPORTED\n"); return -1; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/scsi/sym53c8xx.h linux/drivers/scsi/sym53c8xx.h --- /opt/kernel/linux-2.4.9/drivers/scsi/sym53c8xx.h Wed Aug 15 23:23:24 2001 +++ linux/drivers/scsi/sym53c8xx.h Tue Aug 21 11:11:56 2001 @@ -96,7 +96,8 @@ this_id: 7, \ sg_tablesize: SCSI_NCR_SG_TABLESIZE, \ cmd_per_lun: SCSI_NCR_CMD_PER_LUN, \ - use_clustering: DISABLE_CLUSTERING} + use_clustering: DISABLE_CLUSTERING, \ + can_dma_32: 1} #else diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/drivers/scsi/sym53c8xx_comm.h linux/drivers/scsi/sym53c8xx_comm.h --- /opt/kernel/linux-2.4.9/drivers/scsi/sym53c8xx_comm.h Thu Jul 5 20:28:16 2001 +++ linux/drivers/scsi/sym53c8xx_comm.h Tue Aug 21 11:03:52 2001 @@ -2186,7 +2186,7 @@ (int) (PciDeviceFn(pdev) & 7)); #ifdef SCSI_NCR_DYNAMIC_DMA_MAPPING - if (!pci_dma_supported(pdev, (dma_addr_t) (0xffffffffUL))) { + if (!pci_dma_supported(pdev, 0xffffffff)) { printk(KERN_WARNING NAME53C8XX "32 BIT PCI BUS DMA ADDRESSING NOT SUPPORTED\n"); return -1; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/fs/buffer.c linux/fs/buffer.c --- /opt/kernel/linux-2.4.9/fs/buffer.c Wed Aug 15 23:25:49 2001 +++ linux/fs/buffer.c Tue Aug 21 11:11:56 2001 @@ -1331,13 +1331,11 @@ bh->b_page = page; if (offset >= PAGE_SIZE) BUG(); - if (PageHighMem(page)) - /* - * This catches illegal uses and preserves the offset: - */ - bh->b_data = (char *)(0 + offset); - else - bh->b_data = page_address(page) + offset; + /* + * ->virtual is NULL on highmem pages, so we can catch the + * offset even though using page_address on it + */ + bh->b_data = page_address(page) + offset; } /* diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-alpha/pci.h linux/include/asm-alpha/pci.h --- /opt/kernel/linux-2.4.9/include/asm-alpha/pci.h Mon May 21 22:38:41 2001 +++ linux/include/asm-alpha/pci.h Tue Aug 21 11:03:52 2001 @@ -144,7 +144,7 @@ only drive the low 24-bits during PCI bus mastering, then you would pass 0x00ffffff as the mask to this function. */ -extern int pci_dma_supported(struct pci_dev *hwdev, dma_addr_t mask); +extern int pci_dma_supported(struct pci_dev *hwdev, u64 mask); /* Return the index of the PCI controller for device PDEV. */ extern int pci_controller_num(struct pci_dev *pdev); diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-alpha/scatterlist.h linux/include/asm-alpha/scatterlist.h --- /opt/kernel/linux-2.4.9/include/asm-alpha/scatterlist.h Tue Feb 8 05:09:05 2000 +++ linux/include/asm-alpha/scatterlist.h Tue Aug 21 11:03:52 2001 @@ -5,8 +5,6 @@ struct scatterlist { char *address; /* Source/target vaddr. */ - char *alt_address; /* Location of actual if address is a - dma indirect buffer, else NULL. */ dma_addr_t dma_address; unsigned int length; unsigned int dma_length; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-arm/pci.h linux/include/asm-arm/pci.h --- /opt/kernel/linux-2.4.9/include/asm-arm/pci.h Sun Aug 12 20:14:00 2001 +++ linux/include/asm-arm/pci.h Tue Aug 21 11:03:52 2001 @@ -152,7 +152,7 @@ * only drive the low 24-bits during PCI bus mastering, then * you would pass 0x00ffffff as the mask to this function. */ -static inline int pci_dma_supported(struct pci_dev *hwdev, dma_addr_t mask) +static inline int pci_dma_supported(struct pci_dev *hwdev, u64 mask) { return 1; } diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-arm/scatterlist.h linux/include/asm-arm/scatterlist.h --- /opt/kernel/linux-2.4.9/include/asm-arm/scatterlist.h Sun Sep 3 20:19:11 2000 +++ linux/include/asm-arm/scatterlist.h Tue Aug 21 11:03:52 2001 @@ -5,7 +5,6 @@ struct scatterlist { char *address; /* virtual address */ - char *alt_address; /* indirect dma address, or NULL */ dma_addr_t dma_address; /* dma address */ unsigned int length; /* length */ }; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-i386/kmap_types.h linux/include/asm-i386/kmap_types.h --- /opt/kernel/linux-2.4.9/include/asm-i386/kmap_types.h Thu Apr 12 21:11:39 2001 +++ linux/include/asm-i386/kmap_types.h Tue Aug 21 11:11:56 2001 @@ -6,6 +6,7 @@ KM_BOUNCE_WRITE, KM_SKB_DATA, KM_SKB_DATA_SOFTIRQ, + KM_BH_IRQ, KM_TYPE_NR }; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-i386/pci.h linux/include/asm-i386/pci.h --- /opt/kernel/linux-2.4.9/include/asm-i386/pci.h Wed Aug 15 23:21:44 2001 +++ linux/include/asm-i386/pci.h Tue Aug 21 11:55:11 2001 @@ -34,6 +34,15 @@ struct pci_dev; +/* The PCI address space does equal the physical memory + * address space. The networking and block device layers use + * this boolean for bounce buffer decisions. + */ +#define PCI_DMA_BUS_IS_PHYS (1) + +/* This is always fine. */ +#define pci_dac_cycles_ok(pci_dev) (1) + /* Allocate and map kernel buffer using consistent mode DMA for a device. * hwdev should be valid struct pci_dev pointer for PCI devices, * NULL for PCI-like buses (ISA, EISA). @@ -55,6 +64,22 @@ extern void pci_free_consistent(struct pci_dev *hwdev, size_t size, void *vaddr, dma_addr_t dma_handle); +static __inline__ void *pci64_alloc_consistent(struct pci_dev *hwdev, size_t size, + dma64_addr_t *dma_handle) +{ + dma_addr_t tmp; + void *ret; + + ret = pci_alloc_consistent(hwdev, size, &tmp); + if (ret != NULL) + *dma_handle = (dma64_addr_t) tmp; + + return ret; +} + +#define pci64_free_consistent(pdev,sz,addr,dma) \ + pci_free_consistent(pdev,sz,addr,(dma_addr_t)dma) + /* Map a single buffer of the indicated size for DMA in streaming mode. * The 32-bit bus address to use is returned. * @@ -84,6 +109,46 @@ /* Nothing to do */ } +/* + * pci_{map,unmap}_single_page maps a kernel page to a dma_addr_t. identical + * to pci_map_single, but takes a struct page instead of a virtual address + */ +static inline dma_addr_t pci_map_page(struct pci_dev *hwdev, struct page *page, + unsigned long offset, size_t size, int direction) +{ + if (direction == PCI_DMA_NONE) + BUG(); + + return (page - mem_map) * PAGE_SIZE + offset; +} + +static inline void pci_unmap_page(struct pci_dev *hwdev, dma_addr_t dma_address, + size_t size, int direction) +{ + if (direction == PCI_DMA_NONE) + BUG(); + /* Nothing to do */ +} + +/* 64-bit variants */ +static inline dma64_addr_t pci64_map_page(struct pci_dev *hwdev, struct page *page, + unsigned long offset, size_t size, int direction) +{ + if (direction == PCI_DMA_NONE) + BUG(); + + return (((dma64_addr_t) (page - mem_map)) * + ((dma64_addr_t) PAGE_SIZE)) + (dma64_addr_t) offset; +} + +static inline void pci64_unmap_page(struct pci_dev *hwdev, dma64_addr_t dma_address, + size_t size, int direction) +{ + if (direction == PCI_DMA_NONE) + BUG(); + /* Nothing to do */ +} + /* Map a set of buffers described by scatterlist in streaming * mode for DMA. This is the scather-gather version of the * above pci_map_single interface. Here the scatter gather list @@ -102,8 +167,26 @@ static inline int pci_map_sg(struct pci_dev *hwdev, struct scatterlist *sg, int nents, int direction) { + int i; + if (direction == PCI_DMA_NONE) BUG(); + + /* + * temporary 2.4 hack + */ + for (i = 0; i < nents; i++ ) { + if (sg[i].address && sg[i].page) + BUG(); + else if (!sg[i].address && !sg[i].page) + BUG(); + + if (sg[i].address) + sg[i].dma_address = virt_to_bus(sg[i].address); + else + sg[i].dma_address = page_to_bus(sg[i].page) + sg[i].offset; + } + return nents; } @@ -119,6 +202,9 @@ /* Nothing to do */ } +#define pci64_map_sg pci_map_sg +#define pci64_unmap_sg pci_unmap_sg + /* Make physical memory consistent for a single * streaming mode DMA translation after a transfer. * @@ -152,12 +238,15 @@ /* Nothing to do */ } +#define pci64_dma_sync_single pci_dma_sync_single +#define pci64_dma_sync_sg pci_dma_sync_sg + /* Return whether the given PCI device DMA address mask can * be supported properly. For example, if your device can * only drive the low 24-bits during PCI bus mastering, then * you would pass 0x00ffffff as the mask to this function. */ -static inline int pci_dma_supported(struct pci_dev *hwdev, dma_addr_t mask) +static inline int pci_dma_supported(struct pci_dev *hwdev, u64 mask) { /* * we fall back to GFP_DMA when the mask isn't all 1s, @@ -173,10 +262,10 @@ /* These macros should be used after a pci_map_sg call has been done * to get bus addresses of each of the SG entries and their lengths. * You should only work with the number of sg entries pci_map_sg - * returns, or alternatively stop on the first sg_dma_len(sg) which - * is 0. + * returns. */ -#define sg_dma_address(sg) (virt_to_bus((sg)->address)) +#define sg_dma_address(sg) ((dma_addr_t) ((sg)->dma_address)) +#define sg_dma64_address(sg) ((sg)->dma_address) #define sg_dma_len(sg) ((sg)->length) /* Return the index of the PCI controller for device. */ diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-i386/scatterlist.h linux/include/asm-i386/scatterlist.h --- /opt/kernel/linux-2.4.9/include/asm-i386/scatterlist.h Mon Dec 30 12:01:10 1996 +++ linux/include/asm-i386/scatterlist.h Tue Aug 21 11:03:52 2001 @@ -2,9 +2,12 @@ #define _I386_SCATTERLIST_H struct scatterlist { - char * address; /* Location data is to be transferred to */ - char * alt_address; /* Location of actual if address is a - * dma indirect buffer. NULL otherwise */ + char * address; /* Location data is to be transferred to, NULL for + * highmem page */ + struct page * page; /* Location for highmem page, if any */ + unsigned int offset;/* for highmem, page offset */ + + dma64_addr_t dma_address; unsigned int length; }; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-i386/types.h linux/include/asm-i386/types.h --- /opt/kernel/linux-2.4.9/include/asm-i386/types.h Thu Jan 27 17:58:15 2000 +++ linux/include/asm-i386/types.h Tue Aug 21 11:03:52 2001 @@ -27,6 +27,8 @@ */ #ifdef __KERNEL__ +#include + typedef signed char s8; typedef unsigned char u8; @@ -44,6 +46,11 @@ /* Dma addresses are 32-bits wide. */ typedef u32 dma_addr_t; +#ifdef CONFIG_HIGHMEM +typedef u64 dma64_addr_t; +#else +typedef u32 dma64_addr_t; +#endif #endif /* __KERNEL__ */ diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-ia64/pci.h linux/include/asm-ia64/pci.h --- /opt/kernel/linux-2.4.9/include/asm-ia64/pci.h Wed May 16 19:31:27 2001 +++ linux/include/asm-ia64/pci.h Tue Aug 21 11:03:52 2001 @@ -52,7 +52,7 @@ * you would pass 0x00ffffff as the mask to this function. */ static inline int -pci_dma_supported (struct pci_dev *hwdev, dma_addr_t mask) +pci_dma_supported (struct pci_dev *hwdev, u64 mask) { return 1; } diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-ia64/scatterlist.h linux/include/asm-ia64/scatterlist.h --- /opt/kernel/linux-2.4.9/include/asm-ia64/scatterlist.h Sat Aug 12 04:09:06 2000 +++ linux/include/asm-ia64/scatterlist.h Tue Aug 21 11:03:52 2001 @@ -8,11 +8,6 @@ struct scatterlist { char *address; /* location data is to be transferred to */ - /* - * Location of actual buffer if ADDRESS points to a DMA - * indirection buffer, NULL otherwise: - */ - char *alt_address; char *orig_address; /* Save away the original buffer address (used by pci-dma.c) */ unsigned int length; /* buffer length */ }; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-m68k/scatterlist.h linux/include/asm-m68k/scatterlist.h --- /opt/kernel/linux-2.4.9/include/asm-m68k/scatterlist.h Tue May 11 18:57:14 1999 +++ linux/include/asm-m68k/scatterlist.h Tue Aug 21 11:03:52 2001 @@ -3,8 +3,6 @@ struct scatterlist { char * address; /* Location data is to be transferred to */ - char * alt_address; /* Location of actual if address is a - * dma indirect buffer. NULL otherwise */ unsigned int length; unsigned long dvma_address; }; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-mips/pci.h linux/include/asm-mips/pci.h --- /opt/kernel/linux-2.4.9/include/asm-mips/pci.h Mon Jul 2 22:56:40 2001 +++ linux/include/asm-mips/pci.h Tue Aug 21 11:03:52 2001 @@ -206,7 +206,7 @@ * only drive the low 24-bits during PCI bus mastering, then * you would pass 0x00ffffff as the mask to this function. */ -extern inline int pci_dma_supported(struct pci_dev *hwdev, dma_addr_t mask) +extern inline int pci_dma_supported(struct pci_dev *hwdev, u64 mask) { /* * we fall back to GFP_DMA when the mask isn't all 1s, diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-mips/scatterlist.h linux/include/asm-mips/scatterlist.h --- /opt/kernel/linux-2.4.9/include/asm-mips/scatterlist.h Tue Dec 16 21:46:12 1997 +++ linux/include/asm-mips/scatterlist.h Tue Aug 21 11:03:52 2001 @@ -3,8 +3,6 @@ struct scatterlist { char * address; /* Location data is to be transferred to */ - char * alt_address; /* Location of actual if address is a - * dma indirect buffer. NULL otherwise */ unsigned int length; __u32 dvma_address; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-mips64/pci.h linux/include/asm-mips64/pci.h --- /opt/kernel/linux-2.4.9/include/asm-mips64/pci.h Wed Jul 4 20:50:39 2001 +++ linux/include/asm-mips64/pci.h Tue Aug 21 11:03:52 2001 @@ -195,7 +195,7 @@ #endif } -extern inline int pci_dma_supported(struct pci_dev *hwdev, dma_addr_t mask) +extern inline int pci_dma_supported(struct pci_dev *hwdev, u64 mask) { /* * we fall back to GFP_DMA when the mask isn't all 1s, diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-mips64/scatterlist.h linux/include/asm-mips64/scatterlist.h --- /opt/kernel/linux-2.4.9/include/asm-mips64/scatterlist.h Fri Feb 25 07:53:35 2000 +++ linux/include/asm-mips64/scatterlist.h Tue Aug 21 11:03:52 2001 @@ -3,8 +3,6 @@ struct scatterlist { char * address; /* Location data is to be transferred to */ - char * alt_address; /* Location of actual if address is a - * dma indirect buffer. NULL otherwise */ unsigned int length; __u32 dvma_address; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-parisc/pci.h linux/include/asm-parisc/pci.h --- /opt/kernel/linux-2.4.9/include/asm-parisc/pci.h Wed May 16 19:31:27 2001 +++ linux/include/asm-parisc/pci.h Tue Aug 21 11:03:52 2001 @@ -113,7 +113,7 @@ ** See Documentation/DMA-mapping.txt */ struct pci_dma_ops { - int (*dma_supported)(struct pci_dev *dev, dma_addr_t mask); + int (*dma_supported)(struct pci_dev *dev, u64 mask); void *(*alloc_consistent)(struct pci_dev *dev, size_t size, dma_addr_t *iova); void (*free_consistent)(struct pci_dev *dev, size_t size, void *vaddr, dma_addr_t iova); dma_addr_t (*map_single)(struct pci_dev *dev, void *addr, size_t size, int direction); diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-parisc/scatterlist.h linux/include/asm-parisc/scatterlist.h --- /opt/kernel/linux-2.4.9/include/asm-parisc/scatterlist.h Tue Dec 5 21:29:39 2000 +++ linux/include/asm-parisc/scatterlist.h Tue Aug 21 11:03:52 2001 @@ -3,8 +3,6 @@ struct scatterlist { char * address; /* Location data is to be transferred to */ - char * alt_address; /* Location of actual if address is a - * dma indirect buffer. NULL otherwise */ unsigned int length; /* an IOVA can be 64-bits on some PA-Risc platforms. */ diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-ppc/pci.h linux/include/asm-ppc/pci.h --- /opt/kernel/linux-2.4.9/include/asm-ppc/pci.h Tue May 22 00:02:06 2001 +++ linux/include/asm-ppc/pci.h Tue Aug 21 11:03:52 2001 @@ -108,7 +108,7 @@ * only drive the low 24-bits during PCI bus mastering, then * you would pass 0x00ffffff as the mask to this function. */ -extern inline int pci_dma_supported(struct pci_dev *hwdev, dma_addr_t mask) +extern inline int pci_dma_supported(struct pci_dev *hwdev, u64 mask) { return 1; } diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-ppc/scatterlist.h linux/include/asm-ppc/scatterlist.h --- /opt/kernel/linux-2.4.9/include/asm-ppc/scatterlist.h Tue May 22 00:02:06 2001 +++ linux/include/asm-ppc/scatterlist.h Tue Aug 21 11:03:52 2001 @@ -9,8 +9,6 @@ struct scatterlist { char * address; /* Location data is to be transferred to */ - char * alt_address; /* Location of actual if address is a - * dma indirect buffer. NULL otherwise */ unsigned int length; }; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-s390/scatterlist.h linux/include/asm-s390/scatterlist.h --- /opt/kernel/linux-2.4.9/include/asm-s390/scatterlist.h Tue Feb 13 23:13:44 2001 +++ linux/include/asm-s390/scatterlist.h Tue Aug 21 11:03:52 2001 @@ -3,8 +3,6 @@ struct scatterlist { char * address; /* Location data is to be transferred to */ - char * alt_address; /* Location of actual if address is a - * dma indirect buffer. NULL otherwise */ unsigned int length; }; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-s390x/scatterlist.h linux/include/asm-s390x/scatterlist.h --- /opt/kernel/linux-2.4.9/include/asm-s390x/scatterlist.h Tue Feb 13 23:13:44 2001 +++ linux/include/asm-s390x/scatterlist.h Tue Aug 21 11:03:52 2001 @@ -3,8 +3,6 @@ struct scatterlist { char * address; /* Location data is to be transferred to */ - char * alt_address; /* Location of actual if address is a - * dma indirect buffer. NULL otherwise */ unsigned int length; }; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-sh/pci.h linux/include/asm-sh/pci.h --- /opt/kernel/linux-2.4.9/include/asm-sh/pci.h Wed Jun 27 22:55:29 2001 +++ linux/include/asm-sh/pci.h Tue Aug 21 11:03:52 2001 @@ -167,7 +167,7 @@ * only drive the low 24-bits during PCI bus mastering, then * you would pass 0x00ffffff as the mask to this function. */ -extern inline int pci_dma_supported(struct pci_dev *hwdev, dma_addr_t mask) +extern inline int pci_dma_supported(struct pci_dev *hwdev, u64 mask) { return 1; } diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-sh/scatterlist.h linux/include/asm-sh/scatterlist.h --- /opt/kernel/linux-2.4.9/include/asm-sh/scatterlist.h Sun Mar 5 18:33:55 2000 +++ linux/include/asm-sh/scatterlist.h Tue Aug 21 11:03:52 2001 @@ -3,8 +3,6 @@ struct scatterlist { char * address; /* Location data is to be transferred to */ - char * alt_address; /* Location of actual if address is a - * dma indirect buffer. NULL otherwise */ unsigned int length; }; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-sparc/pci.h linux/include/asm-sparc/pci.h --- /opt/kernel/linux-2.4.9/include/asm-sparc/pci.h Wed May 16 19:31:27 2001 +++ linux/include/asm-sparc/pci.h Tue Aug 21 11:03:52 2001 @@ -108,7 +108,7 @@ * only drive the low 24-bits during PCI bus mastering, then * you would pass 0x00ffffff as the mask to this function. */ -extern inline int pci_dma_supported(struct pci_dev *hwdev, dma_addr_t mask) +extern inline int pci_dma_supported(struct pci_dev *hwdev, u64 mask) { return 1; } diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-sparc/scatterlist.h linux/include/asm-sparc/scatterlist.h --- /opt/kernel/linux-2.4.9/include/asm-sparc/scatterlist.h Tue Feb 1 08:37:19 2000 +++ linux/include/asm-sparc/scatterlist.h Tue Aug 21 11:03:52 2001 @@ -6,8 +6,6 @@ struct scatterlist { char * address; /* Location data is to be transferred to */ - char * alt_address; /* Location of actual if address is a - * dma indirect buffer. NULL otherwise */ unsigned int length; __u32 dvma_address; /* A place to hang host-specific addresses at. */ diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-sparc64/pci.h linux/include/asm-sparc64/pci.h --- /opt/kernel/linux-2.4.9/include/asm-sparc64/pci.h Wed May 16 19:31:27 2001 +++ linux/include/asm-sparc64/pci.h Tue Aug 21 11:03:52 2001 @@ -28,6 +28,31 @@ /* Dynamic DMA mapping stuff. */ +/* PCI 64-bit addressing works for all slots on all controller + * types on sparc64. However, it requires that the device + * can drive enough of the 64 bits. + */ +#define PCI64_REQUIRED_MASK (~(dma64_addr_t)0) +#define PCI64_ADDR_BASE 0xfffc000000000000 + +/* The PCI address space does not equal the physical memory + * address space. The networking and block device layers use + * this boolean for bounce buffer decisions. + */ +#define PCI_DMA_BUS_IS_PHYS (0) + +/* Internal shorthand. */ +#define __PCI_DMA_FLAG_MUST_DAC (PCI_DMA_FLAG_HUGE_MAPS|\ + PCI_DMA_FLAG_DAC_ONLY) + +/* We want the driver to backoff to SAC addresses unless the + * usage of DAC addressing is absolutely required. + */ +#define pci_dac_cycles_ok(pci_dev) \ + (((pci_dev)->dma_flags & (__PCI_DMA_FLAG_MUST_DAC)) ? \ + (((pci_dev)->dma_mask & PCI64_REQUIRED_MASK) == PCI64_REQUIRED_MASK) : \ + 0) + #include struct pci_dev; @@ -36,6 +61,7 @@ * hwdev should be valid struct pci_dev pointer for PCI devices. */ extern void *pci_alloc_consistent(struct pci_dev *hwdev, size_t size, dma_addr_t *dma_handle); +extern void *pci64_alloc_consistent(struct pci_dev *hwdev, size_t size, dma64_addr_t *dma_handle); /* Free and unmap a consistent DMA buffer. * cpu_addr is what was returned from pci_alloc_consistent, @@ -46,6 +72,8 @@ * past this call are illegal. */ extern void pci_free_consistent(struct pci_dev *hwdev, size_t size, void *vaddr, dma_addr_t dma_handle); +extern void pci64_free_consistent(struct pci_dev *hwdev, size_t size, + void *vaddr, dma64_addr_t dma_handle); /* Map a single buffer of the indicated size for DMA in streaming mode. * The 32-bit bus address to use is returned. @@ -64,6 +92,20 @@ */ extern void pci_unmap_single(struct pci_dev *hwdev, dma_addr_t dma_addr, size_t size, int direction); +/* No highmem on sparc64, plus we have an IOMMU, so mapping pages is easy. */ +#define pci_map_page(dev, page, off, size, dir) \ + pci_map_single(dev, (page_address(page) + (off)), size, dir) +#define pci_unmap_page(dev,addr,sz,dir) pci_unmap_single(dev,addr,sz,dir) + +/* The 64-bit cases might have to do something interesting if + * PCI_DMA_FLAG_HUGE_MAPS is set in hwdev->dma_flags. + */ +extern dma64_addr_t pci64_map_page(struct pci_dev *hwdev, + struct page *page, unsigned long offset, + size_t size, int direction); +extern void pci64_unmap_page(struct pci_dev *hwdev, dma64_addr_t dma_addr, + size_t size, int direction); + /* Map a set of buffers described by scatterlist in streaming * mode for DMA. This is the scather-gather version of the * above pci_map_single interface. Here the scatter gather list @@ -79,13 +121,19 @@ * Device ownership issues as mentioned above for pci_map_single are * the same here. */ -extern int pci_map_sg(struct pci_dev *hwdev, struct scatterlist *sg, int nents, int direction); +extern int pci_map_sg(struct pci_dev *hwdev, struct scatterlist *sg, + int nents, int direction); +extern int pci64_map_sg(struct pci_dev *hwdev, struct scatterlist *sg, + int nents, int direction); /* Unmap a set of streaming mode DMA translations. * Again, cpu read rules concerning calls here are the same as for * pci_unmap_single() above. */ -extern void pci_unmap_sg(struct pci_dev *hwdev, struct scatterlist *sg, int nhwents, int direction); +extern void pci_unmap_sg(struct pci_dev *hwdev, struct scatterlist *sg, + int nhwents, int direction); +extern void pci64_unmap_sg(struct pci_dev *hwdev, struct scatterlist *sg, + int nhwents, int direction); /* Make physical memory consistent for a single * streaming mode DMA translation after a transfer. @@ -96,7 +144,10 @@ * next point you give the PCI dma address back to the card, the * device again owns the buffer. */ -extern void pci_dma_sync_single(struct pci_dev *hwdev, dma_addr_t dma_handle, size_t size, int direction); +extern void pci_dma_sync_single(struct pci_dev *hwdev, dma_addr_t dma_handle, + size_t size, int direction); +extern void pci64_dma_sync_single(struct pci_dev *hwdev, dma64_addr_t dma_handle, + size_t size, int direction); /* Make physical memory consistent for a set of streaming * mode DMA translations after a transfer. @@ -105,13 +156,14 @@ * same rules and usage. */ extern void pci_dma_sync_sg(struct pci_dev *hwdev, struct scatterlist *sg, int nelems, int direction); +extern void pci64_dma_sync_sg(struct pci_dev *hwdev, struct scatterlist *sg, int nelems, int direction); /* Return whether the given PCI device DMA address mask can * be supported properly. For example, if your device can * only drive the low 24-bits during PCI bus mastering, then * you would pass 0x00ffffff as the mask to this function. */ -extern int pci_dma_supported(struct pci_dev *hwdev, dma_addr_t mask); +extern int pci_dma_supported(struct pci_dev *hwdev, u64 mask); /* Return the index of the PCI controller for device PDEV. */ diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-sparc64/scatterlist.h linux/include/asm-sparc64/scatterlist.h --- /opt/kernel/linux-2.4.9/include/asm-sparc64/scatterlist.h Tue Dec 21 07:05:52 1999 +++ linux/include/asm-sparc64/scatterlist.h Tue Aug 21 11:03:52 2001 @@ -5,17 +5,24 @@ #include struct scatterlist { - char * address; /* Location data is to be transferred to */ - char * alt_address; /* Location of actual if address is a - * dma indirect buffer. NULL otherwise */ - unsigned int length; + /* This will disappear in 2.5.x */ + char *address; - __u32 dvma_address; /* A place to hang host-specific addresses at. */ - __u32 dvma_length; + /* These two are only valid if ADDRESS member of this + * struct is NULL. + */ + struct page *page; + unsigned int offset; + + unsigned int length; + + dma64_addr_t dma_address; + __u32 dma_length; }; -#define sg_dma_address(sg) ((sg)->dvma_address) -#define sg_dma_len(sg) ((sg)->dvma_length) +#define sg_dma_address(sg) ((dma_addr_t) ((sg)->dma_address)) +#define sg_dma64_address(sg) ((sg)->dma_address) +#define sg_dma_len(sg) ((sg)->dma_length) #define ISA_DMA_THRESHOLD (~0UL) diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/asm-sparc64/types.h linux/include/asm-sparc64/types.h --- /opt/kernel/linux-2.4.9/include/asm-sparc64/types.h Tue Feb 1 08:37:19 2000 +++ linux/include/asm-sparc64/types.h Tue Aug 21 11:03:52 2001 @@ -45,9 +45,10 @@ #define BITS_PER_LONG 64 -/* Dma addresses are 32-bits wide for now. */ +/* Dma addresses come in 32-bit and 64-bit flavours. */ typedef u32 dma_addr_t; +typedef u64 dma64_addr_t; #endif /* __KERNEL__ */ diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/linux/blkdev.h linux/include/linux/blkdev.h --- /opt/kernel/linux-2.4.9/include/linux/blkdev.h Wed Aug 15 23:21:30 2001 +++ linux/include/linux/blkdev.h Tue Aug 21 12:54:13 2001 @@ -36,7 +36,7 @@ unsigned long hard_sector, hard_nr_sectors; unsigned int nr_segments; unsigned int nr_hw_segments; - unsigned long current_nr_sectors; + unsigned long current_nr_sectors, hard_cur_sectors; void * special; char * buffer; struct completion * waiting; @@ -110,6 +110,14 @@ */ char head_active; + struct page *bounce_limit; + + /* + * segment boundary mask, one segment cannot cross a certain boundary + * in memory + */ + dma_addr_t seg_bound_mask; + /* * Is meant to protect the queue in the future instead of * io_request_lock @@ -122,6 +130,26 @@ wait_queue_head_t wait_for_request; }; +extern unsigned long blk_max_low_pfn; + +#define BLK_BOUNCE_HIGH (blk_max_low_pfn * PAGE_SIZE) + +extern void blk_queue_bounce_limit(request_queue_t *, unsigned long); + +#ifdef CONFIG_HIGHMEM +extern struct buffer_head *create_bounce(int, struct buffer_head *); +extern inline struct buffer_head *blk_queue_bounce(request_queue_t *q, int rw, + struct buffer_head *bh) +{ + if (bh->b_page <= q->bounce_limit) + return bh; + + return create_bounce(rw, bh); +} +#else +#define blk_queue_bounce(q, rw, bh) (bh) +#endif + struct blk_dev_struct { /* * queue_proc has to be atomic @@ -149,8 +177,7 @@ extern void grok_partitions(struct gendisk *dev, int drive, unsigned minors, long size); extern void register_disk(struct gendisk *dev, kdev_t first, unsigned minors, struct block_device_operations *ops, long size); extern void generic_make_request(int rw, struct buffer_head * bh); -extern request_queue_t *blk_get_queue(kdev_t dev); -extern inline request_queue_t *__blk_get_queue(kdev_t dev); +extern inline request_queue_t *blk_get_queue(kdev_t dev); extern void blkdev_release_request(struct request *); /* @@ -161,6 +188,8 @@ extern void blk_queue_headactive(request_queue_t *, int); extern void blk_queue_make_request(request_queue_t *, make_request_fn *); extern void generic_unplug_device(void *); +extern inline int blk_seg_merge_ok(request_queue_t *, struct buffer_head *, + struct buffer_head *); extern int * blk_size[MAX_BLKDEV]; diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/linux/fs.h linux/include/linux/fs.h --- /opt/kernel/linux-2.4.9/include/linux/fs.h Wed Aug 15 23:21:11 2001 +++ linux/include/linux/fs.h Tue Aug 21 11:54:54 2001 @@ -277,6 +277,8 @@ #define bh_offset(bh) ((unsigned long)(bh)->b_data & ~PAGE_MASK) +#define bh_bus(bh) (page_to_bus((bh)->b_page) + bh_offset((bh))) + extern void set_bh_page(struct buffer_head *bh, struct page *page, unsigned long offset); #define touch_buffer(bh) SetPageReferenced(bh->b_page) diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/linux/highmem.h linux/include/linux/highmem.h --- /opt/kernel/linux-2.4.9/include/linux/highmem.h Wed Aug 15 23:21:21 2001 +++ linux/include/linux/highmem.h Tue Aug 21 11:13:01 2001 @@ -13,8 +13,7 @@ /* declarations for linux/mm/highmem.c */ FASTCALL(unsigned int nr_free_highpages(void)); -extern struct buffer_head * create_bounce(int rw, struct buffer_head * bh_orig); - +extern struct buffer_head *create_bounce(int rw, struct buffer_head * bh_orig); static inline char *bh_kmap(struct buffer_head *bh) { @@ -26,6 +25,42 @@ kunmap(bh->b_page); } +/* + * remember to add offset! and never ever reenable interrupts between a + * bh_kmap_irq and bh_kunmap_irq!! + */ +static inline char *bh_kmap_irq(struct buffer_head *bh, unsigned long *flags) +{ + unsigned long addr; + + __save_flags(*flags); + + /* + * could be low + */ + if (!PageHighMem(bh->b_page)) + return bh->b_data; + + /* + * it's a highmem page + */ + __cli(); + addr = (unsigned long) kmap_atomic(bh->b_page, KM_BH_IRQ); + + if (addr & ~PAGE_MASK) + BUG(); + + return (char *) addr + bh_offset(bh); +} + +static inline void bh_kunmap_irq(char *buffer, unsigned long *flags) +{ + unsigned long ptr = (unsigned long) buffer & PAGE_MASK; + + kunmap_atomic((void *) ptr, KM_BH_IRQ); + __restore_flags(*flags); +} + #else /* CONFIG_HIGHMEM */ static inline unsigned int nr_free_highpages(void) { return 0; } @@ -37,8 +72,10 @@ #define kmap_atomic(page,idx) kmap(page) #define kunmap_atomic(page,idx) kunmap(page) -#define bh_kmap(bh) ((bh)->b_data) -#define bh_kunmap(bh) do { } while (0) +#define bh_kmap(bh) ((bh)->b_data) +#define bh_kunmap(bh) do { } while (0) +#define bh_kmap_irq(bh, flags) ((bh)->b_data) +#define bh_kunmap_irq(bh, flags) do { } while (0) #endif /* CONFIG_HIGHMEM */ diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/linux/ide.h linux/include/linux/ide.h --- /opt/kernel/linux-2.4.9/include/linux/ide.h Wed Aug 15 23:22:22 2001 +++ linux/include/linux/ide.h Tue Aug 21 12:55:23 2001 @@ -507,6 +507,7 @@ unsigned reset : 1; /* reset after probe */ unsigned autodma : 1; /* automatically try to enable DMA at boot */ unsigned udma_four : 1; /* 1=ATA-66 capable, 0=default */ + unsigned highmem : 1; /* can do full 32-bit dma */ byte channel; /* for dual-port chips: 0=primary, 1=secondary */ #ifdef CONFIG_BLK_DEV_IDEPCI struct pci_dev *pci_dev; /* for pci chipsets */ @@ -812,6 +813,21 @@ ide_preempt, /* insert rq in front of current request */ ide_end /* insert rq at end of list, but don't wait for it */ } ide_action_t; + +/* + * temporarily mapping a (possible) highmem bio + */ +#define ide_rq_offset(rq) (((rq)->hard_cur_sectors - (rq)->current_nr_sectors) << 9) + +extern inline void *ide_map_buffer(struct request *rq, unsigned long *flags) +{ + return bh_kmap_irq(rq->bh, flags) + ide_rq_offset(rq); +} + +extern inline void ide_unmap_buffer(char *buffer, unsigned long *flags) +{ + bh_kunmap_irq(buffer, flags); +} /* * This function issues a special IDE device request diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/include/linux/pci.h linux/include/linux/pci.h --- /opt/kernel/linux-2.4.9/include/linux/pci.h Wed Aug 15 23:21:44 2001 +++ linux/include/linux/pci.h Tue Aug 21 11:55:11 2001 @@ -314,6 +314,32 @@ #define PCI_DMA_FROMDEVICE 2 #define PCI_DMA_NONE 3 +/* These are the boolean attributes stored in pci_dev->dma_flags. */ + +/* Device may hold an enormous number of mappings at once? */ +#define PCI_DMA_FLAG_HUGE_MAPS 0x00000001 + +/* The device only supports Dual Address Cycles. */ +#define PCI_DMA_FLAG_DAC_ONLY 0x00000002 + +/* XXX This needs more thought. For scatterlist mappings, there is no + * XXX way we can "extend" the scatterlist to make sure things fit into + * XXX these rules. The block/networking/etc. queueing layer will need + * XXX to prevent this situation from making it's way to the driver. + * XXX Then it is up to the PCI platform layer to make sure not to + * XXX perform any kind of "coalescing" that would reintroduce the 4GB + * XXX crossing violation. + * + * The device does not handle a mapping which crosses a 4GB address + * boundary. Usually this is because the device implements it's address + * registers such that the upper 32-bits are fixed during the DMA transfer. + * In other cases, this limitation arises due to hardware bugs. + */ +#define PCI_DMA_NO_4GB_CROSS 0x00000004 + +/* Reserved for arch-specific flags. */ +#define PCI_DMA_FLAG_ARCHMASK 0xf0000000 + #define DEVICE_COUNT_COMPATIBLE 4 #define DEVICE_COUNT_IRQ 2 #define DEVICE_COUNT_DMA 2 @@ -353,11 +379,12 @@ struct pci_driver *driver; /* which driver has allocated this device */ void *driver_data; /* data private to the driver */ - dma_addr_t dma_mask; /* Mask of the bits of bus address this + u64 dma_mask; /* Mask of the bits of bus address this device implements. Normally this is 0xffffffff. You only need to change this if your device has broken DMA or supports 64-bit transfers. */ + unsigned int dma_flags; /* See PCI_DMA_FLAG_* above */ u32 current_state; /* Current operating state. In ACPI-speak, this is D0-D3, D0 being fully functional, @@ -559,7 +586,8 @@ int pci_enable_device(struct pci_dev *dev); void pci_disable_device(struct pci_dev *dev); void pci_set_master(struct pci_dev *dev); -int pci_set_dma_mask(struct pci_dev *dev, dma_addr_t mask); +int pci_set_dma_mask(struct pci_dev *dev, u64 mask); +void pci_change_dma_flag(struct pci_dev *dev, unsigned int on, unsigned int off); int pci_assign_resource(struct pci_dev *dev, int i); /* Power management related routines */ @@ -641,7 +669,8 @@ static inline int pci_enable_device(struct pci_dev *dev) { return -EIO; } static inline void pci_disable_device(struct pci_dev *dev) { } static inline int pci_module_init(struct pci_driver *drv) { return -ENODEV; } -static inline int pci_set_dma_mask(struct pci_dev *dev, dma_addr_t mask) { return -EIO; } +static inline int pci_set_dma_mask(struct pci_dev *dev, u64 mask) { return -EIO; } +static inline void pci_change_dma_flag(struct pci_dev *dev, unsigned int on, unsigned int off) { } static inline int pci_assign_resource(struct pci_dev *dev, int i) { return -EBUSY;} static inline int pci_register_driver(struct pci_driver *drv) { return 0;} static inline void pci_unregister_driver(struct pci_driver *drv) { } diff -urN --exclude-from /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.9/kernel/ksyms.c linux/kernel/ksyms.c --- /opt/kernel/linux-2.4.9/kernel/ksyms.c Mon Aug 13 02:35:38 2001 +++ linux/kernel/ksyms.c Tue Aug 21 11:11:56 2001 @@ -121,6 +121,8 @@ EXPORT_SYMBOL(kunmap_high); EXPORT_SYMBOL(highmem_start_page); EXPORT_SYMBOL(create_bounce); +EXPORT_SYMBOL(kmap_prot); +EXPORT_SYMBOL(kmap_pte); #endif /* filesystem internal functions */