diff options
| -rw-r--r-- | arch/powerpc/include/asm/tce.h | 4 | ||||
| -rw-r--r-- | arch/powerpc/platforms/powernv/pci.c | 84 | ||||
| -rw-r--r-- | arch/powerpc/platforms/pseries/iommu.c | 6 |
3 files changed, 78 insertions, 16 deletions
diff --git a/arch/powerpc/include/asm/tce.h b/arch/powerpc/include/asm/tce.h index 3dead82d5979..743f36b38e5d 100644 --- a/arch/powerpc/include/asm/tce.h +++ b/arch/powerpc/include/asm/tce.h @@ -31,7 +31,9 @@ */ #define TCE_VB 0 #define TCE_PCI 1 -#define TCE_PCI_SW_INVAL 2 +#define TCE_PCI_SWINV_CREATE 2 +#define TCE_PCI_SWINV_FREE 4 +#define TCE_PCI_SWINV_PAIR 8 /* TCE page size is 4096 bytes (1 << 12) */ diff --git a/arch/powerpc/platforms/powernv/pci.c b/arch/powerpc/platforms/powernv/pci.c index 85bb66d7f933..8b90d945399a 100644 --- a/arch/powerpc/platforms/powernv/pci.c +++ b/arch/powerpc/platforms/powernv/pci.c @@ -257,12 +257,54 @@ struct pci_ops pnv_pci_ops = { .write = pnv_pci_write_config, }; + +static void pnv_tce_invalidate(struct iommu_table *tbl, + u64 *startp, u64 *endp) +{ + u64 __iomem *invalidate = (u64 __iomem *)tbl->it_index; + unsigned long start, end, inc; + + start = __pa(startp); + end = __pa(endp); + + + /* BML uses this case for p6/p7/galaxy2: Shift addr and put in node */ + if (tbl->it_busno) { + start <<= 12; + end <<= 12; + inc = 128 << 12; + start |= tbl->it_busno; + end |= tbl->it_busno; + } + /* p7ioc-style invalidation, 2 TCEs per write */ + else if (tbl->it_type & TCE_PCI_SWINV_PAIR) { + start |= (1ull << 63); + end |= (1ull << 63); + inc = 16; + } + /* Default (older HW) */ + else + inc = 128; + + end |= inc - 1; /* round up end to be different than start */ + + mb(); /* Ensure above stores are visible */ + while (start <= end) { + __raw_writeq(start, invalidate); + start += inc; + } + /* The iommu layer will do another mb() for us on build() and + * we don't care on free() + */ +} + + static int pnv_tce_build(struct iommu_table *tbl, long index, long npages, unsigned long uaddr, enum dma_data_direction direction, struct dma_attrs *attrs) { u64 proto_tce; - u64 *tcep; + u64 *tcep, *tces; u64 rpn; proto_tce = TCE_PCI_READ; // Read allowed @@ -270,25 +312,33 @@ static int pnv_tce_build(struct iommu_table *tbl, long index, long npages, if (direction != DMA_TO_DEVICE) proto_tce |= TCE_PCI_WRITE; - tcep = ((u64 *)tbl->it_base) + index; + tces = tcep = ((u64 *)tbl->it_base) + index - tbl->it_offset; + rpn = __pa(uaddr) >> TCE_SHIFT; - while (npages--) { - /* can't move this out since we might cross LMB boundary */ - rpn = (virt_to_abs(uaddr)) >> TCE_SHIFT; - *tcep = proto_tce | (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT; + while (npages--) + *(tcep++) = proto_tce | (rpn++ << TCE_RPN_SHIFT); + + /* Some implementations won't cache invalid TCEs and thus may not + * need that flush. We'll probably turn it_type into a bit mask + * of flags if that becomes the case + */ + if (tbl->it_type & TCE_PCI_SWINV_CREATE) + pnv_tce_invalidate(tbl, tces, tcep - 1); - uaddr += TCE_PAGE_SIZE; - tcep++; - } return 0; } static void pnv_tce_free(struct iommu_table *tbl, long index, long npages) { - u64 *tcep = ((u64 *)tbl->it_base) + index; + u64 *tcep, *tces; + + tces = tcep = ((u64 *)tbl->it_base) + index - tbl->it_offset; while (npages--) *(tcep++) = 0; + + if (tbl->it_type & TCE_PCI_SWINV_FREE) + pnv_tce_invalidate(tbl, tces, tcep - 1); } void pnv_pci_setup_iommu_table(struct iommu_table *tbl, @@ -308,13 +358,14 @@ static struct iommu_table * __devinit pnv_pci_setup_bml_iommu(struct pci_controller *hose) { struct iommu_table *tbl; - const __be64 *basep; + const __be64 *basep, *swinvp; const __be32 *sizep; basep = of_get_property(hose->dn, "linux,tce-base", NULL); sizep = of_get_property(hose->dn, "linux,tce-size", NULL); if (basep == NULL || sizep == NULL) { - pr_err("PCI: %s has missing tce entries !\n", hose->dn->full_name); + pr_err("PCI: %s has missing tce entries !\n", + hose->dn->full_name); return NULL; } tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, hose->node); @@ -323,6 +374,15 @@ pnv_pci_setup_bml_iommu(struct pci_controller *hose) pnv_pci_setup_iommu_table(tbl, __va(be64_to_cpup(basep)), be32_to_cpup(sizep), 0); iommu_init_table(tbl, hose->node); + + /* Deal with SW invalidated TCEs when needed (BML way) */ + swinvp = of_get_property(hose->dn, "linux,tce-sw-invalidate-info", + NULL); + if (swinvp) { + tbl->it_busno = swinvp[1]; + tbl->it_index = (unsigned long)ioremap(swinvp[0], 8); + tbl->it_type = TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE; + } return tbl; } diff --git a/arch/powerpc/platforms/pseries/iommu.c b/arch/powerpc/platforms/pseries/iommu.c index ea70667d4fc3..c442f2b1980f 100644 --- a/arch/powerpc/platforms/pseries/iommu.c +++ b/arch/powerpc/platforms/pseries/iommu.c @@ -106,7 +106,7 @@ static int tce_build_pSeries(struct iommu_table *tbl, long index, tcep++; } - if (tbl->it_type == TCE_PCI_SW_INVAL) + if (tbl->it_type == TCE_PCI_SWINV_CREATE) tce_invalidate_pSeries_sw(tbl, tces, tcep - 1); return 0; } @@ -121,7 +121,7 @@ static void tce_free_pSeries(struct iommu_table *tbl, long index, long npages) while (npages--) *(tcep++) = 0; - if (tbl->it_type == TCE_PCI_SW_INVAL) + if (tbl->it_type == TCE_PCI_SWINV_FREE) tce_invalidate_pSeries_sw(tbl, tces, tcep - 1); } @@ -511,7 +511,7 @@ static void iommu_table_setparms(struct pci_controller *phb, */ tbl->it_index = (unsigned long) ioremap(sw_inval[0], 8); tbl->it_busno = sw_inval[1]; /* overload this with magic */ - tbl->it_type = TCE_PCI_SW_INVAL; + tbl->it_type = TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE; } } |