dma: phytium: fix more bugs of pswiotlb

arch/arm64/Kconfig

@@ -1313,6 +1313,16 @@ config SOCIONEXT_SYNQUACER_PREITS
 
 	  If unsure, say Y.
 
+config PHYTIUM_ERRATUM_FT3386
+	bool "FT3386: enable Phytium FT3386 pswiotlb can improve dma performance"
+	depends on PSWIOTLB
+	default y
+	help
+	  Phytium FT3386 pswiotlb can improve D2H dma performance and
+	  should be enabled by default.
+
+	  If unsure, say Y.
+
 endmenu # "ARM errata workarounds via the alternatives framework"
 
 choice

arch/arm64/include/asm/cputype.h

@@ -103,6 +103,7 @@
 #define APM_CPU_PART_XGENE		0x000
 #define APM_CPU_VAR_POTENZA		0x00
 
+
 #define CAVIUM_CPU_PART_THUNDERX	0x0A1
 #define CAVIUM_CPU_PART_THUNDERX_81XX	0x0A2
 #define CAVIUM_CPU_PART_THUNDERX_83XX	0x0A3
@@ -164,6 +165,7 @@
 #define PHYTIUM_CPU_PART_FTC862		0x862
 #define PHYTIUM_CPU_SOCID_PS24080	0x6
 
+#define MIDR_FT_FTC862	MIDR_CPU_MODEL(ARM_CPU_IMP_PHYTIUM, PHYTIUM_CPU_PART_FTC862)
 #define MIDR_CORTEX_A53 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53)
 #define MIDR_CORTEX_A57 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57)
 #define MIDR_CORTEX_A72 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A72)

arch/arm64/kernel/cpu_errata.c

@@ -495,6 +495,28 @@ static const struct midr_range erratum_ac03_cpu_38_list[] = {
 };
 #endif
 
+#ifdef CONFIG_PHYTIUM_ERRATUM_FT3386
+#define SOC_ID_PS23064	0x8
+#define SOC_ID_PS24080	0x6
+#define SYS_AIDR_EL1    sys_reg(3, 1, 0, 0, 7)
+bool __read_mostly is_ps_socs;
+static bool
+should_enable_phytium_ft3386_pswiotlb(const struct arm64_cpu_capabilities *entry, int unused)
+{
+	u32 model;
+	u32 soc_id;
+
+	soc_id = read_sysreg_s(SYS_AIDR_EL1);
+	model = read_cpuid_id();
+	if ((soc_id == SOC_ID_PS23064 || soc_id == SOC_ID_PS24080)
+				&& model == entry->midr_range.model) {
+		is_ps_socs = true;
+		return true;
+	} else
+		return false;
+}
+#endif
+
 const struct arm64_cpu_capabilities arm64_errata[] = {
 #ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE
 	{
@@ -808,6 +830,14 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
 		ERRATA_MIDR_RANGE_LIST(erratum_spec_unpriv_load_list),
 	},
 #endif
+#ifdef CONFIG_PHYTIUM_ERRATUM_FT3386
+	{
+		.desc = "Phytium erratum FT3386",
+		.capability = ARM64_WORKAROUND_PHYTIUM_FT3386,
+		ERRATA_MIDR_ALL_VERSIONS(MIDR_FT_FTC862),
+		.matches = should_enable_phytium_ft3386_pswiotlb,
+	},
+#endif
 #ifdef CONFIG_AMPERE_ERRATUM_AC03_CPU_38
 	{
 		.desc = "AmpereOne erratum AC03_CPU_38",

arch/arm64/mm/dma-mapping.c

@@ -13,6 +13,10 @@
 #include <asm/cacheflush.h>
 #include <asm/xen/xen-ops.h>
 
+#ifdef CONFIG_PSWIOTLB
+#include <linux/pswiotlb.h>
+#endif
+
 void arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
 			      enum dma_data_direction dir)
 {
@@ -61,5 +65,9 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
 	if (iommu)
 		iommu_setup_dma_ops(dev, dma_base, dma_base + size - 1);
 
+#ifdef CONFIG_PSWIOTLB
+	pswiotlb_setup_dma_ops(dev);
+#endif
+
 	xen_setup_dma_ops(dev);
 }

arch/arm64/tools/cpucaps

@@ -106,3 +106,4 @@ WORKAROUND_SPECULATIVE_SSBS
 WORKAROUND_SPECULATIVE_UNPRIV_LOAD
 HAS_LS64
 HAS_LS64_V
+WORKAROUND_PHYTIUM_FT3386

drivers/pci/pci.c

@@ -36,9 +36,6 @@
 #include <linux/suspend.h>
 #endif
 #include "pci.h"
-#ifdef CONFIG_PSWIOTLB
-#include <linux/pswiotlb.h>
-#endif
 
 DEFINE_MUTEX(pci_slot_mutex);
 
@@ -4558,15 +4555,6 @@ void __weak pcibios_set_master(struct pci_dev *dev)
  */
 void pci_set_master(struct pci_dev *dev)
 {
-#ifdef CONFIG_PSWIOTLB
-	if ((pswiotlb_force_disable != true) &&
-		is_phytium_ps_socs()) {
-		dev->dev.can_use_pswiotlb = pswiotlb_is_dev_in_passthroughlist(dev);
-		dev_info(&dev->dev, "The device %s use pswiotlb because vendor 0x%04x %s in pswiotlb passthroughlist\n",
-					dev->dev.can_use_pswiotlb ? "would" : "would NOT",
-					dev->vendor, dev->dev.can_use_pswiotlb ? "is NOT" : "is");
-	}
-#endif
 	__pci_set_master(dev, true);
 	pcibios_set_master(dev);
 }

drivers/pci/probe.c

@@ -20,9 +20,6 @@
 #include <linux/pm_runtime.h>
 #include <linux/bitfield.h>
 #include "pci.h"
-#ifdef CONFIG_PSWIOTLB
-#include <linux/pswiotlb.h>
-#endif
 
 #define CARDBUS_LATENCY_TIMER	176	/* secondary latency timer */
 #define CARDBUS_RESERVE_BUSNR	3
@@ -2579,13 +2576,8 @@ void pci_device_add(struct pci_dev *dev, struct pci_bus *bus)
 
 	dma_set_max_seg_size(&dev->dev, 65536);
 	dma_set_seg_boundary(&dev->dev, 0xffffffff);
-#ifdef CONFIG_PSWIOTLB
-	if ((pswiotlb_force_disable != true) &&
-		is_phytium_ps_socs()) {
-		pswiotlb_store_local_node(dev, bus);
-		dma_set_seg_boundary(&dev->dev, 0xffffffffffff);
-	}
-#endif
+
+	pci_configure_pswiotlb(dev, bus);
 
 	pcie_failed_link_retrain(dev);
 

drivers/pci/quirks.c

@@ -32,6 +32,9 @@
 #include <linux/suspend.h>
 #include <linux/switchtec.h>
 #include "pci.h"
+#ifdef CONFIG_PSWIOTLB
+#include <linux/pswiotlb.h>
+#endif
 
 /*
  * Retrain the link of a downstream PCIe port by hand if necessary.
@@ -6470,6 +6473,17 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_XILINX, 0x5020, of_pci_make_dev_node);
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_XILINX, 0x5021, of_pci_make_dev_node);
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_REDHAT, 0x0005, of_pci_make_dev_node);
 
+void pci_configure_pswiotlb(struct pci_dev *dev, struct pci_bus *bus)
+{
+#ifdef CONFIG_PSWIOTLB
+       if ((pswiotlb_force_disable != true) &&
+               is_phytium_ps_socs()) {
+               pswiotlb_store_local_node(dev, bus);
+               dma_set_seg_boundary(&dev->dev, 0xffffffffffff);
+       }
+#endif
+}
+
 /*
  * Devices known to require a longer delay before first config space access
  * after reset recovery or resume from D3cold:

include/linux/device.h

@@ -658,7 +658,10 @@ struct device_physical_location {
  * @dma_io_tlb_lock:	Protects changes to the list of active pools.
  * @dma_uses_io_tlb: %true if device has used the software IO TLB.
  * @dma_p_io_tlb_mem: Phytium Software IO TLB allocator.  Not for driver use.
+ * @orig_dma_ops:    Original DMA mapping operations for this device.
+ * @local_node:	NUMA node this device is really belong to.
  * @dma_uses_p_io_tlb: %true if device has used the Phytium software IO TLB.
+ * @can_use_pswiotlb: %true if device can use the Phytium software IO TLB.
  * @archdata:	For arch-specific additions.
  * @of_node:	Associated device tree node.
  * @fwnode:	Associated device node supplied by platform firmware.
@@ -767,8 +770,16 @@ struct device {
 #endif
 #ifdef CONFIG_PSWIOTLB
 	struct p_io_tlb_mem *dma_p_io_tlb_mem;
-	bool dma_uses_p_io_tlb;
-	bool can_use_pswiotlb;
+#ifdef CONFIG_DMA_OPS
+	const struct dma_map_ops *orig_dma_ops;
+#endif
+	struct {
+#ifdef CONFIG_NUMA
+		int local_node;
+#endif
+		bool dma_uses_p_io_tlb;
+		bool can_use_pswiotlb;
+	};
 #endif
 #ifdef CONFIG_SWIOTLB_DYNAMIC
 	struct list_head dma_io_tlb_pools;
@@ -783,9 +794,6 @@ struct device {
 
 #ifdef CONFIG_NUMA
 	int		numa_node;	/* NUMA node this device is close to */
-#ifdef CONFIG_PSWIOTLB
-	int     local_node; /* NUMA node this device is really belong to */
-#endif
 #endif
 	dev_t			devt;	/* dev_t, creates the sysfs "dev" */
 	u32			id;	/* device instance */

include/linux/pci.h

@@ -2314,9 +2314,12 @@ enum pci_fixup_pass {
 
 #ifdef CONFIG_PCI_QUIRKS
 void pci_fixup_device(enum pci_fixup_pass pass, struct pci_dev *dev);
+void pci_configure_pswiotlb(struct pci_dev *dev, struct pci_bus *bus);
 #else
 static inline void pci_fixup_device(enum pci_fixup_pass pass,
 				    struct pci_dev *dev) { }
+static inline void pci_configure_pswiotlb(struct pci_dev *dev,
+					struct pci_bus *bus) { }
 #endif
 
 void __iomem *pcim_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen);

include/linux/pswiotlb.h

@@ -18,15 +18,11 @@ struct scatterlist;
 extern bool pswiotlb_force_disable;
 struct p_io_tlb_pool;
 
-#define SOC_ID_PS23064	0x8
-#define SOC_ID_PS24080	0x6
-#define MIDR_PS    0x700F8620
-#define SYS_AIDR_EL1    sys_reg(3, 1, 0, 0, 7)
 #define PSWIOTLB_VERBOSE	(1 << 0) /* verbose initialization */
 #define PSWIOTLB_FORCEOFF	(1 << 1) /* force phytium bounce buffering off*/
 #define PSWIOTLB_ANY	(1 << 2) /* allow any memory for the buffer */
 #define PSWIOTLB_FREE_THRESHOLD 30
-static bool is_ps_socs;
+extern bool __read_mostly is_ps_socs;
 
 /*
  * Maximum allowable number of contiguous slabs to map,
@@ -79,6 +75,7 @@ dma_addr_t pswiotlb_map(struct device *dev, int nid, phys_addr_t phys,
 void pswiotlb_store_local_node(struct pci_dev *dev, struct pci_bus *bus);
 void iommu_dma_unmap_sg_pswiotlb(struct device *dev, struct scatterlist *sg, unsigned long iova,
 			size_t mapped, int nents, enum dma_data_direction dir, unsigned long attrs);
+void pswiotlb_setup_dma_ops(struct device *dev);
 #ifdef CONFIG_PSWIOTLB
 struct pswiotlb_passthroughlist {
 	struct list_head node;
@@ -211,19 +208,9 @@ struct p_io_tlb_pool *pswiotlb_find_pool(struct device *dev, int nid, phys_addr_
 
 static inline bool is_phytium_ps_socs(void)
 {
-	unsigned int soc_id;
-	unsigned int midr;
-
 	if (likely(is_ps_socs))
 		return true;
-
-	soc_id = read_sysreg_s(SYS_AIDR_EL1);
-	midr = read_cpuid_id();
-	if ((soc_id == SOC_ID_PS23064 || soc_id == SOC_ID_PS24080)
-				&& midr == MIDR_PS) {
-		is_ps_socs = true;
-		return true;
-	} else
+	else
 		return false;
 }
 

include/trace/events/pswiotlb.h

@@ -20,22 +20,23 @@ TRACE_EVENT(pswiotlb_bounced,
 		__field(u64,	dma_mask)
 		__field(dma_addr_t,	dev_addr)
 		__field(size_t,	size)
-		__field(bool,	force)
+		__field(bool,	forceoff)
 	),
 
 	TP_fast_assign(
 		__assign_str(dev_name, dev_name(dev));
 		__entry->dma_mask = (dev->dma_mask ? *dev->dma_mask : 0);
 		__entry->dev_addr = dev_addr;
 		__entry->size = size;
+		__entry->forceoff = pswiotlb_force_disable;
 	),
 
 	TP_printk("dev_name: %s dma_mask=%llx dev_addr=%llx size=%zu %s",
 		__get_str(dev_name),
 		__entry->dma_mask,
 		(unsigned long long)__entry->dev_addr,
 		__entry->size,
-		__entry->force ? "NORMAL" : "FORCEOFF")
+		__entry->forceoff ? "FORCEOFF" : "NORMAL")
 );
 
 #endif /*  _TRACE_PSWIOTLB_H */

kernel/dma/contiguous.c

@@ -362,13 +362,13 @@ static struct page *cma_alloc_aligned(struct cma *cma, size_t size, gfp_t gfp)
  */
 struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp)
 {
+#ifdef CONFIG_DMA_NUMA_CMA
+	int nid = dev_to_node(dev);
+#endif
 #ifdef CONFIG_PSWIOTLB
 	if (check_if_pswiotlb_is_applicable(dev))
 		return NULL;
 #endif
-#ifdef CONFIG_DMA_NUMA_CMA
-	int nid = dev_to_node(dev);
-#endif
 
 	/* CMA can be used only in the context which permits sleeping */
 	if (!gfpflags_allow_blocking(gfp))