From: Joerg Roedel <jroedel@suse.de>
Date: Tue, 22 Dec 2015 13:34:22 +0100
Subject: iommu/amd: Make dma_ops_domain->next_index percpu
Git-commit: 5f6bed50050625f1b76fa2c0dfd0ce887cb0420a
Patch-mainline: v4.5-rc1
References: fate#321026

Make this pointer percpu so that we start searching for new
addresses in the range we last stopped and which is has a
higher probability of being still in the cache.

Signed-off-by: Joerg Roedel <jroedel@suse.de>
---
 drivers/iommu/amd_iommu.c | 39 +++++++++++++++++++++++++++++----------
 1 file changed, 29 insertions(+), 10 deletions(-)

--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -37,6 +37,7 @@
 #include <linux/msi.h>
 #include <linux/dma-contiguous.h>
 #include <linux/irqdomain.h>
+#include <linux/percpu.h>
 #include <asm/irq_remapping.h>
 #include <asm/io_apic.h>
 #include <asm/apic.h>
@@ -151,7 +152,7 @@ struct dma_ops_domain {
 	unsigned long aperture_size;
 
 	/* aperture index we start searching for free addresses */
-	unsigned long next_index;
+	u32 __percpu *next_index;
 
 	/* address space relevant data */
 	struct aperture_range *aperture[APERTURE_MAX_RANGES];
@@ -1736,18 +1737,30 @@ static unsigned long dma_ops_area_alloc(
 {
 	unsigned long boundary_size, mask;
 	unsigned long address = -1;
-	int start = dom->next_index;
-	int i;
+	u32 start, i;
+
+	preempt_disable();
 
 	mask = dma_get_seg_boundary(dev);
 
+	start = this_cpu_read(*dom->next_index);
+
+	/* Sanity check - is it really necessary? */
+	if (unlikely(start > APERTURE_MAX_RANGES)) {
+		start = 0;
+		this_cpu_write(*dom->next_index, 0);
+	}
+
 	boundary_size = mask + 1 ? ALIGN(mask + 1, PAGE_SIZE) >> PAGE_SHIFT :
 				   1UL << (BITS_PER_LONG - PAGE_SHIFT);
 
 	for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
 		struct aperture_range *range;
+		int index;
+
+		index = (start + i) % APERTURE_MAX_RANGES;
 
-		range = dom->aperture[(start + i) % APERTURE_MAX_RANGES];
+		range = dom->aperture[index];
 
 		if (!range || range->offset >= dma_mask)
 			continue;
@@ -1757,11 +1770,13 @@ static unsigned long dma_ops_area_alloc(
 						 align_mask);
 		if (address != -1) {
 			address = range->offset + (address << PAGE_SHIFT);
-			dom->next_index = i;
+			this_cpu_write(*dom->next_index, index);
 			break;
 		}
 	}
 
+	preempt_enable();
+
 	return address;
 }
 
@@ -1773,10 +1788,6 @@ static unsigned long dma_ops_alloc_addre
 {
 	unsigned long address = -1;
 
-#ifdef CONFIG_IOMMU_STRESS
-	dom->next_index = 0;
-#endif
-
 	while (address == -1) {
 		address = dma_ops_area_alloc(dev, dom, pages,
 					     align_mask, dma_mask);
@@ -2004,6 +2015,8 @@ static void dma_ops_domain_free(struct d
 	if (!dom)
 		return;
 
+	free_percpu(dom->next_index);
+
 	del_domain_from_list(&dom->domain);
 
 	free_pagetable(&dom->domain);
@@ -2029,6 +2042,7 @@ static void dma_ops_domain_free(struct d
 static struct dma_ops_domain *dma_ops_domain_alloc(void)
 {
 	struct dma_ops_domain *dma_dom;
+	int cpu;
 
 	dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
 	if (!dma_dom)
@@ -2037,6 +2051,10 @@ static struct dma_ops_domain *dma_ops_do
 	if (protection_domain_init(&dma_dom->domain))
 		goto free_dma_dom;
 
+	dma_dom->next_index = alloc_percpu(u32);
+	if (!dma_dom->next_index)
+		goto free_dma_dom;
+
 	dma_dom->domain.mode = PAGE_MODE_2_LEVEL;
 	dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
 	dma_dom->domain.flags = PD_DMA_OPS_MASK;
@@ -2054,8 +2072,9 @@ static struct dma_ops_domain *dma_ops_do
 	 * a valid dma-address. So we can use 0 as error value
 	 */
 	dma_dom->aperture[0]->bitmap[0] = 1;
-	dma_dom->next_index = 0;
 
+	for_each_possible_cpu(cpu)
+		*per_cpu_ptr(dma_dom->next_index, cpu) = 0;
 
 	return dma_dom;
 
