From fa87e62d357ccf73831cd52ed316593d0968020d Mon Sep 17 00:00:00 2001
From: Dmitry Cherkasov <dcherkassov@gmail.com>
Date: Mon, 17 Sep 2012 19:36:19 +0200
Subject: drm/radeon: add 2-level VM pagetables support v9
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Git-commit: fa87e62d357ccf73831cd52ed316593d0968020d
Patch-mainline: v3.7-rc1

PDE/PTE update code uses CP ring for memory writes.
All page table entries are preallocated for now in alloc_pt().

It is made as whole because it's hard to divide it to several patches
that compile and doesn't break anything being applied separately.

Tested on cayman card.

v2: rebased on top of "refactor set_page chipset interface v3",
    code cleanups

v3: switched offsets calc macros to inline funcs where possible,
    remove pd_addr from radeon_vm, switched RADEON_BLOCK_SIZE define,
    to 9 (and PTE_COUNT to 1 << BLOCK_SIZE)

v4 (ck): move "incr" documentation to previous patch, cleanup and
         document RADEON_VM_* constants, change commit message to
         our usual format, simplify patch allot by removing
         everything current not necessary, disable SI workaround.

v5: (agd5f): Fix typo in tables_size calculation in
             radeon_vm_alloc_pt().  Second line should have been
             '+=' rather than '='.

v6: fix npdes calculation. In scenario when pfns to be mapped overlap
two PDE spans:

   +-----------+-------------+
   | PDE span  | PDE span    |
   +-----------+----+--------+
          |         |
          +---------+
          | pfns    |
          +---------+

the following npdes calculation gives incorrect result:

npdes = (nptes >> RADEON_VM_BLOCK_SIZE) + 1;

For the case above picture it should give npdes = 2, but gives one.

This patch corrects it by rounding last pfn up to 512 border,
first - down to 512 border and then subtracting and dividing by 512.

v7: Make npde calculation clearer, fix ndw calculation.

v8: (agd5f): reserve enough for 2 full VM PTs, add some
             additional comments.

v9: fix typo in npde calculation

Signed-off-by: Dmitry Cherkasov <Dmitrii.Cherkasov@amd.com>
Signed-off-by: Christian König <deathsimple@vodafone.de>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
Acked-by: Michal Srb <msrb@suse.com>
---
 drivers/gpu/drm/radeon/ni.c          |    4 +-
 drivers/gpu/drm/radeon/radeon.h      |   12 ++++-
 drivers/gpu/drm/radeon/radeon_gart.c |   91 +++++++++++++++++++++++++---------
 drivers/gpu/drm/radeon/si.c          |    4 +-
 4 files changed, 81 insertions(+), 30 deletions(-)

Index: linux-3.0-SLE11-SP3/drivers/gpu/drm/radeon/ni.c
===================================================================
--- linux-3.0-SLE11-SP3.orig/drivers/gpu/drm/radeon/ni.c
+++ linux-3.0-SLE11-SP3/drivers/gpu/drm/radeon/ni.c
@@ -782,7 +782,7 @@ static int cayman_pcie_gart_enable(struc
 	       (u32)(rdev->dummy_page.addr >> 12));
 	WREG32(VM_CONTEXT1_CNTL2, 0);
 	WREG32(VM_CONTEXT1_CNTL, 0);
-	WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
+	WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
 				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
 
 	cayman_pcie_gart_tlb_flush(rdev);
@@ -1580,7 +1580,7 @@ void cayman_vm_flush(struct radeon_devic
 	radeon_ring_write(ring, vm->last_pfn);
 
 	radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2), 0));
-	radeon_ring_write(ring, vm->pt_gpu_addr >> 12);
+	radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
 
 	/* flush hdp cache */
 	radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
Index: linux-3.0-SLE11-SP3/drivers/gpu/drm/radeon/radeon.h
===================================================================
--- linux-3.0-SLE11-SP3.orig/drivers/gpu/drm/radeon/radeon.h
+++ linux-3.0-SLE11-SP3/drivers/gpu/drm/radeon/radeon.h
@@ -648,15 +648,23 @@ struct radeon_ring {
  * VM
  */
 
+/* maximum number of VMIDs */
 #define RADEON_NUM_VM	16
 
+/* defines number of bits in page table versus page directory,
+ * a page is 4KB so we have 12 bits offset, 9 bits in the page
+ * table and the remaining 19 bits are in the page directory */
+#define RADEON_VM_BLOCK_SIZE   9
+
+/* number of entries in page table */
+#define RADEON_VM_PTE_COUNT (1 << RADEON_VM_BLOCK_SIZE)
+
 struct radeon_vm {
 	struct list_head		list;
 	struct list_head		va;
 	unsigned			id;
 	unsigned			last_pfn;
-	u64				pt_gpu_addr;
-	u64				*pt;
+	u64				pd_gpu_addr;
 	struct radeon_sa_bo		*sa_bo;
 	struct mutex			mutex;
 	/* last fence for cs using this vm */
Index: linux-3.0-SLE11-SP3/drivers/gpu/drm/radeon/radeon_gart.c
===================================================================
--- linux-3.0-SLE11-SP3.orig/drivers/gpu/drm/radeon/radeon_gart.c
+++ linux-3.0-SLE11-SP3/drivers/gpu/drm/radeon/radeon_gart.c
@@ -423,6 +423,18 @@ void radeon_gart_fini(struct radeon_devi
  */
 
 /**
+ * radeon_vm_directory_size - returns the size of the page directory in bytes
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Calculate the size of the page directory in bytes (cayman+).
+ */
+static unsigned radeon_vm_directory_size(struct radeon_device *rdev)
+{
+	return (rdev->vm_manager.max_pfn >> RADEON_VM_BLOCK_SIZE) * 8;
+}
+
+/**
  * radeon_vm_manager_init - init the vm manager
  *
  * @rdev: radeon_device pointer
@@ -435,10 +447,14 @@ int radeon_vm_manager_init(struct radeon
 	struct radeon_vm *vm;
 	struct radeon_bo_va *bo_va;
 	int r;
+	unsigned size;
 
 	if (!rdev->vm_manager.enabled) {
+		size = RADEON_GPU_PAGE_ALIGN(radeon_vm_directory_size(rdev));
+		size += RADEON_GPU_PAGE_ALIGN(rdev->vm_manager.max_pfn * 8);
+		size *= 2;
 		r = radeon_sa_bo_manager_init(rdev, &rdev->vm_manager.sa_manager,
-					      rdev->vm_manager.max_pfn * 8,
+					      size,
 					      RADEON_GEM_DOMAIN_VRAM);
 		if (r) {
 			dev_err(rdev->dev, "failed to allocate vm bo (%dKB)\n",
@@ -489,7 +505,6 @@ static void radeon_vm_free_pt(struct rad
 
 	list_del_init(&vm->list);
 	radeon_sa_bo_free(rdev, &vm->sa_bo, vm->fence);
-	vm->pt = NULL;
 
 	list_for_each_entry(bo_va, &vm->va, vm_list) {
 		bo_va->valid = false;
@@ -545,11 +560,17 @@ int radeon_vm_alloc_pt(struct radeon_dev
 {
 	struct radeon_vm *vm_evict;
 	int r;
+	u64 *pd_addr;
+	int tables_size;
 
 	if (vm == NULL) {
 		return -EINVAL;
 	}
 
+	/* allocate enough to cover the current VM size */
+	tables_size = RADEON_GPU_PAGE_ALIGN(radeon_vm_directory_size(rdev));
+	tables_size += RADEON_GPU_PAGE_ALIGN(vm->last_pfn * 8);
+
 	if (vm->sa_bo != NULL) {
 		/* update lru */
 		list_del_init(&vm->list);
@@ -559,8 +580,7 @@ int radeon_vm_alloc_pt(struct radeon_dev
 
 retry:
 	r = radeon_sa_bo_new(rdev, &rdev->vm_manager.sa_manager, &vm->sa_bo,
-			     RADEON_GPU_PAGE_ALIGN(vm->last_pfn * 8),
-			     RADEON_GPU_PAGE_SIZE, false);
+			     tables_size, RADEON_GPU_PAGE_SIZE, false);
 	if (r == -ENOMEM) {
 		if (list_empty(&rdev->vm_manager.lru_vm)) {
 			return r;
@@ -575,9 +595,9 @@ retry:
 		return r;
 	}
 
-	vm->pt = radeon_sa_bo_cpu_addr(vm->sa_bo);
-	vm->pt_gpu_addr = radeon_sa_bo_gpu_addr(vm->sa_bo);
-	memset(vm->pt, 0, RADEON_GPU_PAGE_ALIGN(vm->last_pfn * 8));
+	pd_addr = radeon_sa_bo_cpu_addr(vm->sa_bo);
+	vm->pd_gpu_addr = radeon_sa_bo_gpu_addr(vm->sa_bo);
+	memset(pd_addr, 0, tables_size);
 
 	list_add_tail(&vm->list, &rdev->vm_manager.lru_vm);
 	return radeon_vm_bo_update_pte(rdev, vm, rdev->ring_tmp_bo.bo,
@@ -865,8 +885,9 @@ int radeon_vm_bo_update_pte(struct radeo
 	struct radeon_ring *ring = &rdev->ring[ridx];
 	struct radeon_semaphore *sem = NULL;
 	struct radeon_bo_va *bo_va;
-	unsigned ngpu_pages, ndw;
-	uint64_t pfn, addr;
+	unsigned nptes, npdes, ndw;
+	uint64_t pe, addr;
+	uint64_t pfn;
 	int r;
 
 	/* nothing to do if vm isn't bound */
@@ -888,10 +909,8 @@ int radeon_vm_bo_update_pte(struct radeo
 	if ((bo_va->valid && mem) || (!bo_va->valid && mem == NULL))
 		return 0;
 
-	ngpu_pages = radeon_bo_ngpu_pages(bo);
 	bo_va->flags &= ~RADEON_VM_PAGE_VALID;
 	bo_va->flags &= ~RADEON_VM_PAGE_SYSTEM;
-	pfn = bo_va->soffset / RADEON_GPU_PAGE_SIZE;
 	if (mem) {
 		addr = mem->start << PAGE_SHIFT;
 		if (mem->mem_type != TTM_PL_SYSTEM) {
@@ -920,9 +939,26 @@ int radeon_vm_bo_update_pte(struct radeo
 	}
 
 	/* estimate number of dw needed */
+	/* reserve space for 32-bit padding */
 	ndw = 32;
-	ndw += (ngpu_pages >> 12) * 3;
-	ndw += ngpu_pages * 2;
+
+	nptes = radeon_bo_ngpu_pages(bo);
+
+	pfn = (bo_va->soffset / RADEON_GPU_PAGE_SIZE);
+
+	/* handle cases where a bo spans several pdes  */
+	npdes = (ALIGN(pfn + nptes, RADEON_VM_PTE_COUNT) -
+		 (pfn & ~(RADEON_VM_PTE_COUNT - 1))) >> RADEON_VM_BLOCK_SIZE;
+
+	/* reserve space for one header for every 2k dwords */
+	ndw += (nptes >> 11) * 3;
+	/* reserve space for pte addresses */
+	ndw += nptes * 2;
+
+	/* reserve space for one header for every 2k dwords */
+	ndw += (npdes >> 11) * 3;
+	/* reserve space for pde addresses */
+	ndw += npdes * 2;
 
 	r = radeon_ring_lock(rdev, ring, ndw);
 	if (r) {
@@ -934,8 +970,22 @@ int radeon_vm_bo_update_pte(struct radeo
 		radeon_fence_note_sync(vm->fence, ridx);
 	}
 
-	radeon_asic_vm_set_page(rdev, vm->pt_gpu_addr + pfn * 8, addr,
-				ngpu_pages, RADEON_GPU_PAGE_SIZE, bo_va->flags);
+	/* update page table entries */
+	pe = vm->pd_gpu_addr;
+	pe += radeon_vm_directory_size(rdev);
+	pe += (bo_va->soffset / RADEON_GPU_PAGE_SIZE) * 8;
+
+	radeon_asic_vm_set_page(rdev, pe, addr, nptes,
+				RADEON_GPU_PAGE_SIZE, bo_va->flags);
+
+	/* update page directory entries */
+	addr = pe;
+
+	pe = vm->pd_gpu_addr;
+	pe += ((bo_va->soffset / RADEON_GPU_PAGE_SIZE) >> RADEON_VM_BLOCK_SIZE) * 8;
+
+	radeon_asic_vm_set_page(rdev, pe, addr, npdes,
+				RADEON_VM_PTE_COUNT * 8, RADEON_VM_PAGE_VALID);
 
 	radeon_fence_unref(&vm->fence);
 	r = radeon_fence_emit(rdev, &vm->fence, ridx);
@@ -1017,18 +1067,11 @@ int radeon_vm_init(struct radeon_device
 
 	vm->id = 0;
 	vm->fence = NULL;
+	vm->last_pfn = 0;
 	mutex_init(&vm->mutex);
 	INIT_LIST_HEAD(&vm->list);
 	INIT_LIST_HEAD(&vm->va);
-	/* SI requires equal sized PTs for all VMs, so always set
-	 * last_pfn to max_pfn.  cayman allows variable sized
-	 * pts so we can grow then as needed.  Once we switch
-	 * to two level pts we can unify this again.
-	 */
-	if (rdev->family >= CHIP_TAHITI)
-		vm->last_pfn = rdev->vm_manager.max_pfn;
-	else
-		vm->last_pfn = 0;
+
 	/* map the ib pool buffer at 0 in virtual address space, set
 	 * read only
 	 */
Index: linux-3.0-SLE11-SP3/drivers/gpu/drm/radeon/si.c
===================================================================
--- linux-3.0-SLE11-SP3.orig/drivers/gpu/drm/radeon/si.c
+++ linux-3.0-SLE11-SP3/drivers/gpu/drm/radeon/si.c
@@ -2426,7 +2426,7 @@ static int si_pcie_gart_enable(struct ra
 	WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
 	       (u32)(rdev->dummy_page.addr >> 12));
 	WREG32(VM_CONTEXT1_CNTL2, 0);
-	WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
+	WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
 				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
 
 	si_pcie_gart_tlb_flush(rdev);
@@ -2804,7 +2804,7 @@ void si_vm_flush(struct radeon_device *r
 		radeon_ring_write(ring, PACKET0(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR
 						+ ((vm->id - 8) << 2), 0));
 	}
-	radeon_ring_write(ring, vm->pt_gpu_addr >> 12);
+	radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
 
 	/* flush hdp cache */
 	radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
