GPUCORE-42157, MIDCET-5076: Check if new PGDs require more pages
Check if a new PGD required allocating more pages to the memory
pool.
Bug: 315967882
Provenance: https://code.ipdelivery.arm.com/c/GPU/mali-ddk/+/6751
Change-Id: I9881b2ba342e59487fe93dcf2d00f09af4d69ed8
Merged-In: I9881b2ba342e59487fe93dcf2d00f09af4d69ed8
Signed-off-by: Michael Stokes <mjstokes@google.com>
diff --git a/mali_kbase/mmu/mali_kbase_mmu.c b/mali_kbase/mmu/mali_kbase_mmu.c
index 363aea9..c19bcf4 100644
--- a/mali_kbase/mmu/mali_kbase_mmu.c
+++ b/mali_kbase/mmu/mali_kbase_mmu.c
@@ -1915,6 +1915,8 @@
* @level_high: The higher bound for the levels for which the PGD allocs are required
* @new_pgds: Ptr to an array (size MIDGARD_MMU_BOTTOMLEVEL+1) to write the
* newly allocated PGD addresses to.
+ * @pool_grown: True if new PGDs required the memory pool to grow to allocate more pages,
+ * or false otherwise
*
* Numerically, level_low < level_high, not to be confused with top level and
* bottom level concepts for MMU PGDs. They are only used as low and high bounds
@@ -1925,14 +1927,18 @@
* * -ENOMEM - allocation failed for a PGD.
*/
static int mmu_insert_alloc_pgds(struct kbase_device *kbdev, struct kbase_mmu_table *mmut,
- phys_addr_t *new_pgds, int level_low, int level_high)
+ phys_addr_t *new_pgds, int level_low, int level_high,
+ bool *pool_grown)
{
int err = 0;
int i;
lockdep_assert_held(&mmut->mmu_lock);
+ *pool_grown = false;
for (i = level_low; i <= level_high; i++) {
+ if (new_pgds[i] != KBASE_INVALID_PHYSICAL_ADDRESS)
+ continue;
do {
new_pgds[i] = kbase_mmu_alloc_pgd(kbdev, mmut);
if (new_pgds[i] != KBASE_INVALID_PHYSICAL_ADDRESS)
@@ -1945,17 +1951,9 @@
if (err) {
dev_err(kbdev->dev, "%s: kbase_mem_pool_grow() returned error %d",
__func__, err);
-
- /* Free all PGDs allocated in previous successful iterations
- * from (i-1) to level_low
- */
- for (i = (i - 1); i >= level_low; i--) {
- if (new_pgds[i] != KBASE_INVALID_PHYSICAL_ADDRESS)
- kbase_mmu_free_pgd(kbdev, mmut, new_pgds[i]);
- }
-
return err;
}
+ *pool_grown = true;
} while (1);
}
@@ -1985,6 +1983,8 @@
if (WARN_ON(kctx == NULL))
return -EINVAL;
+ lockdep_assert_held(&kctx->reg_lock);
+
/* 64-bit address range is the max */
KBASE_DEBUG_ASSERT(start_vpfn <= (U64_MAX / PAGE_SIZE));
@@ -2020,6 +2020,7 @@
struct page *p;
register unsigned int num_of_valid_entries;
bool newly_created_pgd = false;
+ bool pool_grown;
if (count > remain)
count = remain;
@@ -2027,6 +2028,10 @@
cur_level = MIDGARD_MMU_BOTTOMLEVEL;
insert_level = cur_level;
+ for (l = MIDGARD_MMU_TOPLEVEL + 1; l <= cur_level; l++)
+ new_pgds[l] = KBASE_INVALID_PHYSICAL_ADDRESS;
+
+repeat_page_table_walk:
/*
* Repeatedly calling mmu_get_lowest_valid_pgd() is clearly
* suboptimal. We don't have to re-parse the whole tree
@@ -2041,7 +2046,7 @@
if (err) {
dev_err(kbdev->dev, "%s: mmu_get_lowest_valid_pgd() returned error %d",
__func__, err);
- goto fail_unlock;
+ goto fail_unlock_free_pgds;
}
/* No valid pgd at cur_level */
@@ -2050,9 +2055,12 @@
* down to the lowest valid pgd at insert_level
*/
err = mmu_insert_alloc_pgds(kbdev, mmut, new_pgds, (insert_level + 1),
- cur_level);
+ cur_level, &pool_grown);
if (err)
- goto fail_unlock;
+ goto fail_unlock_free_pgds;
+
+ if (pool_grown)
+ goto repeat_page_table_walk;
newly_created_pgd = true;
@@ -2138,9 +2146,9 @@
fail_unlock_free_pgds:
/* Free the pgds allocated by us from insert_level+1 to bottom level */
for (l = cur_level; l > insert_level; l--)
- kbase_mmu_free_pgd(kbdev, mmut, new_pgds[l]);
+ if (new_pgds[l] != KBASE_INVALID_PHYSICAL_ADDRESS)
+ kbase_mmu_free_pgd(kbdev, mmut, new_pgds[l]);
-fail_unlock:
if (insert_vpfn != (start_vpfn * GPU_PAGES_PER_CPU_PAGE)) {
/* Invalidate the pages we have partially completed */
mmu_insert_pages_failure_recovery(kbdev, mmut, start_vpfn * GPU_PAGES_PER_CPU_PAGE,
@@ -2286,6 +2294,9 @@
int l, cur_level, insert_level;
struct tagged_addr *start_phys = phys;
+ if (mmut->kctx)
+ lockdep_assert_held(&mmut->kctx->reg_lock);
+
/* Note that 0 is a valid start_vpfn */
/* 64-bit address range is the max */
KBASE_DEBUG_ASSERT(start_vpfn <= (U64_MAX / PAGE_SIZE));
@@ -2308,6 +2319,7 @@
register unsigned int num_of_valid_entries;
bool newly_created_pgd = false;
enum kbase_mmu_op_type flush_op;
+ bool pool_grown;
if (count > remain)
count = remain;
@@ -2319,6 +2331,10 @@
insert_level = cur_level;
+ for (l = MIDGARD_MMU_TOPLEVEL + 1; l <= cur_level; l++)
+ new_pgds[l] = KBASE_INVALID_PHYSICAL_ADDRESS;
+
+repeat_page_table_walk:
/*
* Repeatedly calling mmu_get_lowest_valid_pgd() is clearly
* suboptimal. We don't have to re-parse the whole tree
@@ -2333,7 +2349,7 @@
if (err) {
dev_err(kbdev->dev, "%s: mmu_get_lowest_valid_pgd() returned error %d",
__func__, err);
- goto fail_unlock;
+ goto fail_unlock_free_pgds;
}
/* No valid pgd at cur_level */
@@ -2342,9 +2358,12 @@
* down to the lowest valid pgd at insert_level
*/
err = mmu_insert_alloc_pgds(kbdev, mmut, new_pgds, (insert_level + 1),
- cur_level);
+ cur_level, &pool_grown);
if (err)
- goto fail_unlock;
+ goto fail_unlock_free_pgds;
+
+ if (pool_grown)
+ goto repeat_page_table_walk;
newly_created_pgd = true;
@@ -2458,9 +2477,9 @@
fail_unlock_free_pgds:
/* Free the pgds allocated by us from insert_level+1 to bottom level */
for (l = cur_level; l > insert_level; l--)
- kbase_mmu_free_pgd(kbdev, mmut, new_pgds[l]);
+ if (new_pgds[l] != KBASE_INVALID_PHYSICAL_ADDRESS)
+ kbase_mmu_free_pgd(kbdev, mmut, new_pgds[l]);
-fail_unlock:
if (insert_vpfn != (start_vpfn * GPU_PAGES_PER_CPU_PAGE)) {
/* Invalidate the pages we have partially completed */
mmu_insert_pages_failure_recovery(kbdev, mmut, start_vpfn * GPU_PAGES_PER_CPU_PAGE,