| // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note |
| /* |
| * |
| * (C) COPYRIGHT 2010-2022 ARM Limited. All rights reserved. |
| * |
| * This program is free software and is provided to you under the terms of the |
| * GNU General Public License version 2 as published by the Free Software |
| * Foundation, and any use by you of this program is subject to the terms |
| * of such GNU license. |
| * |
| * This program is distributed in the hope that it will be useful, |
| * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| * GNU General Public License for more details. |
| * |
| * You should have received a copy of the GNU General Public License |
| * along with this program; if not, you can access it online at |
| * http://www.gnu.org/licenses/gpl-2.0.html. |
| * |
| */ |
| |
| /** |
| * Base kernel memory APIs |
| */ |
| #include <linux/dma-buf.h> |
| #include <linux/kernel.h> |
| #include <linux/bug.h> |
| #include <linux/compat.h> |
| #include <linux/version.h> |
| #include <linux/log2.h> |
| #if IS_ENABLED(CONFIG_OF) |
| #include <linux/of_platform.h> |
| #endif |
| |
| #include <mali_kbase_config.h> |
| #include <mali_kbase.h> |
| #include <gpu/mali_kbase_gpu_regmap.h> |
| #include <mali_kbase_cache_policy.h> |
| #include <mali_kbase_hw.h> |
| #include <tl/mali_kbase_tracepoints.h> |
| #include <mali_kbase_native_mgm.h> |
| #include <mali_kbase_mem_pool_group.h> |
| #include <mmu/mali_kbase_mmu.h> |
| #include <mali_kbase_config_defaults.h> |
| #include <mali_kbase_trace_gpu_mem.h> |
| |
| /* |
| * Alignment of objects allocated by the GPU inside a just-in-time memory |
| * region whose size is given by an end address |
| * |
| * This is the alignment of objects allocated by the GPU, but possibly not |
| * fully written to. When taken into account with |
| * KBASE_GPU_ALLOCATED_OBJECT_MAX_BYTES it gives the maximum number of bytes |
| * that the JIT memory report size can exceed the actual backed memory size. |
| */ |
| #define KBASE_GPU_ALLOCATED_OBJECT_ALIGN_BYTES (128u) |
| |
| /* |
| * Maximum size of objects allocated by the GPU inside a just-in-time memory |
| * region whose size is given by an end address |
| * |
| * This is the maximum size of objects allocated by the GPU, but possibly not |
| * fully written to. When taken into account with |
| * KBASE_GPU_ALLOCATED_OBJECT_ALIGN_BYTES it gives the maximum number of bytes |
| * that the JIT memory report size can exceed the actual backed memory size. |
| */ |
| #define KBASE_GPU_ALLOCATED_OBJECT_MAX_BYTES (512u) |
| |
| |
| /* Forward declarations */ |
| static void free_partial_locked(struct kbase_context *kctx, |
| struct kbase_mem_pool *pool, struct tagged_addr tp); |
| |
| static size_t kbase_get_num_cpu_va_bits(struct kbase_context *kctx) |
| { |
| #if defined(CONFIG_ARM64) |
| /* VA_BITS can be as high as 48 bits, but all bits are available for |
| * both user and kernel. |
| */ |
| size_t cpu_va_bits = VA_BITS; |
| #elif defined(CONFIG_X86_64) |
| /* x86_64 can access 48 bits of VA, but the 48th is used to denote |
| * kernel (1) vs userspace (0), so the max here is 47. |
| */ |
| size_t cpu_va_bits = 47; |
| #elif defined(CONFIG_ARM) || defined(CONFIG_X86_32) |
| size_t cpu_va_bits = sizeof(void *) * BITS_PER_BYTE; |
| #else |
| #error "Unknown CPU VA width for this architecture" |
| #endif |
| |
| #if IS_ENABLED(CONFIG_64BIT) |
| if (kbase_ctx_flag(kctx, KCTX_COMPAT)) |
| cpu_va_bits = 32; |
| #endif |
| |
| return cpu_va_bits; |
| } |
| |
| /* This function finds out which RB tree the given pfn from the GPU VA belongs |
| * to based on the memory zone the pfn refers to |
| */ |
| static struct rb_root *kbase_gpu_va_to_rbtree(struct kbase_context *kctx, |
| u64 gpu_pfn) |
| { |
| struct rb_root *rbtree = NULL; |
| struct kbase_reg_zone *exec_va_zone = |
| kbase_ctx_reg_zone_get(kctx, KBASE_REG_ZONE_EXEC_VA); |
| |
| /* The gpu_pfn can only be greater than the starting pfn of the EXEC_VA |
| * zone if this has been initialized. |
| */ |
| if (gpu_pfn >= exec_va_zone->base_pfn) |
| rbtree = &kctx->reg_rbtree_exec; |
| else { |
| u64 same_va_end; |
| |
| #if IS_ENABLED(CONFIG_64BIT) |
| if (kbase_ctx_flag(kctx, KCTX_COMPAT)) { |
| #endif /* CONFIG_64BIT */ |
| same_va_end = KBASE_REG_ZONE_CUSTOM_VA_BASE; |
| #if IS_ENABLED(CONFIG_64BIT) |
| } else { |
| struct kbase_reg_zone *same_va_zone = |
| kbase_ctx_reg_zone_get(kctx, |
| KBASE_REG_ZONE_SAME_VA); |
| same_va_end = kbase_reg_zone_end_pfn(same_va_zone); |
| } |
| #endif /* CONFIG_64BIT */ |
| |
| if (gpu_pfn >= same_va_end) |
| rbtree = &kctx->reg_rbtree_custom; |
| else |
| rbtree = &kctx->reg_rbtree_same; |
| } |
| |
| return rbtree; |
| } |
| |
| /* This function inserts a region into the tree. */ |
| static void kbase_region_tracker_insert(struct kbase_va_region *new_reg) |
| { |
| u64 start_pfn = new_reg->start_pfn; |
| struct rb_node **link = NULL; |
| struct rb_node *parent = NULL; |
| struct rb_root *rbtree = NULL; |
| |
| rbtree = new_reg->rbtree; |
| |
| link = &(rbtree->rb_node); |
| /* Find the right place in the tree using tree search */ |
| while (*link) { |
| struct kbase_va_region *old_reg; |
| |
| parent = *link; |
| old_reg = rb_entry(parent, struct kbase_va_region, rblink); |
| |
| /* RBTree requires no duplicate entries. */ |
| KBASE_DEBUG_ASSERT(old_reg->start_pfn != start_pfn); |
| |
| if (old_reg->start_pfn > start_pfn) |
| link = &(*link)->rb_left; |
| else |
| link = &(*link)->rb_right; |
| } |
| |
| /* Put the new node there, and rebalance tree */ |
| rb_link_node(&(new_reg->rblink), parent, link); |
| |
| rb_insert_color(&(new_reg->rblink), rbtree); |
| } |
| |
| static struct kbase_va_region *find_region_enclosing_range_rbtree( |
| struct rb_root *rbtree, u64 start_pfn, size_t nr_pages) |
| { |
| struct rb_node *rbnode; |
| struct kbase_va_region *reg; |
| u64 end_pfn = start_pfn + nr_pages; |
| |
| rbnode = rbtree->rb_node; |
| |
| while (rbnode) { |
| u64 tmp_start_pfn, tmp_end_pfn; |
| |
| reg = rb_entry(rbnode, struct kbase_va_region, rblink); |
| tmp_start_pfn = reg->start_pfn; |
| tmp_end_pfn = reg->start_pfn + reg->nr_pages; |
| |
| /* If start is lower than this, go left. */ |
| if (start_pfn < tmp_start_pfn) |
| rbnode = rbnode->rb_left; |
| /* If end is higher than this, then go right. */ |
| else if (end_pfn > tmp_end_pfn) |
| rbnode = rbnode->rb_right; |
| else /* Enclosing */ |
| return reg; |
| } |
| |
| return NULL; |
| } |
| |
| struct kbase_va_region *kbase_find_region_enclosing_address( |
| struct rb_root *rbtree, u64 gpu_addr) |
| { |
| u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; |
| struct rb_node *rbnode; |
| struct kbase_va_region *reg; |
| |
| rbnode = rbtree->rb_node; |
| |
| while (rbnode) { |
| u64 tmp_start_pfn, tmp_end_pfn; |
| |
| reg = rb_entry(rbnode, struct kbase_va_region, rblink); |
| tmp_start_pfn = reg->start_pfn; |
| tmp_end_pfn = reg->start_pfn + reg->nr_pages; |
| |
| /* If start is lower than this, go left. */ |
| if (gpu_pfn < tmp_start_pfn) |
| rbnode = rbnode->rb_left; |
| /* If end is higher than this, then go right. */ |
| else if (gpu_pfn >= tmp_end_pfn) |
| rbnode = rbnode->rb_right; |
| else /* Enclosing */ |
| return reg; |
| } |
| |
| return NULL; |
| } |
| |
| /* Find region enclosing given address. */ |
| struct kbase_va_region *kbase_region_tracker_find_region_enclosing_address( |
| struct kbase_context *kctx, u64 gpu_addr) |
| { |
| u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; |
| struct rb_root *rbtree = NULL; |
| |
| KBASE_DEBUG_ASSERT(kctx != NULL); |
| |
| lockdep_assert_held(&kctx->reg_lock); |
| |
| rbtree = kbase_gpu_va_to_rbtree(kctx, gpu_pfn); |
| |
| return kbase_find_region_enclosing_address(rbtree, gpu_addr); |
| } |
| |
| KBASE_EXPORT_TEST_API(kbase_region_tracker_find_region_enclosing_address); |
| |
| struct kbase_va_region *kbase_find_region_base_address( |
| struct rb_root *rbtree, u64 gpu_addr) |
| { |
| u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; |
| struct rb_node *rbnode = NULL; |
| struct kbase_va_region *reg = NULL; |
| |
| rbnode = rbtree->rb_node; |
| |
| while (rbnode) { |
| reg = rb_entry(rbnode, struct kbase_va_region, rblink); |
| if (reg->start_pfn > gpu_pfn) |
| rbnode = rbnode->rb_left; |
| else if (reg->start_pfn < gpu_pfn) |
| rbnode = rbnode->rb_right; |
| else |
| return reg; |
| } |
| |
| return NULL; |
| } |
| |
| /* Find region with given base address */ |
| struct kbase_va_region *kbase_region_tracker_find_region_base_address( |
| struct kbase_context *kctx, u64 gpu_addr) |
| { |
| u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; |
| struct rb_root *rbtree = NULL; |
| |
| lockdep_assert_held(&kctx->reg_lock); |
| |
| rbtree = kbase_gpu_va_to_rbtree(kctx, gpu_pfn); |
| |
| return kbase_find_region_base_address(rbtree, gpu_addr); |
| } |
| |
| KBASE_EXPORT_TEST_API(kbase_region_tracker_find_region_base_address); |
| |
| /* Find region meeting given requirements */ |
| static struct kbase_va_region *kbase_region_tracker_find_region_meeting_reqs( |
| struct kbase_va_region *reg_reqs, |
| size_t nr_pages, size_t align_offset, size_t align_mask, |
| u64 *out_start_pfn) |
| { |
| struct rb_node *rbnode = NULL; |
| struct kbase_va_region *reg = NULL; |
| struct rb_root *rbtree = NULL; |
| |
| /* Note that this search is a linear search, as we do not have a target |
| * address in mind, so does not benefit from the rbtree search |
| */ |
| rbtree = reg_reqs->rbtree; |
| |
| for (rbnode = rb_first(rbtree); rbnode; rbnode = rb_next(rbnode)) { |
| reg = rb_entry(rbnode, struct kbase_va_region, rblink); |
| if ((reg->nr_pages >= nr_pages) && |
| (reg->flags & KBASE_REG_FREE)) { |
| /* Check alignment */ |
| u64 start_pfn = reg->start_pfn; |
| |
| /* When align_offset == align, this sequence is |
| * equivalent to: |
| * (start_pfn + align_mask) & ~(align_mask) |
| * |
| * Otherwise, it aligns to n*align + offset, for the |
| * lowest value n that makes this still >start_pfn |
| */ |
| start_pfn += align_mask; |
| start_pfn -= (start_pfn - align_offset) & (align_mask); |
| |
| if (!(reg_reqs->flags & KBASE_REG_GPU_NX)) { |
| /* Can't end at 4GB boundary */ |
| if (0 == ((start_pfn + nr_pages) & BASE_MEM_PFN_MASK_4GB)) |
| start_pfn += align_offset; |
| |
| /* Can't start at 4GB boundary */ |
| if (0 == (start_pfn & BASE_MEM_PFN_MASK_4GB)) |
| start_pfn += align_offset; |
| |
| if (!((start_pfn + nr_pages) & BASE_MEM_PFN_MASK_4GB) || |
| !(start_pfn & BASE_MEM_PFN_MASK_4GB)) |
| continue; |
| } else if (reg_reqs->flags & |
| KBASE_REG_GPU_VA_SAME_4GB_PAGE) { |
| u64 end_pfn = start_pfn + nr_pages - 1; |
| |
| if ((start_pfn & ~BASE_MEM_PFN_MASK_4GB) != |
| (end_pfn & ~BASE_MEM_PFN_MASK_4GB)) |
| start_pfn = end_pfn & ~BASE_MEM_PFN_MASK_4GB; |
| } |
| |
| if ((start_pfn >= reg->start_pfn) && |
| (start_pfn <= (reg->start_pfn + reg->nr_pages - 1)) && |
| ((start_pfn + nr_pages - 1) <= (reg->start_pfn + reg->nr_pages - 1))) { |
| *out_start_pfn = start_pfn; |
| return reg; |
| } |
| } |
| } |
| |
| return NULL; |
| } |
| |
| /** |
| * Remove a region object from the global list. |
| * @reg: Region object to remove |
| * |
| * The region reg is removed, possibly by merging with other free and |
| * compatible adjacent regions. It must be called with the context |
| * region lock held. The associated memory is not released (see |
| * kbase_free_alloced_region). Internal use only. |
| */ |
| int kbase_remove_va_region(struct kbase_va_region *reg) |
| { |
| struct rb_node *rbprev; |
| struct kbase_va_region *prev = NULL; |
| struct rb_node *rbnext; |
| struct kbase_va_region *next = NULL; |
| struct rb_root *reg_rbtree = NULL; |
| |
| int merged_front = 0; |
| int merged_back = 0; |
| int err = 0; |
| |
| reg_rbtree = reg->rbtree; |
| |
| /* Try to merge with the previous block first */ |
| rbprev = rb_prev(&(reg->rblink)); |
| if (rbprev) { |
| prev = rb_entry(rbprev, struct kbase_va_region, rblink); |
| if (prev->flags & KBASE_REG_FREE) { |
| /* We're compatible with the previous VMA, merge with |
| * it |
| */ |
| WARN_ON((prev->flags & KBASE_REG_ZONE_MASK) != |
| (reg->flags & KBASE_REG_ZONE_MASK)); |
| prev->nr_pages += reg->nr_pages; |
| rb_erase(&(reg->rblink), reg_rbtree); |
| reg = prev; |
| merged_front = 1; |
| } |
| } |
| |
| /* Try to merge with the next block second */ |
| /* Note we do the lookup here as the tree may have been rebalanced. */ |
| rbnext = rb_next(&(reg->rblink)); |
| if (rbnext) { |
| /* We're compatible with the next VMA, merge with it */ |
| next = rb_entry(rbnext, struct kbase_va_region, rblink); |
| if (next->flags & KBASE_REG_FREE) { |
| WARN_ON((next->flags & KBASE_REG_ZONE_MASK) != |
| (reg->flags & KBASE_REG_ZONE_MASK)); |
| next->start_pfn = reg->start_pfn; |
| next->nr_pages += reg->nr_pages; |
| rb_erase(&(reg->rblink), reg_rbtree); |
| merged_back = 1; |
| if (merged_front) { |
| /* We already merged with prev, free it */ |
| kfree(reg); |
| } |
| } |
| } |
| |
| /* If we failed to merge then we need to add a new block */ |
| if (!(merged_front || merged_back)) { |
| /* |
| * We didn't merge anything. Add a new free |
| * placeholder and remove the original one. |
| */ |
| struct kbase_va_region *free_reg; |
| |
| free_reg = kbase_alloc_free_region(reg_rbtree, |
| reg->start_pfn, reg->nr_pages, |
| reg->flags & KBASE_REG_ZONE_MASK); |
| if (!free_reg) { |
| err = -ENOMEM; |
| goto out; |
| } |
| rb_replace_node(&(reg->rblink), &(free_reg->rblink), reg_rbtree); |
| } |
| |
| out: |
| return err; |
| } |
| |
| KBASE_EXPORT_TEST_API(kbase_remove_va_region); |
| |
| /** |
| * kbase_insert_va_region_nolock - Insert a VA region to the list, |
| * replacing the existing one. |
| * |
| * @new_reg: The new region to insert |
| * @at_reg: The region to replace |
| * @start_pfn: The Page Frame Number to insert at |
| * @nr_pages: The number of pages of the region |
| */ |
| static int kbase_insert_va_region_nolock(struct kbase_va_region *new_reg, |
| struct kbase_va_region *at_reg, u64 start_pfn, size_t nr_pages) |
| { |
| struct rb_root *reg_rbtree = NULL; |
| int err = 0; |
| |
| reg_rbtree = at_reg->rbtree; |
| |
| /* Must be a free region */ |
| KBASE_DEBUG_ASSERT((at_reg->flags & KBASE_REG_FREE) != 0); |
| /* start_pfn should be contained within at_reg */ |
| KBASE_DEBUG_ASSERT((start_pfn >= at_reg->start_pfn) && (start_pfn < at_reg->start_pfn + at_reg->nr_pages)); |
| /* at least nr_pages from start_pfn should be contained within at_reg */ |
| KBASE_DEBUG_ASSERT(start_pfn + nr_pages <= at_reg->start_pfn + at_reg->nr_pages); |
| |
| new_reg->start_pfn = start_pfn; |
| new_reg->nr_pages = nr_pages; |
| |
| /* Regions are a whole use, so swap and delete old one. */ |
| if (at_reg->start_pfn == start_pfn && at_reg->nr_pages == nr_pages) { |
| rb_replace_node(&(at_reg->rblink), &(new_reg->rblink), |
| reg_rbtree); |
| kfree(at_reg); |
| } |
| /* New region replaces the start of the old one, so insert before. */ |
| else if (at_reg->start_pfn == start_pfn) { |
| at_reg->start_pfn += nr_pages; |
| KBASE_DEBUG_ASSERT(at_reg->nr_pages >= nr_pages); |
| at_reg->nr_pages -= nr_pages; |
| |
| kbase_region_tracker_insert(new_reg); |
| } |
| /* New region replaces the end of the old one, so insert after. */ |
| else if ((at_reg->start_pfn + at_reg->nr_pages) == (start_pfn + nr_pages)) { |
| at_reg->nr_pages -= nr_pages; |
| |
| kbase_region_tracker_insert(new_reg); |
| } |
| /* New region splits the old one, so insert and create new */ |
| else { |
| struct kbase_va_region *new_front_reg; |
| |
| new_front_reg = kbase_alloc_free_region(reg_rbtree, |
| at_reg->start_pfn, |
| start_pfn - at_reg->start_pfn, |
| at_reg->flags & KBASE_REG_ZONE_MASK); |
| |
| if (new_front_reg) { |
| at_reg->nr_pages -= nr_pages + new_front_reg->nr_pages; |
| at_reg->start_pfn = start_pfn + nr_pages; |
| |
| kbase_region_tracker_insert(new_front_reg); |
| kbase_region_tracker_insert(new_reg); |
| } else { |
| err = -ENOMEM; |
| } |
| } |
| |
| return err; |
| } |
| |
| /** |
| * kbase_add_va_region - Add a VA region to the region list for a context. |
| * |
| * @kctx: kbase context containing the region |
| * @reg: the region to add |
| * @addr: the address to insert the region at |
| * @nr_pages: the number of pages in the region |
| * @align: the minimum alignment in pages |
| */ |
| int kbase_add_va_region(struct kbase_context *kctx, |
| struct kbase_va_region *reg, u64 addr, |
| size_t nr_pages, size_t align) |
| { |
| int err = 0; |
| struct kbase_device *kbdev = kctx->kbdev; |
| int cpu_va_bits = kbase_get_num_cpu_va_bits(kctx); |
| int gpu_pc_bits = |
| kbdev->gpu_props.props.core_props.log2_program_counter_size; |
| |
| KBASE_DEBUG_ASSERT(kctx != NULL); |
| KBASE_DEBUG_ASSERT(reg != NULL); |
| |
| lockdep_assert_held(&kctx->reg_lock); |
| |
| /* The executable allocation from the SAME_VA zone would already have an |
| * appropriately aligned GPU VA chosen for it. |
| * Also the executable allocation from EXEC_VA zone doesn't need the |
| * special alignment. |
| */ |
| if (!(reg->flags & KBASE_REG_GPU_NX) && !addr && |
| ((reg->flags & KBASE_REG_ZONE_MASK) != KBASE_REG_ZONE_EXEC_VA)) { |
| if (cpu_va_bits > gpu_pc_bits) { |
| align = max(align, (size_t)((1ULL << gpu_pc_bits) |
| >> PAGE_SHIFT)); |
| } |
| } |
| |
| do { |
| err = kbase_add_va_region_rbtree(kbdev, reg, addr, nr_pages, |
| align); |
| if (err != -ENOMEM) |
| break; |
| |
| /* |
| * If the allocation is not from the same zone as JIT |
| * then don't retry, we're out of VA and there is |
| * nothing which can be done about it. |
| */ |
| if ((reg->flags & KBASE_REG_ZONE_MASK) != |
| KBASE_REG_ZONE_CUSTOM_VA) |
| break; |
| } while (kbase_jit_evict(kctx)); |
| |
| return err; |
| } |
| |
| KBASE_EXPORT_TEST_API(kbase_add_va_region); |
| |
| /** |
| * kbase_add_va_region_rbtree - Insert a region into its corresponding rbtree |
| * |
| * Insert a region into the rbtree that was specified when the region was |
| * created. If addr is 0 a free area in the rbtree is used, otherwise the |
| * specified address is used. |
| * |
| * @kbdev: The kbase device |
| * @reg: The region to add |
| * @addr: The address to add the region at, or 0 to map at any available address |
| * @nr_pages: The size of the region in pages |
| * @align: The minimum alignment in pages |
| */ |
| int kbase_add_va_region_rbtree(struct kbase_device *kbdev, |
| struct kbase_va_region *reg, |
| u64 addr, size_t nr_pages, size_t align) |
| { |
| struct device *const dev = kbdev->dev; |
| struct rb_root *rbtree = NULL; |
| struct kbase_va_region *tmp; |
| u64 gpu_pfn = addr >> PAGE_SHIFT; |
| int err = 0; |
| |
| rbtree = reg->rbtree; |
| |
| if (!align) |
| align = 1; |
| |
| /* must be a power of 2 */ |
| KBASE_DEBUG_ASSERT(is_power_of_2(align)); |
| KBASE_DEBUG_ASSERT(nr_pages > 0); |
| |
| /* Path 1: Map a specific address. Find the enclosing region, |
| * which *must* be free. |
| */ |
| if (gpu_pfn) { |
| KBASE_DEBUG_ASSERT(!(gpu_pfn & (align - 1))); |
| |
| tmp = find_region_enclosing_range_rbtree(rbtree, gpu_pfn, |
| nr_pages); |
| if (kbase_is_region_invalid(tmp)) { |
| dev_warn(dev, "Enclosing region not found or invalid: 0x%08llx gpu_pfn, %zu nr_pages", gpu_pfn, nr_pages); |
| err = -ENOMEM; |
| goto exit; |
| } else if (!kbase_is_region_free(tmp)) { |
| dev_warn(dev, "!(tmp->flags & KBASE_REG_FREE): tmp->start_pfn=0x%llx tmp->flags=0x%lx tmp->nr_pages=0x%zx gpu_pfn=0x%llx nr_pages=0x%zx\n", |
| tmp->start_pfn, tmp->flags, |
| tmp->nr_pages, gpu_pfn, nr_pages); |
| err = -ENOMEM; |
| goto exit; |
| } |
| |
| err = kbase_insert_va_region_nolock(reg, tmp, gpu_pfn, |
| nr_pages); |
| if (err) { |
| dev_warn(dev, "Failed to insert va region"); |
| err = -ENOMEM; |
| } |
| } else { |
| /* Path 2: Map any free address which meets the requirements. */ |
| u64 start_pfn; |
| size_t align_offset = align; |
| size_t align_mask = align - 1; |
| |
| #if !MALI_USE_CSF |
| if ((reg->flags & KBASE_REG_TILER_ALIGN_TOP)) { |
| WARN(align > 1, "%s with align %lx might not be honored for KBASE_REG_TILER_ALIGN_TOP memory", |
| __func__, |
| (unsigned long)align); |
| align_mask = reg->extension - 1; |
| align_offset = reg->extension - reg->initial_commit; |
| } |
| #endif /* !MALI_USE_CSF */ |
| |
| tmp = kbase_region_tracker_find_region_meeting_reqs(reg, |
| nr_pages, align_offset, align_mask, |
| &start_pfn); |
| if (tmp) { |
| err = kbase_insert_va_region_nolock(reg, tmp, |
| start_pfn, nr_pages); |
| if (unlikely(err)) { |
| dev_warn(dev, "Failed to insert region: 0x%08llx start_pfn, %zu nr_pages", |
| start_pfn, nr_pages); |
| } |
| } else { |
| dev_dbg(dev, "Failed to find a suitable region: %zu nr_pages, %zu align_offset, %zu align_mask\n", |
| nr_pages, align_offset, align_mask); |
| err = -ENOMEM; |
| } |
| } |
| |
| exit: |
| return err; |
| } |
| |
| /* |
| * @brief Initialize the internal region tracker data structure. |
| */ |
| static void kbase_region_tracker_ds_init(struct kbase_context *kctx, |
| struct kbase_va_region *same_va_reg, |
| struct kbase_va_region *custom_va_reg) |
| { |
| kctx->reg_rbtree_same = RB_ROOT; |
| kbase_region_tracker_insert(same_va_reg); |
| |
| /* Although custom_va_reg and exec_va_reg don't always exist, |
| * initialize unconditionally because of the mem_view debugfs |
| * implementation which relies on them being empty. |
| * |
| * The difference between the two is that the EXEC_VA region |
| * is never initialized at this stage. |
| */ |
| kctx->reg_rbtree_custom = RB_ROOT; |
| kctx->reg_rbtree_exec = RB_ROOT; |
| |
| if (custom_va_reg) |
| kbase_region_tracker_insert(custom_va_reg); |
| } |
| |
| static void kbase_region_tracker_erase_rbtree(struct rb_root *rbtree) |
| { |
| struct rb_node *rbnode; |
| struct kbase_va_region *reg; |
| |
| do { |
| rbnode = rb_first(rbtree); |
| if (rbnode) { |
| rb_erase(rbnode, rbtree); |
| reg = rb_entry(rbnode, struct kbase_va_region, rblink); |
| WARN_ON(reg->va_refcnt != 1); |
| /* Reset the start_pfn - as the rbtree is being |
| * destroyed and we've already erased this region, there |
| * is no further need to attempt to remove it. |
| * This won't affect the cleanup if the region was |
| * being used as a sticky resource as the cleanup |
| * related to sticky resources anyways need to be |
| * performed before the term of region tracker. |
| */ |
| reg->start_pfn = 0; |
| kbase_free_alloced_region(reg); |
| } |
| } while (rbnode); |
| } |
| |
| void kbase_region_tracker_term(struct kbase_context *kctx) |
| { |
| kbase_gpu_vm_lock(kctx); |
| kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_same); |
| kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_custom); |
| kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_exec); |
| #if MALI_USE_CSF |
| WARN_ON(!list_empty(&kctx->csf.event_pages_head)); |
| #endif |
| kbase_gpu_vm_unlock(kctx); |
| } |
| |
| void kbase_region_tracker_term_rbtree(struct rb_root *rbtree) |
| { |
| kbase_region_tracker_erase_rbtree(rbtree); |
| } |
| |
| static size_t kbase_get_same_va_bits(struct kbase_context *kctx) |
| { |
| return min(kbase_get_num_cpu_va_bits(kctx), |
| (size_t) kctx->kbdev->gpu_props.mmu.va_bits); |
| } |
| |
| int kbase_region_tracker_init(struct kbase_context *kctx) |
| { |
| struct kbase_va_region *same_va_reg; |
| struct kbase_va_region *custom_va_reg = NULL; |
| size_t same_va_bits = kbase_get_same_va_bits(kctx); |
| u64 custom_va_size = KBASE_REG_ZONE_CUSTOM_VA_SIZE; |
| u64 gpu_va_limit = (1ULL << kctx->kbdev->gpu_props.mmu.va_bits) >> PAGE_SHIFT; |
| u64 same_va_pages; |
| u64 same_va_base = 1u; |
| int err; |
| |
| /* Take the lock as kbase_free_alloced_region requires it */ |
| kbase_gpu_vm_lock(kctx); |
| |
| same_va_pages = (1ULL << (same_va_bits - PAGE_SHIFT)) - same_va_base; |
| /* all have SAME_VA */ |
| same_va_reg = |
| kbase_alloc_free_region(&kctx->reg_rbtree_same, same_va_base, |
| same_va_pages, KBASE_REG_ZONE_SAME_VA); |
| |
| if (!same_va_reg) { |
| err = -ENOMEM; |
| goto fail_unlock; |
| } |
| kbase_ctx_reg_zone_init(kctx, KBASE_REG_ZONE_SAME_VA, same_va_base, |
| same_va_pages); |
| |
| #if IS_ENABLED(CONFIG_64BIT) |
| /* 32-bit clients have custom VA zones */ |
| if (kbase_ctx_flag(kctx, KCTX_COMPAT)) { |
| #endif |
| if (gpu_va_limit <= KBASE_REG_ZONE_CUSTOM_VA_BASE) { |
| err = -EINVAL; |
| goto fail_free_same_va; |
| } |
| /* If the current size of TMEM is out of range of the |
| * virtual address space addressable by the MMU then |
| * we should shrink it to fit |
| */ |
| if ((KBASE_REG_ZONE_CUSTOM_VA_BASE + KBASE_REG_ZONE_CUSTOM_VA_SIZE) >= gpu_va_limit) |
| custom_va_size = gpu_va_limit - KBASE_REG_ZONE_CUSTOM_VA_BASE; |
| |
| custom_va_reg = kbase_alloc_free_region( |
| &kctx->reg_rbtree_custom, |
| KBASE_REG_ZONE_CUSTOM_VA_BASE, |
| custom_va_size, KBASE_REG_ZONE_CUSTOM_VA); |
| |
| if (!custom_va_reg) { |
| err = -ENOMEM; |
| goto fail_free_same_va; |
| } |
| kbase_ctx_reg_zone_init(kctx, KBASE_REG_ZONE_CUSTOM_VA, |
| KBASE_REG_ZONE_CUSTOM_VA_BASE, |
| custom_va_size); |
| #if IS_ENABLED(CONFIG_64BIT) |
| } else { |
| custom_va_size = 0; |
| } |
| #endif |
| /* EXEC_VA zone's codepaths are slightly easier when its base_pfn is |
| * initially U64_MAX |
| */ |
| kbase_ctx_reg_zone_init(kctx, KBASE_REG_ZONE_EXEC_VA, U64_MAX, 0u); |
| /* Other zones are 0: kbase_create_context() uses vzalloc */ |
| |
| kbase_region_tracker_ds_init(kctx, same_va_reg, custom_va_reg); |
| |
| kctx->gpu_va_end = same_va_base + same_va_pages + custom_va_size; |
| kctx->jit_va = false; |
| |
| #if MALI_USE_CSF |
| INIT_LIST_HEAD(&kctx->csf.event_pages_head); |
| #endif |
| |
| kbase_gpu_vm_unlock(kctx); |
| return 0; |
| |
| fail_free_same_va: |
| kbase_free_alloced_region(same_va_reg); |
| fail_unlock: |
| kbase_gpu_vm_unlock(kctx); |
| return err; |
| } |
| |
| static bool kbase_has_exec_va_zone_locked(struct kbase_context *kctx) |
| { |
| struct kbase_reg_zone *exec_va_zone; |
| |
| lockdep_assert_held(&kctx->reg_lock); |
| exec_va_zone = kbase_ctx_reg_zone_get(kctx, KBASE_REG_ZONE_EXEC_VA); |
| |
| return (exec_va_zone->base_pfn != U64_MAX); |
| } |
| |
| bool kbase_has_exec_va_zone(struct kbase_context *kctx) |
| { |
| bool has_exec_va_zone; |
| |
| kbase_gpu_vm_lock(kctx); |
| has_exec_va_zone = kbase_has_exec_va_zone_locked(kctx); |
| kbase_gpu_vm_unlock(kctx); |
| |
| return has_exec_va_zone; |
| } |
| |
| /** |
| * Determine if any allocations have been made on a context's region tracker |
| * @kctx: KBase context |
| * |
| * Check the context to determine if any allocations have been made yet from |
| * any of its zones. This check should be done before resizing a zone, e.g. to |
| * make space to add a second zone. |
| * |
| * Whilst a zone without allocations can be resized whilst other zones have |
| * allocations, we still check all of @kctx 's zones anyway: this is a stronger |
| * guarantee and should be adhered to when creating new zones anyway. |
| * |
| * Allocations from kbdev zones are not counted. |
| * |
| * Return: true if any allocs exist on any zone, false otherwise |
| */ |
| static bool kbase_region_tracker_has_allocs(struct kbase_context *kctx) |
| { |
| unsigned int zone_idx; |
| |
| lockdep_assert_held(&kctx->reg_lock); |
| |
| for (zone_idx = 0; zone_idx < KBASE_REG_ZONE_MAX; ++zone_idx) { |
| struct kbase_reg_zone *zone; |
| struct kbase_va_region *reg; |
| u64 zone_base_addr; |
| unsigned long zone_bits = KBASE_REG_ZONE(zone_idx); |
| unsigned long reg_zone; |
| |
| zone = kbase_ctx_reg_zone_get(kctx, zone_bits); |
| zone_base_addr = zone->base_pfn << PAGE_SHIFT; |
| |
| reg = kbase_region_tracker_find_region_base_address( |
| kctx, zone_base_addr); |
| |
| if (!zone->va_size_pages) { |
| WARN(reg, |
| "Should not have found a region that starts at 0x%.16llx for zone 0x%lx", |
| (unsigned long long)zone_base_addr, zone_bits); |
| continue; |
| } |
| |
| if (WARN(!reg, |
| "There should always be a region that starts at 0x%.16llx for zone 0x%lx, couldn't find it", |
| (unsigned long long)zone_base_addr, zone_bits)) |
| return true; /* Safest return value */ |
| |
| reg_zone = reg->flags & KBASE_REG_ZONE_MASK; |
| if (WARN(reg_zone != zone_bits, |
| "The region that starts at 0x%.16llx should be in zone 0x%lx but was found in the wrong zone 0x%lx", |
| (unsigned long long)zone_base_addr, zone_bits, |
| reg_zone)) |
| return true; /* Safest return value */ |
| |
| /* Unless the region is completely free, of the same size as |
| * the original zone, then it has allocs |
| */ |
| if ((!(reg->flags & KBASE_REG_FREE)) || |
| (reg->nr_pages != zone->va_size_pages)) |
| return true; |
| } |
| |
| /* All zones are the same size as originally made, so there are no |
| * allocs |
| */ |
| return false; |
| } |
| |
| #if IS_ENABLED(CONFIG_64BIT) |
| static int kbase_region_tracker_init_jit_64(struct kbase_context *kctx, |
| u64 jit_va_pages) |
| { |
| struct kbase_va_region *same_va_reg; |
| struct kbase_reg_zone *same_va_zone; |
| u64 same_va_zone_base_addr; |
| const unsigned long same_va_zone_bits = KBASE_REG_ZONE_SAME_VA; |
| struct kbase_va_region *custom_va_reg; |
| u64 jit_va_start; |
| |
| lockdep_assert_held(&kctx->reg_lock); |
| |
| /* |
| * Modify the same VA free region after creation. The caller has |
| * ensured that allocations haven't been made, as any allocations could |
| * cause an overlap to happen with existing same VA allocations and the |
| * custom VA zone. |
| */ |
| same_va_zone = kbase_ctx_reg_zone_get(kctx, same_va_zone_bits); |
| same_va_zone_base_addr = same_va_zone->base_pfn << PAGE_SHIFT; |
| |
| same_va_reg = kbase_region_tracker_find_region_base_address( |
| kctx, same_va_zone_base_addr); |
| if (WARN(!same_va_reg, |
| "Already found a free region at the start of every zone, but now cannot find any region for zone base 0x%.16llx zone 0x%lx", |
| (unsigned long long)same_va_zone_base_addr, same_va_zone_bits)) |
| return -ENOMEM; |
| |
| /* kbase_region_tracker_has_allocs() in the caller has already ensured |
| * that all of the zones have no allocs, so no need to check that again |
| * on same_va_reg |
| */ |
| WARN_ON((!(same_va_reg->flags & KBASE_REG_FREE)) || |
| same_va_reg->nr_pages != same_va_zone->va_size_pages); |
| |
| if (same_va_reg->nr_pages < jit_va_pages || |
| same_va_zone->va_size_pages < jit_va_pages) |
| return -ENOMEM; |
| |
| /* It's safe to adjust the same VA zone now */ |
| same_va_reg->nr_pages -= jit_va_pages; |
| same_va_zone->va_size_pages -= jit_va_pages; |
| jit_va_start = kbase_reg_zone_end_pfn(same_va_zone); |
| |
| /* |
| * Create a custom VA zone at the end of the VA for allocations which |
| * JIT can use so it doesn't have to allocate VA from the kernel. |
| */ |
| custom_va_reg = |
| kbase_alloc_free_region(&kctx->reg_rbtree_custom, jit_va_start, |
| jit_va_pages, KBASE_REG_ZONE_CUSTOM_VA); |
| |
| /* |
| * The context will be destroyed if we fail here so no point |
| * reverting the change we made to same_va. |
| */ |
| if (!custom_va_reg) |
| return -ENOMEM; |
| /* Since this is 64-bit, the custom zone will not have been |
| * initialized, so initialize it now |
| */ |
| kbase_ctx_reg_zone_init(kctx, KBASE_REG_ZONE_CUSTOM_VA, jit_va_start, |
| jit_va_pages); |
| |
| kbase_region_tracker_insert(custom_va_reg); |
| return 0; |
| } |
| #endif |
| |
| int kbase_region_tracker_init_jit(struct kbase_context *kctx, u64 jit_va_pages, |
| int max_allocations, int trim_level, int group_id, |
| u64 phys_pages_limit) |
| { |
| int err = 0; |
| |
| if (trim_level < 0 || trim_level > BASE_JIT_MAX_TRIM_LEVEL) |
| return -EINVAL; |
| |
| if (group_id < 0 || group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS) |
| return -EINVAL; |
| |
| if (phys_pages_limit > jit_va_pages) |
| return -EINVAL; |
| |
| #if MALI_JIT_PRESSURE_LIMIT_BASE |
| if (phys_pages_limit != jit_va_pages) |
| kbase_ctx_flag_set(kctx, KCTX_JPL_ENABLED); |
| #endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ |
| |
| kbase_gpu_vm_lock(kctx); |
| |
| /* Verify that a JIT_VA zone has not been created already. */ |
| if (kctx->jit_va) { |
| err = -EINVAL; |
| goto exit_unlock; |
| } |
| |
| /* If in 64-bit, we always lookup the SAME_VA zone. To ensure it has no |
| * allocs, we can ensure there are no allocs anywhere. |
| * |
| * This check is also useful in 32-bit, just to make sure init of the |
| * zone is always done before any allocs. |
| */ |
| if (kbase_region_tracker_has_allocs(kctx)) { |
| err = -ENOMEM; |
| goto exit_unlock; |
| } |
| |
| #if IS_ENABLED(CONFIG_64BIT) |
| if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) |
| err = kbase_region_tracker_init_jit_64(kctx, jit_va_pages); |
| #endif |
| /* |
| * Nothing to do for 32-bit clients, JIT uses the existing |
| * custom VA zone. |
| */ |
| |
| if (!err) { |
| kctx->jit_max_allocations = max_allocations; |
| kctx->trim_level = trim_level; |
| kctx->jit_va = true; |
| kctx->jit_group_id = group_id; |
| #if MALI_JIT_PRESSURE_LIMIT_BASE |
| kctx->jit_phys_pages_limit = phys_pages_limit; |
| dev_dbg(kctx->kbdev->dev, "phys_pages_limit set to %llu\n", |
| phys_pages_limit); |
| #endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ |
| } |
| |
| exit_unlock: |
| kbase_gpu_vm_unlock(kctx); |
| |
| return err; |
| } |
| |
| int kbase_region_tracker_init_exec(struct kbase_context *kctx, u64 exec_va_pages) |
| { |
| struct kbase_va_region *exec_va_reg; |
| struct kbase_reg_zone *exec_va_zone; |
| struct kbase_reg_zone *target_zone; |
| struct kbase_va_region *target_reg; |
| u64 target_zone_base_addr; |
| unsigned long target_zone_bits; |
| u64 exec_va_start; |
| int err; |
| |
| /* The EXEC_VA zone shall be created by making space either: |
| * - for 64-bit clients, at the end of the process's address space |
| * - for 32-bit clients, in the CUSTOM zone |
| * |
| * Firstly, verify that the number of EXEC_VA pages requested by the |
| * client is reasonable and then make sure that it is not greater than |
| * the address space itself before calculating the base address of the |
| * new zone. |
| */ |
| if (exec_va_pages == 0 || exec_va_pages > KBASE_REG_ZONE_EXEC_VA_MAX_PAGES) |
| return -EINVAL; |
| |
| kbase_gpu_vm_lock(kctx); |
| |
| /* Verify that we've not already created a EXEC_VA zone, and that the |
| * EXEC_VA zone must come before JIT's CUSTOM_VA. |
| */ |
| if (kbase_has_exec_va_zone_locked(kctx) || kctx->jit_va) { |
| err = -EPERM; |
| goto exit_unlock; |
| } |
| |
| if (exec_va_pages > kctx->gpu_va_end) { |
| err = -ENOMEM; |
| goto exit_unlock; |
| } |
| |
| /* Verify no allocations have already been made */ |
| if (kbase_region_tracker_has_allocs(kctx)) { |
| err = -ENOMEM; |
| goto exit_unlock; |
| } |
| |
| #if IS_ENABLED(CONFIG_64BIT) |
| if (kbase_ctx_flag(kctx, KCTX_COMPAT)) { |
| #endif |
| /* 32-bit client: take from CUSTOM_VA zone */ |
| target_zone_bits = KBASE_REG_ZONE_CUSTOM_VA; |
| #if IS_ENABLED(CONFIG_64BIT) |
| } else { |
| /* 64-bit client: take from SAME_VA zone */ |
| target_zone_bits = KBASE_REG_ZONE_SAME_VA; |
| } |
| #endif |
| target_zone = kbase_ctx_reg_zone_get(kctx, target_zone_bits); |
| target_zone_base_addr = target_zone->base_pfn << PAGE_SHIFT; |
| |
| target_reg = kbase_region_tracker_find_region_base_address( |
| kctx, target_zone_base_addr); |
| if (WARN(!target_reg, |
| "Already found a free region at the start of every zone, but now cannot find any region for zone base 0x%.16llx zone 0x%lx", |
| (unsigned long long)target_zone_base_addr, target_zone_bits)) { |
| err = -ENOMEM; |
| goto exit_unlock; |
| } |
| /* kbase_region_tracker_has_allocs() above has already ensured that all |
| * of the zones have no allocs, so no need to check that again on |
| * target_reg |
| */ |
| WARN_ON((!(target_reg->flags & KBASE_REG_FREE)) || |
| target_reg->nr_pages != target_zone->va_size_pages); |
| |
| if (target_reg->nr_pages <= exec_va_pages || |
| target_zone->va_size_pages <= exec_va_pages) { |
| err = -ENOMEM; |
| goto exit_unlock; |
| } |
| |
| /* Taken from the end of the target zone */ |
| exec_va_start = kbase_reg_zone_end_pfn(target_zone) - exec_va_pages; |
| |
| exec_va_reg = kbase_alloc_free_region(&kctx->reg_rbtree_exec, |
| exec_va_start, |
| exec_va_pages, |
| KBASE_REG_ZONE_EXEC_VA); |
| if (!exec_va_reg) { |
| err = -ENOMEM; |
| goto exit_unlock; |
| } |
| /* Update EXEC_VA zone |
| * |
| * not using kbase_ctx_reg_zone_init() - it was already initialized |
| */ |
| exec_va_zone = kbase_ctx_reg_zone_get(kctx, KBASE_REG_ZONE_EXEC_VA); |
| exec_va_zone->base_pfn = exec_va_start; |
| exec_va_zone->va_size_pages = exec_va_pages; |
| |
| /* Update target zone and corresponding region */ |
| target_reg->nr_pages -= exec_va_pages; |
| target_zone->va_size_pages -= exec_va_pages; |
| |
| kbase_region_tracker_insert(exec_va_reg); |
| err = 0; |
| |
| exit_unlock: |
| kbase_gpu_vm_unlock(kctx); |
| return err; |
| } |
| |
| #if MALI_USE_CSF |
| void kbase_mcu_shared_interface_region_tracker_term(struct kbase_device *kbdev) |
| { |
| kbase_region_tracker_term_rbtree(&kbdev->csf.shared_reg_rbtree); |
| } |
| |
| int kbase_mcu_shared_interface_region_tracker_init(struct kbase_device *kbdev) |
| { |
| struct kbase_va_region *shared_reg; |
| u64 shared_reg_start_pfn; |
| u64 shared_reg_size; |
| |
| shared_reg_start_pfn = KBASE_REG_ZONE_MCU_SHARED_BASE; |
| shared_reg_size = KBASE_REG_ZONE_MCU_SHARED_SIZE; |
| |
| kbdev->csf.shared_reg_rbtree = RB_ROOT; |
| |
| shared_reg = kbase_alloc_free_region(&kbdev->csf.shared_reg_rbtree, |
| shared_reg_start_pfn, |
| shared_reg_size, |
| KBASE_REG_ZONE_MCU_SHARED); |
| if (!shared_reg) |
| return -ENOMEM; |
| |
| kbase_region_tracker_insert(shared_reg); |
| return 0; |
| } |
| #endif |
| |
| int kbase_mem_init(struct kbase_device *kbdev) |
| { |
| int err = 0; |
| struct kbasep_mem_device *memdev; |
| #if IS_ENABLED(CONFIG_OF) |
| struct device_node *mgm_node = NULL; |
| #endif |
| |
| KBASE_DEBUG_ASSERT(kbdev); |
| |
| memdev = &kbdev->memdev; |
| |
| kbase_mem_pool_group_config_set_max_size(&kbdev->mem_pool_defaults, |
| KBASE_MEM_POOL_MAX_SIZE_KCTX); |
| |
| /* Initialize memory usage */ |
| atomic_set(&memdev->used_pages, 0); |
| |
| spin_lock_init(&kbdev->gpu_mem_usage_lock); |
| kbdev->total_gpu_pages = 0; |
| kbdev->dma_buf_pages = 0; |
| kbdev->process_root = RB_ROOT; |
| kbdev->dma_buf_root = RB_ROOT; |
| mutex_init(&kbdev->dma_buf_lock); |
| |
| #ifdef IR_THRESHOLD |
| atomic_set(&memdev->ir_threshold, IR_THRESHOLD); |
| #else |
| atomic_set(&memdev->ir_threshold, DEFAULT_IR_THRESHOLD); |
| #endif |
| |
| kbdev->mgm_dev = &kbase_native_mgm_dev; |
| |
| #if IS_ENABLED(CONFIG_OF) |
| /* Check to see whether or not a platform-specific memory group manager |
| * is configured and available. |
| */ |
| mgm_node = of_parse_phandle(kbdev->dev->of_node, |
| "physical-memory-group-manager", 0); |
| if (!mgm_node) { |
| dev_info(kbdev->dev, |
| "No memory group manager is configured\n"); |
| } else { |
| struct platform_device *const pdev = |
| of_find_device_by_node(mgm_node); |
| |
| if (!pdev) { |
| dev_err(kbdev->dev, |
| "The configured memory group manager was not found\n"); |
| } else { |
| kbdev->mgm_dev = platform_get_drvdata(pdev); |
| if (!kbdev->mgm_dev) { |
| dev_info(kbdev->dev, |
| "Memory group manager is not ready\n"); |
| err = -EPROBE_DEFER; |
| } else if (!try_module_get(kbdev->mgm_dev->owner)) { |
| dev_err(kbdev->dev, |
| "Failed to get memory group manger module\n"); |
| err = -ENODEV; |
| kbdev->mgm_dev = NULL; |
| } else { |
| dev_info(kbdev->dev, |
| "Memory group manager successfully loaded\n"); |
| } |
| } |
| of_node_put(mgm_node); |
| } |
| #endif |
| |
| if (likely(!err)) { |
| struct kbase_mem_pool_group_config mem_pool_defaults; |
| |
| kbase_mem_pool_group_config_set_max_size(&mem_pool_defaults, |
| KBASE_MEM_POOL_MAX_SIZE_KBDEV); |
| |
| err = kbase_mem_pool_group_init(&kbdev->mem_pools, kbdev, |
| &mem_pool_defaults, NULL); |
| } |
| |
| return err; |
| } |
| |
| void kbase_mem_halt(struct kbase_device *kbdev) |
| { |
| CSTD_UNUSED(kbdev); |
| } |
| |
| void kbase_mem_term(struct kbase_device *kbdev) |
| { |
| struct kbasep_mem_device *memdev; |
| int pages; |
| |
| KBASE_DEBUG_ASSERT(kbdev); |
| |
| memdev = &kbdev->memdev; |
| |
| pages = atomic_read(&memdev->used_pages); |
| if (pages != 0) |
| dev_warn(kbdev->dev, "%s: %d pages in use!\n", __func__, pages); |
| |
| kbase_mem_pool_group_term(&kbdev->mem_pools); |
| |
| WARN_ON(kbdev->total_gpu_pages); |
| WARN_ON(!RB_EMPTY_ROOT(&kbdev->process_root)); |
| WARN_ON(!RB_EMPTY_ROOT(&kbdev->dma_buf_root)); |
| mutex_destroy(&kbdev->dma_buf_lock); |
| |
| if (kbdev->mgm_dev) |
| module_put(kbdev->mgm_dev->owner); |
| } |
| KBASE_EXPORT_TEST_API(kbase_mem_term); |
| |
| /** |
| * Allocate a free region object. |
| * @rbtree: Backlink to the red-black tree of memory regions. |
| * @start_pfn: The Page Frame Number in GPU virtual address space. |
| * @nr_pages: The size of the region in pages. |
| * @zone: KBASE_REG_ZONE_CUSTOM_VA or KBASE_REG_ZONE_SAME_VA |
| * |
| * The allocated object is not part of any list yet, and is flagged as |
| * KBASE_REG_FREE. No mapping is allocated yet. |
| * |
| * zone is KBASE_REG_ZONE_CUSTOM_VA or KBASE_REG_ZONE_SAME_VA. |
| * |
| */ |
| struct kbase_va_region *kbase_alloc_free_region(struct rb_root *rbtree, |
| u64 start_pfn, size_t nr_pages, int zone) |
| { |
| struct kbase_va_region *new_reg; |
| |
| KBASE_DEBUG_ASSERT(rbtree != NULL); |
| |
| /* zone argument should only contain zone related region flags */ |
| KBASE_DEBUG_ASSERT((zone & ~KBASE_REG_ZONE_MASK) == 0); |
| KBASE_DEBUG_ASSERT(nr_pages > 0); |
| /* 64-bit address range is the max */ |
| KBASE_DEBUG_ASSERT(start_pfn + nr_pages <= (U64_MAX / PAGE_SIZE)); |
| |
| new_reg = kzalloc(sizeof(*new_reg), GFP_KERNEL); |
| |
| if (!new_reg) |
| return NULL; |
| |
| new_reg->va_refcnt = 1; |
| new_reg->cpu_alloc = NULL; /* no alloc bound yet */ |
| new_reg->gpu_alloc = NULL; /* no alloc bound yet */ |
| new_reg->rbtree = rbtree; |
| new_reg->flags = zone | KBASE_REG_FREE; |
| |
| new_reg->flags |= KBASE_REG_GROWABLE; |
| |
| new_reg->start_pfn = start_pfn; |
| new_reg->nr_pages = nr_pages; |
| |
| INIT_LIST_HEAD(&new_reg->jit_node); |
| INIT_LIST_HEAD(&new_reg->link); |
| |
| return new_reg; |
| } |
| |
| KBASE_EXPORT_TEST_API(kbase_alloc_free_region); |
| |
| static struct kbase_context *kbase_reg_flags_to_kctx( |
| struct kbase_va_region *reg) |
| { |
| struct kbase_context *kctx = NULL; |
| struct rb_root *rbtree = reg->rbtree; |
| |
| switch (reg->flags & KBASE_REG_ZONE_MASK) { |
| case KBASE_REG_ZONE_CUSTOM_VA: |
| kctx = container_of(rbtree, struct kbase_context, |
| reg_rbtree_custom); |
| break; |
| case KBASE_REG_ZONE_SAME_VA: |
| kctx = container_of(rbtree, struct kbase_context, |
| reg_rbtree_same); |
| break; |
| case KBASE_REG_ZONE_EXEC_VA: |
| kctx = container_of(rbtree, struct kbase_context, |
| reg_rbtree_exec); |
| break; |
| default: |
| WARN(1, "Unknown zone in region: flags=0x%lx\n", reg->flags); |
| break; |
| } |
| |
| return kctx; |
| } |
| |
| /** |
| * Free a region object. |
| * @reg: Region |
| * |
| * The described region must be freed of any mapping. |
| * |
| * If the region is not flagged as KBASE_REG_FREE, the region's |
| * alloc object will be released. |
| * It is a bug if no alloc object exists for non-free regions. |
| * |
| */ |
| void kbase_free_alloced_region(struct kbase_va_region *reg) |
| { |
| #if MALI_USE_CSF |
| if ((reg->flags & KBASE_REG_ZONE_MASK) == |
| KBASE_REG_ZONE_MCU_SHARED) { |
| kfree(reg); |
| return; |
| } |
| #endif |
| if (!(reg->flags & KBASE_REG_FREE)) { |
| struct kbase_context *kctx = kbase_reg_flags_to_kctx(reg); |
| |
| if (WARN_ON(!kctx)) |
| return; |
| |
| if (WARN_ON(kbase_is_region_invalid(reg))) |
| return; |
| |
| dev_dbg(kctx->kbdev->dev, "Freeing memory region %pK\n", |
| (void *)reg); |
| #if MALI_USE_CSF |
| if (reg->flags & KBASE_REG_CSF_EVENT) |
| kbase_unlink_event_mem_page(kctx, reg); |
| #endif |
| |
| mutex_lock(&kctx->jit_evict_lock); |
| |
| /* |
| * The physical allocation should have been removed from the |
| * eviction list before this function is called. However, in the |
| * case of abnormal process termination or the app leaking the |
| * memory kbase_mem_free_region is not called so it can still be |
| * on the list at termination time of the region tracker. |
| */ |
| if (!list_empty(®->gpu_alloc->evict_node)) { |
| mutex_unlock(&kctx->jit_evict_lock); |
| |
| /* |
| * Unlink the physical allocation before unmaking it |
| * evictable so that the allocation isn't grown back to |
| * its last backed size as we're going to unmap it |
| * anyway. |
| */ |
| reg->cpu_alloc->reg = NULL; |
| if (reg->cpu_alloc != reg->gpu_alloc) |
| reg->gpu_alloc->reg = NULL; |
| |
| /* |
| * If a region has been made evictable then we must |
| * unmake it before trying to free it. |
| * If the memory hasn't been reclaimed it will be |
| * unmapped and freed below, if it has been reclaimed |
| * then the operations below are no-ops. |
| */ |
| if (reg->flags & KBASE_REG_DONT_NEED) { |
| KBASE_DEBUG_ASSERT(reg->cpu_alloc->type == |
| KBASE_MEM_TYPE_NATIVE); |
| kbase_mem_evictable_unmake(reg->gpu_alloc); |
| } |
| } else { |
| mutex_unlock(&kctx->jit_evict_lock); |
| } |
| |
| /* |
| * Remove the region from the sticky resource metadata |
| * list should it be there. |
| */ |
| kbase_sticky_resource_release_force(kctx, NULL, |
| reg->start_pfn << PAGE_SHIFT); |
| |
| kbase_mem_phy_alloc_put(reg->cpu_alloc); |
| kbase_mem_phy_alloc_put(reg->gpu_alloc); |
| |
| reg->flags |= KBASE_REG_VA_FREED; |
| kbase_va_region_alloc_put(kctx, reg); |
| } else { |
| kfree(reg); |
| } |
| } |
| |
| KBASE_EXPORT_TEST_API(kbase_free_alloced_region); |
| |
| int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align) |
| { |
| int err; |
| size_t i = 0; |
| unsigned long attr; |
| unsigned long mask = ~KBASE_REG_MEMATTR_MASK; |
| unsigned long gwt_mask = ~0; |
| int group_id; |
| struct kbase_mem_phy_alloc *alloc; |
| |
| #ifdef CONFIG_MALI_CINSTR_GWT |
| if (kctx->gwt_enabled) |
| gwt_mask = ~KBASE_REG_GPU_WR; |
| #endif |
| |
| if ((kctx->kbdev->system_coherency == COHERENCY_ACE) && |
| (reg->flags & KBASE_REG_SHARE_BOTH)) |
| attr = KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_OUTER_WA); |
| else |
| attr = KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_WRITE_ALLOC); |
| |
| KBASE_DEBUG_ASSERT(kctx != NULL); |
| KBASE_DEBUG_ASSERT(reg != NULL); |
| |
| err = kbase_add_va_region(kctx, reg, addr, nr_pages, align); |
| if (err) |
| return err; |
| |
| alloc = reg->gpu_alloc; |
| group_id = alloc->group_id; |
| |
| if (reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) { |
| u64 const stride = alloc->imported.alias.stride; |
| |
| KBASE_DEBUG_ASSERT(alloc->imported.alias.aliased); |
| for (i = 0; i < alloc->imported.alias.nents; i++) { |
| if (alloc->imported.alias.aliased[i].alloc) { |
| err = kbase_mmu_insert_pages(kctx->kbdev, |
| &kctx->mmu, |
| reg->start_pfn + (i * stride), |
| alloc->imported.alias.aliased[i].alloc->pages + alloc->imported.alias.aliased[i].offset, |
| alloc->imported.alias.aliased[i].length, |
| reg->flags & gwt_mask, |
| kctx->as_nr, |
| group_id); |
| if (err) |
| goto bad_insert; |
| |
| /* Note: mapping count is tracked at alias |
| * creation time |
| */ |
| } else { |
| err = kbase_mmu_insert_single_page(kctx, |
| reg->start_pfn + i * stride, |
| kctx->aliasing_sink_page, |
| alloc->imported.alias.aliased[i].length, |
| (reg->flags & mask & gwt_mask) | attr, |
| group_id); |
| |
| if (err) |
| goto bad_insert; |
| } |
| } |
| } else { |
| err = kbase_mmu_insert_pages(kctx->kbdev, |
| &kctx->mmu, |
| reg->start_pfn, |
| kbase_get_gpu_phy_pages(reg), |
| kbase_reg_current_backed_size(reg), |
| reg->flags & gwt_mask, |
| kctx->as_nr, |
| group_id); |
| if (err) |
| goto bad_insert; |
| kbase_mem_phy_alloc_gpu_mapped(alloc); |
| } |
| |
| if (reg->flags & KBASE_REG_IMPORT_PAD && |
| !WARN_ON(reg->nr_pages < reg->gpu_alloc->nents) && |
| reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM && |
| reg->gpu_alloc->imported.umm.current_mapping_usage_count) { |
| /* For padded imported dma-buf memory, map the dummy aliasing |
| * page from the end of the dma-buf pages, to the end of the |
| * region using a read only mapping. |
| * |
| * Only map when it's imported dma-buf memory that is currently |
| * mapped. |
| * |
| * Assume reg->gpu_alloc->nents is the number of actual pages |
| * in the dma-buf memory. |
| */ |
| err = kbase_mmu_insert_single_page(kctx, |
| reg->start_pfn + reg->gpu_alloc->nents, |
| kctx->aliasing_sink_page, |
| reg->nr_pages - reg->gpu_alloc->nents, |
| (reg->flags | KBASE_REG_GPU_RD) & |
| ~KBASE_REG_GPU_WR, |
| KBASE_MEM_GROUP_SINK); |
| if (err) |
| goto bad_insert; |
| } |
| |
| return err; |
| |
| bad_insert: |
| kbase_mmu_teardown_pages(kctx->kbdev, &kctx->mmu, |
| reg->start_pfn, reg->nr_pages, |
| kctx->as_nr); |
| |
| kbase_remove_va_region(reg); |
| |
| return err; |
| } |
| |
| KBASE_EXPORT_TEST_API(kbase_gpu_mmap); |
| |
| static void kbase_jd_user_buf_unmap(struct kbase_context *kctx, |
| struct kbase_mem_phy_alloc *alloc, bool writeable); |
| |
| int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg) |
| { |
| int err = 0; |
| |
| if (reg->start_pfn == 0) |
| return 0; |
| |
| if (!reg->gpu_alloc) |
| return -EINVAL; |
| |
| /* Tear down down GPU page tables, depending on memory type. */ |
| switch (reg->gpu_alloc->type) { |
| case KBASE_MEM_TYPE_ALIAS: /* Fall-through */ |
| case KBASE_MEM_TYPE_IMPORTED_UMM: |
| err = kbase_mmu_teardown_pages(kctx->kbdev, &kctx->mmu, |
| reg->start_pfn, reg->nr_pages, kctx->as_nr); |
| break; |
| default: |
| err = kbase_mmu_teardown_pages(kctx->kbdev, &kctx->mmu, |
| reg->start_pfn, kbase_reg_current_backed_size(reg), |
| kctx->as_nr); |
| break; |
| } |
| |
| /* Update tracking, and other cleanup, depending on memory type. */ |
| switch (reg->gpu_alloc->type) { |
| case KBASE_MEM_TYPE_ALIAS: |
| /* We mark the source allocs as unmapped from the GPU when |
| * putting reg's allocs |
| */ |
| break; |
| case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { |
| struct kbase_alloc_import_user_buf *user_buf = |
| ®->gpu_alloc->imported.user_buf; |
| |
| if (user_buf->current_mapping_usage_count & PINNED_ON_IMPORT) { |
| user_buf->current_mapping_usage_count &= |
| ~PINNED_ON_IMPORT; |
| |
| /* The allocation could still have active mappings. */ |
| if (user_buf->current_mapping_usage_count == 0) { |
| kbase_jd_user_buf_unmap(kctx, reg->gpu_alloc, |
| (reg->flags & (KBASE_REG_CPU_WR | |
| KBASE_REG_GPU_WR))); |
| } |
| } |
| } |
| /* Fall-through */ |
| default: |
| kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc); |
| break; |
| } |
| |
| return err; |
| } |
| |
| static struct kbase_cpu_mapping *kbasep_find_enclosing_cpu_mapping( |
| struct kbase_context *kctx, |
| unsigned long uaddr, size_t size, u64 *offset) |
| { |
| struct vm_area_struct *vma; |
| struct kbase_cpu_mapping *map; |
| unsigned long vm_pgoff_in_region; |
| unsigned long vm_off_in_region; |
| unsigned long map_start; |
| size_t map_size; |
| |
| lockdep_assert_held(kbase_mem_get_process_mmap_lock()); |
| |
| if ((uintptr_t) uaddr + size < (uintptr_t) uaddr) /* overflow check */ |
| return NULL; |
| |
| vma = find_vma_intersection(current->mm, uaddr, uaddr+size); |
| |
| if (!vma || vma->vm_start > uaddr) |
| return NULL; |
| if (vma->vm_ops != &kbase_vm_ops) |
| /* Not ours! */ |
| return NULL; |
| |
| map = vma->vm_private_data; |
| |
| if (map->kctx != kctx) |
| /* Not from this context! */ |
| return NULL; |
| |
| vm_pgoff_in_region = vma->vm_pgoff - map->region->start_pfn; |
| vm_off_in_region = vm_pgoff_in_region << PAGE_SHIFT; |
| map_start = vma->vm_start - vm_off_in_region; |
| map_size = map->region->nr_pages << PAGE_SHIFT; |
| |
| if ((uaddr + size) > (map_start + map_size)) |
| /* Not within the CPU mapping */ |
| return NULL; |
| |
| *offset = (uaddr - vma->vm_start) + vm_off_in_region; |
| |
| return map; |
| } |
| |
| int kbasep_find_enclosing_cpu_mapping_offset( |
| struct kbase_context *kctx, |
| unsigned long uaddr, size_t size, u64 *offset) |
| { |
| struct kbase_cpu_mapping *map; |
| |
| kbase_os_mem_map_lock(kctx); |
| |
| map = kbasep_find_enclosing_cpu_mapping(kctx, uaddr, size, offset); |
| |
| kbase_os_mem_map_unlock(kctx); |
| |
| if (!map) |
| return -EINVAL; |
| |
| return 0; |
| } |
| |
| KBASE_EXPORT_TEST_API(kbasep_find_enclosing_cpu_mapping_offset); |
| |
| int kbasep_find_enclosing_gpu_mapping_start_and_offset(struct kbase_context *kctx, |
| u64 gpu_addr, size_t size, u64 *start, u64 *offset) |
| { |
| struct kbase_va_region *region; |
| |
| kbase_gpu_vm_lock(kctx); |
| |
| region = kbase_region_tracker_find_region_enclosing_address(kctx, gpu_addr); |
| |
| if (!region) { |
| kbase_gpu_vm_unlock(kctx); |
| return -EINVAL; |
| } |
| |
| *start = region->start_pfn << PAGE_SHIFT; |
| |
| *offset = gpu_addr - *start; |
| |
| if (((region->start_pfn + region->nr_pages) << PAGE_SHIFT) < (gpu_addr + size)) { |
| kbase_gpu_vm_unlock(kctx); |
| return -EINVAL; |
| } |
| |
| kbase_gpu_vm_unlock(kctx); |
| |
| return 0; |
| } |
| |
| KBASE_EXPORT_TEST_API(kbasep_find_enclosing_gpu_mapping_start_and_offset); |
| |
| void kbase_sync_single(struct kbase_context *kctx, |
| struct tagged_addr t_cpu_pa, struct tagged_addr t_gpu_pa, |
| off_t offset, size_t size, enum kbase_sync_type sync_fn) |
| { |
| struct page *cpu_page; |
| phys_addr_t cpu_pa = as_phys_addr_t(t_cpu_pa); |
| phys_addr_t gpu_pa = as_phys_addr_t(t_gpu_pa); |
| |
| cpu_page = pfn_to_page(PFN_DOWN(cpu_pa)); |
| |
| if (likely(cpu_pa == gpu_pa)) { |
| dma_addr_t dma_addr; |
| |
| BUG_ON(!cpu_page); |
| BUG_ON(offset + size > PAGE_SIZE); |
| |
| dma_addr = kbase_dma_addr(cpu_page) + offset; |
| if (sync_fn == KBASE_SYNC_TO_CPU) |
| dma_sync_single_for_cpu(kctx->kbdev->dev, dma_addr, |
| size, DMA_BIDIRECTIONAL); |
| else if (sync_fn == KBASE_SYNC_TO_DEVICE) |
| dma_sync_single_for_device(kctx->kbdev->dev, dma_addr, |
| size, DMA_BIDIRECTIONAL); |
| } else { |
| void *src = NULL; |
| void *dst = NULL; |
| struct page *gpu_page; |
| |
| if (WARN(!gpu_pa, "No GPU PA found for infinite cache op")) |
| return; |
| |
| gpu_page = pfn_to_page(PFN_DOWN(gpu_pa)); |
| |
| if (sync_fn == KBASE_SYNC_TO_DEVICE) { |
| src = ((unsigned char *)kmap(cpu_page)) + offset; |
| dst = ((unsigned char *)kmap(gpu_page)) + offset; |
| } else if (sync_fn == KBASE_SYNC_TO_CPU) { |
| dma_sync_single_for_cpu(kctx->kbdev->dev, |
| kbase_dma_addr(gpu_page) + offset, |
| size, DMA_BIDIRECTIONAL); |
| src = ((unsigned char *)kmap(gpu_page)) + offset; |
| dst = ((unsigned char *)kmap(cpu_page)) + offset; |
| } |
| memcpy(dst, src, size); |
| kunmap(gpu_page); |
| kunmap(cpu_page); |
| if (sync_fn == KBASE_SYNC_TO_DEVICE) |
| dma_sync_single_for_device(kctx->kbdev->dev, |
| kbase_dma_addr(gpu_page) + offset, |
| size, DMA_BIDIRECTIONAL); |
| } |
| } |
| |
| static int kbase_do_syncset(struct kbase_context *kctx, |
| struct basep_syncset *sset, enum kbase_sync_type sync_fn) |
| { |
| int err = 0; |
| struct kbase_va_region *reg; |
| struct kbase_cpu_mapping *map; |
| unsigned long start; |
| size_t size; |
| struct tagged_addr *cpu_pa; |
| struct tagged_addr *gpu_pa; |
| u64 page_off, page_count; |
| u64 i; |
| u64 offset; |
| |
| kbase_os_mem_map_lock(kctx); |
| kbase_gpu_vm_lock(kctx); |
| |
| /* find the region where the virtual address is contained */ |
| reg = kbase_region_tracker_find_region_enclosing_address(kctx, |
| sset->mem_handle.basep.handle); |
| if (kbase_is_region_invalid_or_free(reg)) { |
| dev_warn(kctx->kbdev->dev, "Can't find a valid region at VA 0x%016llX", |
| sset->mem_handle.basep.handle); |
| err = -EINVAL; |
| goto out_unlock; |
| } |
| |
| /* |
| * Handle imported memory before checking for KBASE_REG_CPU_CACHED. The |
| * CPU mapping cacheability is defined by the owner of the imported |
| * memory, and not by kbase, therefore we must assume that any imported |
| * memory may be cached. |
| */ |
| if (kbase_mem_is_imported(reg->gpu_alloc->type)) { |
| err = kbase_mem_do_sync_imported(kctx, reg, sync_fn); |
| goto out_unlock; |
| } |
| |
| if (!(reg->flags & KBASE_REG_CPU_CACHED)) |
| goto out_unlock; |
| |
| start = (uintptr_t)sset->user_addr; |
| size = (size_t)sset->size; |
| |
| map = kbasep_find_enclosing_cpu_mapping(kctx, start, size, &offset); |
| if (!map) { |
| dev_warn(kctx->kbdev->dev, "Can't find CPU mapping 0x%016lX for VA 0x%016llX", |
| start, sset->mem_handle.basep.handle); |
| err = -EINVAL; |
| goto out_unlock; |
| } |
| |
| page_off = offset >> PAGE_SHIFT; |
| offset &= ~PAGE_MASK; |
| page_count = (size + offset + (PAGE_SIZE - 1)) >> PAGE_SHIFT; |
| cpu_pa = kbase_get_cpu_phy_pages(reg); |
| gpu_pa = kbase_get_gpu_phy_pages(reg); |
| |
| if (page_off > reg->nr_pages || |
| page_off + page_count > reg->nr_pages) { |
| /* Sync overflows the region */ |
| err = -EINVAL; |
| goto out_unlock; |
| } |
| |
| /* Sync first page */ |
| if (as_phys_addr_t(cpu_pa[page_off])) { |
| size_t sz = MIN(((size_t) PAGE_SIZE - offset), size); |
| |
| kbase_sync_single(kctx, cpu_pa[page_off], gpu_pa[page_off], |
| offset, sz, sync_fn); |
| } |
| |
| /* Sync middle pages (if any) */ |
| for (i = 1; page_count > 2 && i < page_count - 1; i++) { |
| /* we grow upwards, so bail on first non-present page */ |
| if (!as_phys_addr_t(cpu_pa[page_off + i])) |
| break; |
| |
| kbase_sync_single(kctx, cpu_pa[page_off + i], |
| gpu_pa[page_off + i], 0, PAGE_SIZE, sync_fn); |
| } |
| |
| /* Sync last page (if any) */ |
| if (page_count > 1 && |
| as_phys_addr_t(cpu_pa[page_off + page_count - 1])) { |
| size_t sz = ((start + size - 1) & ~PAGE_MASK) + 1; |
| |
| kbase_sync_single(kctx, cpu_pa[page_off + page_count - 1], |
| gpu_pa[page_off + page_count - 1], 0, sz, |
| sync_fn); |
| } |
| |
| out_unlock: |
| kbase_gpu_vm_unlock(kctx); |
| kbase_os_mem_map_unlock(kctx); |
| return err; |
| } |
| |
| int kbase_sync_now(struct kbase_context *kctx, struct basep_syncset *sset) |
| { |
| int err = -EINVAL; |
| |
| KBASE_DEBUG_ASSERT(kctx != NULL); |
| KBASE_DEBUG_ASSERT(sset != NULL); |
| |
| if (sset->mem_handle.basep.handle & ~PAGE_MASK) { |
| dev_warn(kctx->kbdev->dev, |
| "mem_handle: passed parameter is invalid"); |
| return -EINVAL; |
| } |
| |
| switch (sset->type) { |
| case BASE_SYNCSET_OP_MSYNC: |
| err = kbase_do_syncset(kctx, sset, KBASE_SYNC_TO_DEVICE); |
| break; |
| |
| case BASE_SYNCSET_OP_CSYNC: |
| err = kbase_do_syncset(kctx, sset, KBASE_SYNC_TO_CPU); |
| break; |
| |
| default: |
| dev_warn(kctx->kbdev->dev, "Unknown msync op %d\n", sset->type); |
| break; |
| } |
| |
| return err; |
| } |
| |
| KBASE_EXPORT_TEST_API(kbase_sync_now); |
| |
| /* vm lock must be held */ |
| int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *reg) |
| { |
| int err; |
| |
| KBASE_DEBUG_ASSERT(kctx != NULL); |
| KBASE_DEBUG_ASSERT(reg != NULL); |
| dev_dbg(kctx->kbdev->dev, "%s %pK in kctx %pK\n", |
| __func__, (void *)reg, (void *)kctx); |
| lockdep_assert_held(&kctx->reg_lock); |
| |
| if (reg->flags & KBASE_REG_NO_USER_FREE) { |
| dev_warn(kctx->kbdev->dev, "Attempt to free GPU memory whose freeing by user space is forbidden!\n"); |
| return -EINVAL; |
| } |
| |
| /* |
| * Unlink the physical allocation before unmaking it evictable so |
| * that the allocation isn't grown back to its last backed size |
| * as we're going to unmap it anyway. |
| */ |
| reg->cpu_alloc->reg = NULL; |
| if (reg->cpu_alloc != reg->gpu_alloc) |
| reg->gpu_alloc->reg = NULL; |
| |
| /* |
| * If a region has been made evictable then we must unmake it |
| * before trying to free it. |
| * If the memory hasn't been reclaimed it will be unmapped and freed |
| * below, if it has been reclaimed then the operations below are no-ops. |
| */ |
| if (reg->flags & KBASE_REG_DONT_NEED) { |
| KBASE_DEBUG_ASSERT(reg->cpu_alloc->type == |
| KBASE_MEM_TYPE_NATIVE); |
| kbase_mem_evictable_unmake(reg->gpu_alloc); |
| } |
| |
| err = kbase_gpu_munmap(kctx, reg); |
| if (err) { |
| dev_warn(kctx->kbdev->dev, "Could not unmap from the GPU...\n"); |
| goto out; |
| } |
| |
| /* This will also free the physical pages */ |
| kbase_free_alloced_region(reg); |
| |
| out: |
| return err; |
| } |
| |
| KBASE_EXPORT_TEST_API(kbase_mem_free_region); |
| |
| /** |
| * Free the region from the GPU and unregister it. |
| * @kctx: KBase context |
| * @gpu_addr: GPU address to free |
| * |
| * This function implements the free operation on a memory segment. |
| * It will loudly fail if called with outstanding mappings. |
| */ |
| int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr) |
| { |
| int err = 0; |
| struct kbase_va_region *reg; |
| |
| KBASE_DEBUG_ASSERT(kctx != NULL); |
| dev_dbg(kctx->kbdev->dev, "%s 0x%llx in kctx %pK\n", |
| __func__, gpu_addr, (void *)kctx); |
| |
| if ((gpu_addr & ~PAGE_MASK) && (gpu_addr >= PAGE_SIZE)) { |
| dev_warn(kctx->kbdev->dev, "kbase_mem_free: gpu_addr parameter is invalid"); |
| return -EINVAL; |
| } |
| |
| if (gpu_addr == 0) { |
| dev_warn(kctx->kbdev->dev, "gpu_addr 0 is reserved for the ringbuffer and it's an error to try to free it using kbase_mem_free\n"); |
| return -EINVAL; |
| } |
| kbase_gpu_vm_lock(kctx); |
| |
| if (gpu_addr >= BASE_MEM_COOKIE_BASE && |
| gpu_addr < BASE_MEM_FIRST_FREE_ADDRESS) { |
| int cookie = PFN_DOWN(gpu_addr - BASE_MEM_COOKIE_BASE); |
| |
| reg = kctx->pending_regions[cookie]; |
| if (!reg) { |
| err = -EINVAL; |
| goto out_unlock; |
| } |
| |
| /* ask to unlink the cookie as we'll free it */ |
| |
| kctx->pending_regions[cookie] = NULL; |
| bitmap_set(kctx->cookies, cookie, 1); |
| |
| kbase_free_alloced_region(reg); |
| } else { |
| /* A real GPU va */ |
| /* Validate the region */ |
| reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); |
| if (kbase_is_region_invalid_or_free(reg)) { |
| dev_warn(kctx->kbdev->dev, "kbase_mem_free called with nonexistent gpu_addr 0x%llX", |
| gpu_addr); |
| err = -EINVAL; |
| goto out_unlock; |
| } |
| |
| if ((reg->flags & KBASE_REG_ZONE_MASK) == KBASE_REG_ZONE_SAME_VA) { |
| /* SAME_VA must be freed through munmap */ |
| dev_warn(kctx->kbdev->dev, "%s called on SAME_VA memory 0x%llX", __func__, |
| gpu_addr); |
| err = -EINVAL; |
| goto out_unlock; |
| } |
| err = kbase_mem_free_region(kctx, reg); |
| } |
| |
| out_unlock: |
| kbase_gpu_vm_unlock(kctx); |
| return err; |
| } |
| |
| KBASE_EXPORT_TEST_API(kbase_mem_free); |
| |
| int kbase_update_region_flags(struct kbase_context *kctx, |
| struct kbase_va_region *reg, unsigned long flags) |
| { |
| KBASE_DEBUG_ASSERT(reg != NULL); |
| KBASE_DEBUG_ASSERT((flags & ~((1ul << BASE_MEM_FLAGS_NR_BITS) - 1)) == 0); |
| |
| reg->flags |= kbase_cache_enabled(flags, reg->nr_pages); |
| /* all memory is now growable */ |
| reg->flags |= KBASE_REG_GROWABLE; |
| |
| if (flags & BASE_MEM_GROW_ON_GPF) |
| reg->flags |= KBASE_REG_PF_GROW; |
| |
| if (flags & BASE_MEM_PROT_CPU_WR) |
| reg->flags |= KBASE_REG_CPU_WR; |
| |
| if (flags & BASE_MEM_PROT_CPU_RD) |
| reg->flags |= KBASE_REG_CPU_RD; |
| |
| if (flags & BASE_MEM_PROT_GPU_WR) |
| reg->flags |= KBASE_REG_GPU_WR; |
| |
| if (flags & BASE_MEM_PROT_GPU_RD) |
| reg->flags |= KBASE_REG_GPU_RD; |
| |
| if (0 == (flags & BASE_MEM_PROT_GPU_EX)) |
| reg->flags |= KBASE_REG_GPU_NX; |
| |
| if (!kbase_device_is_cpu_coherent(kctx->kbdev)) { |
| if (flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED && |
| !(flags & BASE_MEM_UNCACHED_GPU)) |
| return -EINVAL; |
| } else if (flags & (BASE_MEM_COHERENT_SYSTEM | |
| BASE_MEM_COHERENT_SYSTEM_REQUIRED)) { |
| reg->flags |= KBASE_REG_SHARE_BOTH; |
| } |
| |
| if (!(reg->flags & KBASE_REG_SHARE_BOTH) && |
| flags & BASE_MEM_COHERENT_LOCAL) { |
| reg->flags |= KBASE_REG_SHARE_IN; |
| } |
| |
| #if !MALI_USE_CSF |
| if (flags & BASE_MEM_TILER_ALIGN_TOP) |
| reg->flags |= KBASE_REG_TILER_ALIGN_TOP; |
| #endif /* !MALI_USE_CSF */ |
| |
| #if MALI_USE_CSF |
| if (flags & BASE_MEM_CSF_EVENT) { |
| reg->flags |= KBASE_REG_CSF_EVENT; |
| reg->flags |= KBASE_REG_PERMANENT_KERNEL_MAPPING; |
| |
| if (!(reg->flags & KBASE_REG_SHARE_BOTH)) { |
| /* On non coherent platforms need to map as uncached on |
| * both sides. |
| */ |
| reg->flags &= ~KBASE_REG_CPU_CACHED; |
| reg->flags &= ~KBASE_REG_GPU_CACHED; |
| } |
| } |
| #endif |
| |
| /* Set up default MEMATTR usage */ |
| if (!(reg->flags & KBASE_REG_GPU_CACHED)) { |
| if (kctx->kbdev->mmu_mode->flags & |
| KBASE_MMU_MODE_HAS_NON_CACHEABLE) { |
| /* Override shareability, and MEMATTR for uncached */ |
| reg->flags &= ~(KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH); |
| reg->flags |= KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_NON_CACHEABLE); |
| } else { |
| dev_warn(kctx->kbdev->dev, |
| "Can't allocate GPU uncached memory due to MMU in Legacy Mode\n"); |
| return -EINVAL; |
| } |
| #if MALI_USE_CSF |
| } else if (reg->flags & KBASE_REG_CSF_EVENT) { |
| WARN_ON(!(reg->flags & KBASE_REG_SHARE_BOTH)); |
| |
| reg->flags |= |
| KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_SHARED); |
| #endif |
| } else if (kctx->kbdev->system_coherency == COHERENCY_ACE && |
| (reg->flags & KBASE_REG_SHARE_BOTH)) { |
| reg->flags |= |
| KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_DEFAULT_ACE); |
| } else { |
| reg->flags |= |
| KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_DEFAULT); |
| } |
| |
| if (flags & BASEP_MEM_PERMANENT_KERNEL_MAPPING) |
| reg->flags |= KBASE_REG_PERMANENT_KERNEL_MAPPING; |
| |
| if (flags & BASEP_MEM_NO_USER_FREE) |
| reg->flags |= KBASE_REG_NO_USER_FREE; |
| |
| if (flags & BASE_MEM_GPU_VA_SAME_4GB_PAGE) |
| reg->flags |= KBASE_REG_GPU_VA_SAME_4GB_PAGE; |
| |
| return 0; |
| } |
| |
| int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, |
| size_t nr_pages_requested) |
| { |
| int new_page_count __maybe_unused; |
| size_t nr_left = nr_pages_requested; |
| int res; |
| struct kbase_context *kctx; |
| struct kbase_device *kbdev; |
| struct tagged_addr *tp; |
| |
| if (WARN_ON(alloc->type != KBASE_MEM_TYPE_NATIVE) || |
| WARN_ON(alloc->imported.native.kctx == NULL) || |
| WARN_ON(alloc->group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) { |
| return -EINVAL; |
| } |
| |
| if (alloc->reg) { |
| if (nr_pages_requested > alloc->reg->nr_pages - alloc->nents) |
| goto invalid_request; |
| } |
| |
| kctx = alloc->imported.native.kctx; |
| kbdev = kctx->kbdev; |
| |
| if (nr_pages_requested == 0) |
| goto done; /*nothing to do*/ |
| |
| new_page_count = atomic_add_return( |
| nr_pages_requested, &kctx->used_pages); |
| atomic_add(nr_pages_requested, |
| &kctx->kbdev->memdev.used_pages); |
| |
| /* Increase mm counters before we allocate pages so that this |
| * allocation is visible to the OOM killer |
| */ |
| kbase_process_page_usage_inc(kctx, nr_pages_requested); |
| |
| tp = alloc->pages + alloc->nents; |
| |
| #ifdef CONFIG_MALI_2MB_ALLOC |
| /* Check if we have enough pages requested so we can allocate a large |
| * page (512 * 4KB = 2MB ) |
| */ |
| if (nr_left >= (SZ_2M / SZ_4K)) { |
| int nr_lp = nr_left / (SZ_2M / SZ_4K); |
| |
| res = kbase_mem_pool_alloc_pages( |
| &kctx->mem_pools.large[alloc->group_id], |
| nr_lp * (SZ_2M / SZ_4K), |
| tp, |
| true); |
| |
| if (res > 0) { |
| nr_left -= res; |
| tp += res; |
| } |
| |
| if (nr_left) { |
| struct kbase_sub_alloc *sa, *temp_sa; |
| |
| spin_lock(&kctx->mem_partials_lock); |
| |
| list_for_each_entry_safe(sa, temp_sa, |
| &kctx->mem_partials, link) { |
| int pidx = 0; |
| |
| while (nr_left) { |
| pidx = find_next_zero_bit(sa->sub_pages, |
| SZ_2M / SZ_4K, |
| pidx); |
| bitmap_set(sa->sub_pages, pidx, 1); |
| *tp++ = as_tagged_tag(page_to_phys(sa->page + |
| pidx), |
| FROM_PARTIAL); |
| nr_left--; |
| |
| if (bitmap_full(sa->sub_pages, SZ_2M / SZ_4K)) { |
| /* unlink from partial list when full */ |
| list_del_init(&sa->link); |
| break; |
| } |
| } |
| } |
| spin_unlock(&kctx->mem_partials_lock); |
| } |
| |
| /* only if we actually have a chunk left <512. If more it indicates |
| * that we couldn't allocate a 2MB above, so no point to retry here. |
| */ |
| if (nr_left > 0 && nr_left < (SZ_2M / SZ_4K)) { |
| /* create a new partial and suballocate the rest from it */ |
| struct page *np = NULL; |
| |
| do { |
| int err; |
| |
| np = kbase_mem_pool_alloc( |
| &kctx->mem_pools.large[ |
| alloc->group_id]); |
| if (np) |
| break; |
| |
| err = kbase_mem_pool_grow( |
| &kctx->mem_pools.large[alloc->group_id], |
| 1); |
| if (err) |
| break; |
| } while (1); |
| |
| if (np) { |
| int i; |
| struct kbase_sub_alloc *sa; |
| struct page *p; |
| |
| sa = kmalloc(sizeof(*sa), GFP_KERNEL); |
| if (!sa) { |
| kbase_mem_pool_free( |
| &kctx->mem_pools.large[ |
| alloc->group_id], |
| np, |
| false); |
| goto no_new_partial; |
| } |
| |
| /* store pointers back to the control struct */ |
| np->lru.next = (void *)sa; |
| for (p = np; p < np + SZ_2M / SZ_4K; p++) |
| p->lru.prev = (void *)np; |
| INIT_LIST_HEAD(&sa->link); |
| bitmap_zero(sa->sub_pages, SZ_2M / SZ_4K); |
| sa->page = np; |
| |
| for (i = 0; i < nr_left; i++) |
| *tp++ = as_tagged_tag(page_to_phys(np + i), FROM_PARTIAL); |
| |
| bitmap_set(sa->sub_pages, 0, nr_left); |
| nr_left = 0; |
| |
| /* expose for later use */ |
| spin_lock(&kctx->mem_partials_lock); |
| list_add(&sa->link, &kctx->mem_partials); |
| spin_unlock(&kctx->mem_partials_lock); |
| } |
| } |
| } |
| no_new_partial: |
| #endif |
| |
| if (nr_left) { |
| res = kbase_mem_pool_alloc_pages( |
| &kctx->mem_pools.small[alloc->group_id], |
| nr_left, tp, false); |
| if (res <= 0) |
| goto alloc_failed; |
| } |
| |
| KBASE_TLSTREAM_AUX_PAGESALLOC( |
| kbdev, |
| kctx->id, |
| (u64)new_page_count); |
| |
| alloc->nents += nr_pages_requested; |
| |
| kbase_trace_gpu_mem_usage_inc(kctx->kbdev, kctx, nr_pages_requested); |
| |
| done: |
| return 0; |
| |
| alloc_failed: |
| /* rollback needed if got one or more 2MB but failed later */ |
| if (nr_left != nr_pages_requested) { |
| size_t nr_pages_to_free = nr_pages_requested - nr_left; |
| |
| alloc->nents += nr_pages_to_free; |
| |
| kbase_process_page_usage_inc(kctx, nr_pages_to_free); |
| atomic_add(nr_pages_to_free, &kctx->used_pages); |
| atomic_add(nr_pages_to_free, |
| &kctx->kbdev->memdev.used_pages); |
| |
| kbase_free_phy_pages_helper(alloc, nr_pages_to_free); |
| } |
| |
| kbase_process_page_usage_dec(kctx, nr_pages_requested); |
| atomic_sub(nr_pages_requested, &kctx->used_pages); |
| atomic_sub(nr_pages_requested, |
| &kctx->kbdev->memdev.used_pages); |
| |
| invalid_request: |
| return -ENOMEM; |
| } |
| |
| struct tagged_addr *kbase_alloc_phy_pages_helper_locked( |
| struct kbase_mem_phy_alloc *alloc, struct kbase_mem_pool *pool, |
| size_t nr_pages_requested, |
| struct kbase_sub_alloc **prealloc_sa) |
| { |
| int new_page_count __maybe_unused; |
| size_t nr_left = nr_pages_requested; |
| int res; |
| struct kbase_context *kctx; |
| struct kbase_device *kbdev; |
| struct tagged_addr *tp; |
| struct tagged_addr *new_pages = NULL; |
| |
| KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_NATIVE); |
| KBASE_DEBUG_ASSERT(alloc->imported.native.kctx); |
| |
| lockdep_assert_held(&pool->pool_lock); |
| |
| #if !defined(CONFIG_MALI_2MB_ALLOC) |
| WARN_ON(pool->order); |
| #endif |
| |
| if (alloc->reg) { |
| if (nr_pages_requested > alloc->reg->nr_pages - alloc->nents) |
| goto invalid_request; |
| } |
| |
| kctx = alloc->imported.native.kctx; |
| kbdev = kctx->kbdev; |
| |
| lockdep_assert_held(&kctx->mem_partials_lock); |
| |
| if (nr_pages_requested == 0) |
| goto done; /*nothing to do*/ |
| |
| new_page_count = atomic_add_return( |
| nr_pages_requested, &kctx->used_pages); |
| atomic_add(nr_pages_requested, |
| &kctx->kbdev->memdev.used_pages); |
| |
| /* Increase mm counters before we allocate pages so that this |
| * allocation is visible to the OOM killer |
| */ |
| kbase_process_page_usage_inc(kctx, nr_pages_requested); |
| |
| tp = alloc->pages + alloc->nents; |
| new_pages = tp; |
| |
| #ifdef CONFIG_MALI_2MB_ALLOC |
| if (pool->order) { |
| int nr_lp = nr_left / (SZ_2M / SZ_4K); |
| |
| res = kbase_mem_pool_alloc_pages_locked(pool, |
| nr_lp * (SZ_2M / SZ_4K), |
| tp); |
| |
| if (res > 0) { |
| nr_left -= res; |
| tp += res; |
| } |
| |
| if (nr_left) { |
| struct kbase_sub_alloc *sa, *temp_sa; |
| |
| list_for_each_entry_safe(sa, temp_sa, |
| &kctx->mem_partials, link) { |
| int pidx = 0; |
| |
| while (nr_left) { |
| pidx = find_next_zero_bit(sa->sub_pages, |
| SZ_2M / SZ_4K, |
| pidx); |
| bitmap_set(sa->sub_pages, pidx, 1); |
| *tp++ = as_tagged_tag(page_to_phys( |
| sa->page + pidx), |
| FROM_PARTIAL); |
| nr_left--; |
| |
| if (bitmap_full(sa->sub_pages, |
| SZ_2M / SZ_4K)) { |
| /* unlink from partial list when |
| * full |
| */ |
| list_del_init(&sa->link); |
| break; |
| } |
| } |
| } |
| } |
| |
| /* only if we actually have a chunk left <512. If more it |
| * indicates that we couldn't allocate a 2MB above, so no point |
| * to retry here. |
| */ |
| if (nr_left > 0 && nr_left < (SZ_2M / SZ_4K)) { |
| /* create a new partial and suballocate the rest from it |
| */ |
| struct page *np = NULL; |
| |
| np = kbase_mem_pool_alloc_locked(pool); |
| |
| if (np) { |
| int i; |
| struct kbase_sub_alloc *const sa = *prealloc_sa; |
| struct page *p; |
| |
| /* store pointers back to the control struct */ |
| np->lru.next = (void *)sa; |
| for (p = np; p < np + SZ_2M / SZ_4K; p++) |
| p->lru.prev = (void *)np; |
| INIT_LIST_HEAD(&sa->link); |
| bitmap_zero(sa->sub_pages, SZ_2M / SZ_4K); |
| sa->page = np; |
| |
| for (i = 0; i < nr_left; i++) |
| *tp++ = as_tagged_tag( |
| page_to_phys(np + i), |
| FROM_PARTIAL); |
| |
| bitmap_set(sa->sub_pages, 0, nr_left); |
| nr_left = 0; |
| /* Indicate to user that we'll free this memory |
| * later. |
| */ |
| *prealloc_sa = NULL; |
| |
| /* expose for later use */ |
| list_add(&sa->link, &kctx->mem_partials); |
| } |
| } |
| if (nr_left) |
| goto alloc_failed; |
| } else { |
| #endif |
| res = kbase_mem_pool_alloc_pages_locked(pool, |
| nr_left, |
| tp); |
| if (res <= 0) |
| goto alloc_failed; |
| #ifdef CONFIG_MALI_2MB_ALLOC |
| } |
| #endif |
| |
| KBASE_TLSTREAM_AUX_PAGESALLOC( |
| kbdev, |
| kctx->id, |
| (u64)new_page_count); |
| |
| alloc->nents += nr_pages_requested; |
| |
| kbase_trace_gpu_mem_usage_inc(kctx->kbdev, kctx, nr_pages_requested); |
| |
| done: |
| return new_pages; |
| |
| alloc_failed: |
| /* rollback needed if got one or more 2MB but failed later */ |
| if (nr_left != nr_pages_requested) { |
| size_t nr_pages_to_free = nr_pages_requested - nr_left; |
| |
| struct tagged_addr *start_free = alloc->pages + alloc->nents; |
| |
| #ifdef CONFIG_MALI_2MB_ALLOC |
| if (pool->order) { |
| while (nr_pages_to_free) { |
| if (is_huge_head(*start_free)) { |
| kbase_mem_pool_free_pages_locked( |
| pool, 512, |
| start_free, |
| false, /* not dirty */ |
| true); /* return to pool */ |
| nr_pages_to_free -= 512; |
| start_free += 512; |
| } else if (is_partial(*start_free)) { |
| free_partial_locked(kctx, pool, |
| *start_free); |
| nr_pages_to_free--; |
| start_free++; |
| } |
| } |
| } else { |
| #endif |
| kbase_mem_pool_free_pages_locked(pool, |
| nr_pages_to_free, |
| start_free, |
| false, /* not dirty */ |
| true); /* return to pool */ |
| #ifdef CONFIG_MALI_2MB_ALLOC |
| } |
| #endif |
| } |
| |
| kbase_process_page_usage_dec(kctx, nr_pages_requested); |
| atomic_sub(nr_pages_requested, &kctx->used_pages); |
| atomic_sub(nr_pages_requested, &kctx->kbdev->memdev.used_pages); |
| |
| invalid_request: |
| return NULL; |
| } |
| |
| static void free_partial(struct kbase_context *kctx, int group_id, struct |
| tagged_addr tp) |
| { |
| struct page *p, *head_page; |
| struct kbase_sub_alloc *sa; |
| |
| p = as_page(tp); |
| head_page = (struct page *)p->lru.prev; |
| sa = (struct kbase_sub_alloc *)head_page->lru.next; |
| spin_lock(&kctx->mem_partials_lock); |
| clear_bit(p - head_page, sa->sub_pages); |
| if (bitmap_empty(sa->sub_pages, SZ_2M / SZ_4K)) { |
| list_del(&sa->link); |
| kbase_mem_pool_free( |
| &kctx->mem_pools.large[group_id], |
| head_page, |
| true); |
| kfree(sa); |
| } else if (bitmap_weight(sa->sub_pages, SZ_2M / SZ_4K) == |
| SZ_2M / SZ_4K - 1) { |
| /* expose the partial again */ |
| list_add(&sa->link, &kctx->mem_partials); |
| } |
| spin_unlock(&kctx->mem_partials_lock); |
| } |
| |
| int kbase_free_phy_pages_helper( |
| struct kbase_mem_phy_alloc *alloc, |
| size_t nr_pages_to_free) |
| { |
| struct kbase_context *kctx = alloc->imported.native.kctx; |
| struct kbase_device *kbdev = kctx->kbdev; |
| bool syncback; |
| bool reclaimed = (alloc->evicted != 0); |
| struct tagged_addr *start_free; |
| int new_page_count __maybe_unused; |
| size_t freed = 0; |
| |
| if (WARN_ON(alloc->type != KBASE_MEM_TYPE_NATIVE) || |
| WARN_ON(alloc->imported.native.kctx == NULL) || |
| WARN_ON(alloc->nents < nr_pages_to_free) || |
| WARN_ON(alloc->group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) { |
| return -EINVAL; |
| } |
| |
| /* early out if nothing to do */ |
| if (nr_pages_to_free == 0) |
| return 0; |
| |
| start_free = alloc->pages + alloc->nents - nr_pages_to_free; |
| |
| syncback = alloc->properties & KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED; |
| |
| /* pad start_free to a valid start location */ |
| while (nr_pages_to_free && is_huge(*start_free) && |
| !is_huge_head(*start_free)) { |
| nr_pages_to_free--; |
| start_free++; |
| } |
| |
| while (nr_pages_to_free) { |
| if (is_huge_head(*start_free)) { |
| /* This is a 2MB entry, so free all the 512 pages that |
| * it points to |
| */ |
| kbase_mem_pool_free_pages( |
| &kctx->mem_pools.large[alloc->group_id], |
| 512, |
| start_free, |
| syncback, |
| reclaimed); |
| nr_pages_to_free -= 512; |
| start_free += 512; |
| freed += 512; |
| } else if (is_partial(*start_free)) { |
| free_partial(kctx, alloc->group_id, *start_free); |
| nr_pages_to_free--; |
| start_free++; |
| freed++; |
| } else { |
| struct tagged_addr *local_end_free; |
| |
| local_end_free = start_free; |
| while (nr_pages_to_free && |
| !is_huge(*local_end_free) && |
| !is_partial(*local_end_free)) { |
| local_end_free++; |
| nr_pages_to_free--; |
| } |
| kbase_mem_pool_free_pages( |
| &kctx->mem_pools.small[alloc->group_id], |
| local_end_free - start_free, |
| start_free, |
| syncback, |
| reclaimed); |
| freed += local_end_free - start_free; |
| start_free += local_end_free - start_free; |
| } |
| } |
| |
| alloc->nents -= freed; |
| |
| /* |
| * If the allocation was not evicted (i.e. evicted == 0) then |
| * the page accounting needs to be done. |
| */ |
| if (!reclaimed) { |
| kbase_process_page_usage_dec(kctx, freed); |
| new_page_count = atomic_sub_return(freed, |
| &kctx->used_pages); |
| atomic_sub(freed, |
| &kctx->kbdev->memdev.used_pages); |
| |
| KBASE_TLSTREAM_AUX_PAGESALLOC( |
| kbdev, |
| kctx->id, |
| (u64)new_page_count); |
| |
| kbase_trace_gpu_mem_usage_dec(kctx->kbdev, kctx, freed); |
| } |
| |
| return 0; |
| } |
| |
| static void free_partial_locked(struct kbase_context *kctx, |
| struct kbase_mem_pool *pool, struct tagged_addr tp) |
| { |
| struct page *p, *head_page; |
| struct kbase_sub_alloc *sa; |
| |
| lockdep_assert_held(&pool->pool_lock); |
| lockdep_assert_held(&kctx->mem_partials_lock); |
| |
| p = as_page(tp); |
| head_page = (struct page *)p->lru.prev; |
| sa = (struct kbase_sub_alloc *)head_page->lru.next; |
| clear_bit(p - head_page, sa->sub_pages); |
| if (bitmap_empty(sa->sub_pages, SZ_2M / SZ_4K)) { |
| list_del(&sa->link); |
| kbase_mem_pool_free_locked(pool, head_page, true); |
| kfree(sa); |
| } else if (bitmap_weight(sa->sub_pages, SZ_2M / SZ_4K) == |
| SZ_2M / SZ_4K - 1) { |
| /* expose the partial again */ |
| list_add(&sa->link, &kctx->mem_partials); |
| } |
| } |
| |
| void kbase_free_phy_pages_helper_locked(struct kbase_mem_phy_alloc *alloc, |
| struct kbase_mem_pool *pool, struct tagged_addr *pages, |
| size_t nr_pages_to_free) |
| { |
| struct kbase_context *kctx = alloc->imported.native.kctx; |
| struct kbase_device *kbdev = kctx->kbdev; |
| bool syncback; |
| bool reclaimed = (alloc->evicted != 0); |
| struct tagged_addr *start_free; |
| size_t freed = 0; |
| |
| KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_NATIVE); |
| KBASE_DEBUG_ASSERT(alloc->imported.native.kctx); |
| KBASE_DEBUG_ASSERT(alloc->nents >= nr_pages_to_free); |
| |
| lockdep_assert_held(&pool->pool_lock); |
| lockdep_assert_held(&kctx->mem_partials_lock); |
| |
| /* early out if nothing to do */ |
| if (!nr_pages_to_free) |
| return; |
| |
| start_free = pages; |
| |
| syncback = alloc->properties & KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED; |
| |
| /* pad start_free to a valid start location */ |
| while (nr_pages_to_free && is_huge(*start_free) && |
| !is_huge_head(*start_free)) { |
| nr_pages_to_free--; |
| start_free++; |
| } |
| |
| while (nr_pages_to_free) { |
| if (is_huge_head(*start_free)) { |
| /* This is a 2MB entry, so free all the 512 pages that |
| * it points to |
| */ |
| WARN_ON(!pool->order); |
| kbase_mem_pool_free_pages_locked(pool, |
| 512, |
| start_free, |
| syncback, |
| reclaimed); |
| nr_pages_to_free -= 512; |
| start_free += 512; |
| freed += 512; |
| } else if (is_partial(*start_free)) { |
| WARN_ON(!pool->order); |
| free_partial_locked(kctx, pool, *start_free); |
| nr_pages_to_free--; |
| start_free++; |
| freed++; |
| } else { |
| struct tagged_addr *local_end_free; |
| |
| WARN_ON(pool->order); |
| local_end_free = start_free; |
| while (nr_pages_to_free && |
| !is_huge(*local_end_free) && |
| !is_partial(*local_end_free)) { |
| local_end_free++; |
| nr_pages_to_free--; |
| } |
| kbase_mem_pool_free_pages_locked(pool, |
| local_end_free - start_free, |
| start_free, |
| syncback, |
| reclaimed); |
| freed += local_end_free - start_free; |
| start_free += local_end_free - start_free; |
| } |
| } |
| |
| alloc->nents -= freed; |
| |
| /* |
| * If the allocation was not evicted (i.e. evicted == 0) then |
| * the page accounting needs to be done. |
| */ |
| if (!reclaimed) { |
| int new_page_count; |
| |
| kbase_process_page_usage_dec(kctx, freed); |
| new_page_count = atomic_sub_return(freed, |
| &kctx->used_pages); |
| atomic_sub(freed, |
| &kctx->kbdev->memdev.used_pages); |
| |
| KBASE_TLSTREAM_AUX_PAGESALLOC( |
| kbdev, |
| kctx->id, |
| (u64)new_page_count); |
| |
| kbase_trace_gpu_mem_usage_dec(kctx->kbdev, kctx, freed); |
| } |
| } |
| KBASE_EXPORT_TEST_API(kbase_free_phy_pages_helper_locked); |
| |
| #if MALI_USE_CSF |
| /** |
| * kbase_jd_user_buf_unpin_pages - Release the pinned pages of a user buffer. |
| * @alloc: The allocation for the imported user buffer. |
| */ |
| static void kbase_jd_user_buf_unpin_pages(struct kbase_mem_phy_alloc *alloc); |
| #endif |
| |
| void kbase_mem_kref_free(struct kref *kref) |
| { |
| struct kbase_mem_phy_alloc *alloc; |
| |
| alloc = container_of(kref, struct kbase_mem_phy_alloc, kref); |
| |
| switch (alloc->type) { |
| case KBASE_MEM_TYPE_NATIVE: { |
| |
| if (!WARN_ON(!alloc->imported.native.kctx)) { |
| if (alloc->permanent_map) |
| kbase_phy_alloc_mapping_term( |
| alloc->imported.native.kctx, |
| alloc); |
| |
| /* |
| * The physical allocation must have been removed from |
| * the eviction list before trying to free it. |
| */ |
| mutex_lock( |
| &alloc->imported.native.kctx->jit_evict_lock); |
| WARN_ON(!list_empty(&alloc->evict_node)); |
| mutex_unlock( |
| &alloc->imported.native.kctx->jit_evict_lock); |
| |
| kbase_process_page_usage_dec( |
| alloc->imported.native.kctx, |
| alloc->imported.native.nr_struct_pages); |
| } |
| kbase_free_phy_pages_helper(alloc, alloc->nents); |
| break; |
| } |
| case KBASE_MEM_TYPE_ALIAS: { |
| /* just call put on the underlying phy allocs */ |
| size_t i; |
| struct kbase_aliased *aliased; |
| |
| aliased = alloc->imported.alias.aliased; |
| if (aliased) { |
| for (i = 0; i < alloc->imported.alias.nents; i++) |
| if (aliased[i].alloc) { |
| kbase_mem_phy_alloc_gpu_unmapped(aliased[i].alloc); |
| kbase_mem_phy_alloc_put(aliased[i].alloc); |
| } |
| vfree(aliased); |
| } |
| break; |
| } |
| case KBASE_MEM_TYPE_RAW: |
| /* raw pages, external cleanup */ |
| break; |
| case KBASE_MEM_TYPE_IMPORTED_UMM: |
| if (!IS_ENABLED(CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND)) { |
| WARN_ONCE(alloc->imported.umm.current_mapping_usage_count != 1, |
| "WARNING: expected excatly 1 mapping, got %d", |
| alloc->imported.umm.current_mapping_usage_count); |
| dma_buf_unmap_attachment( |
| alloc->imported.umm.dma_attachment, |
| alloc->imported.umm.sgt, |
| DMA_BIDIRECTIONAL); |
| kbase_remove_dma_buf_usage(alloc->imported.umm.kctx, |
| alloc); |
| } |
| dma_buf_detach(alloc->imported.umm.dma_buf, |
| alloc->imported.umm.dma_attachment); |
| dma_buf_put(alloc->imported.umm.dma_buf); |
| break; |
| case KBASE_MEM_TYPE_IMPORTED_USER_BUF: |
| #if MALI_USE_CSF |
| kbase_jd_user_buf_unpin_pages(alloc); |
| #endif |
| if (alloc->imported.user_buf.mm) |
| mmdrop(alloc->imported.user_buf.mm); |
| if (alloc->properties & KBASE_MEM_PHY_ALLOC_LARGE) |
| vfree(alloc->imported.user_buf.pages); |
| else |
| kfree(alloc->imported.user_buf.pages); |
| break; |
| default: |
| WARN(1, "Unexecpted free of type %d\n", alloc->type); |
| break; |
| } |
| |
| /* Free based on allocation type */ |
| if (alloc->properties & KBASE_MEM_PHY_ALLOC_LARGE) |
| vfree(alloc); |
| else |
| kfree(alloc); |
| } |
| |
| KBASE_EXPORT_TEST_API(kbase_mem_kref_free); |
| |
| int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size) |
| { |
| KBASE_DEBUG_ASSERT(reg != NULL); |
| KBASE_DEBUG_ASSERT(vsize > 0); |
| |
| /* validate user provided arguments */ |
| if (size > vsize || vsize > reg->nr_pages) |
| goto out_term; |
| |
| /* Prevent vsize*sizeof from wrapping around. |
| * For instance, if vsize is 2**29+1, we'll allocate 1 byte and the alloc won't fail. |
| */ |
| if ((size_t) vsize > ((size_t) -1 / sizeof(*reg->cpu_alloc->pages))) |
| goto out_term; |
| |
| KBASE_DEBUG_ASSERT(vsize != 0); |
| |
| if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, size) != 0) |
| goto out_term; |
| |
| reg->cpu_alloc->reg = reg; |
| if (reg->cpu_alloc != reg->gpu_alloc) { |
| if (kbase_alloc_phy_pages_helper(reg->gpu_alloc, size) != 0) |
| goto out_rollback; |
| reg->gpu_alloc->reg = reg; |
| } |
| |
| return 0; |
| |
| out_rollback: |
| kbase_free_phy_pages_helper(reg->cpu_alloc, size); |
| out_term: |
| return -1; |
| } |
| |
| KBASE_EXPORT_TEST_API(kbase_alloc_phy_pages); |
| |
| bool kbase_check_alloc_flags(unsigned long flags) |
| { |
| /* Only known input flags should be set. */ |
| if (flags & ~BASE_MEM_FLAGS_INPUT_MASK) |
| return false; |
| |
| /* At least one flag should be set */ |
| if (flags == 0) |
| return false; |
| |
| /* Either the GPU or CPU must be reading from the allocated memory */ |
| if ((flags & (BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_RD)) == 0) |
| return false; |
| |
| /* Either the GPU or CPU must be writing to the allocated memory */ |
| if ((flags & (BASE_MEM_PROT_CPU_WR | BASE_MEM_PROT_GPU_WR)) == 0) |
| return false; |
| |
| /* GPU executable memory cannot: |
| * - Be written by the GPU |
| * - Be grown on GPU page fault |
| */ |
| if ((flags & BASE_MEM_PROT_GPU_EX) && (flags & |
| (BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF))) |
| return false; |
| |
| #if !MALI_USE_CSF |
| /* GPU executable memory also cannot have the top of its initial |
| * commit aligned to 'extension' |
| */ |
| if ((flags & BASE_MEM_PROT_GPU_EX) && (flags & |
| BASE_MEM_TILER_ALIGN_TOP)) |
| return false; |
| #endif /* !MALI_USE_CSF */ |
| |
| /* To have an allocation lie within a 4GB chunk is required only for |
| * TLS memory, which will never be used to contain executable code. |
| */ |
| if ((flags & BASE_MEM_GPU_VA_SAME_4GB_PAGE) && (flags & |
| BASE_MEM_PROT_GPU_EX)) |
| return false; |
| |
| #if !MALI_USE_CSF |
| /* TLS memory should also not be used for tiler heap */ |
| if ((flags & BASE_MEM_GPU_VA_SAME_4GB_PAGE) && (flags & |
| BASE_MEM_TILER_ALIGN_TOP)) |
| return false; |
| #endif /* !MALI_USE_CSF */ |
| |
| /* GPU should have at least read or write access otherwise there is no |
| * reason for allocating. |
| */ |
| if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0) |
| return false; |
| |
| /* BASE_MEM_IMPORT_SHARED is only valid for imported memory */ |
| if ((flags & BASE_MEM_IMPORT_SHARED) == BASE_MEM_IMPORT_SHARED) |
| return false; |
| |
| /* BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP is only valid for imported memory |
| */ |
| if ((flags & BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP) == |
| BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP) |
| return false; |
| |
| /* Should not combine BASE_MEM_COHERENT_LOCAL with |
| * BASE_MEM_COHERENT_SYSTEM |
| */ |
| if ((flags & (BASE_MEM_COHERENT_LOCAL | BASE_MEM_COHERENT_SYSTEM)) == |
| (BASE_MEM_COHERENT_LOCAL | BASE_MEM_COHERENT_SYSTEM)) |
| return false; |
| |
| return true; |
| } |
| |
| bool kbase_check_import_flags(unsigned long flags) |
| { |
| /* Only known input flags should be set. */ |
| if (flags & ~BASE_MEM_FLAGS_INPUT_MASK) |
| return false; |
| |
| /* At least one flag should be set */ |
| if (flags == 0) |
| return false; |
| |
| /* Imported memory cannot be GPU executable */ |
| if (flags & BASE_MEM_PROT_GPU_EX) |
| return false; |
| |
| /* Imported memory cannot grow on page fault */ |
| if (flags & BASE_MEM_GROW_ON_GPF) |
| return false; |
| |
| #if !MALI_USE_CSF |
| /* Imported memory cannot be aligned to the end of its initial commit */ |
| if (flags & BASE_MEM_TILER_ALIGN_TOP) |
| return false; |
| #endif /* !MALI_USE_CSF */ |
| |
| /* GPU should have at least read or write access otherwise there is no |
| * reason for importing. |
| */ |
| if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0) |
| return false; |
| |
| /* Protected memory cannot be read by the CPU */ |
| if ((flags & BASE_MEM_PROTECTED) && (flags & BASE_MEM_PROT_CPU_RD)) |
| return false; |
| |
| return true; |
| } |
| |
| int kbase_check_alloc_sizes(struct kbase_context *kctx, unsigned long flags, |
| u64 va_pages, u64 commit_pages, u64 large_extension) |
| { |
| struct device *dev = kctx->kbdev->dev; |
| int gpu_pc_bits = kctx->kbdev->gpu_props.props.core_props.log2_program_counter_size; |
| u64 gpu_pc_pages_max = 1ULL << gpu_pc_bits >> PAGE_SHIFT; |
| struct kbase_va_region test_reg; |
| |
| /* kbase_va_region's extension member can be of variable size, so check against that type */ |
| test_reg.extension = large_extension; |
| |
| #define KBASE_MSG_PRE "GPU allocation attempted with " |
| |
| if (va_pages == 0) { |
| dev_warn(dev, KBASE_MSG_PRE "0 va_pages!"); |
| return -EINVAL; |
| } |
| |
| if (va_pages > KBASE_MEM_ALLOC_MAX_SIZE) { |
| dev_warn(dev, KBASE_MSG_PRE "va_pages==%lld larger than KBASE_MEM_ALLOC_MAX_SIZE!", |
| (unsigned long long)va_pages); |
| return -ENOMEM; |
| } |
| |
| /* Note: commit_pages is checked against va_pages during |
| * kbase_alloc_phy_pages() |
| */ |
| |
| /* Limit GPU executable allocs to GPU PC size */ |
| if ((flags & BASE_MEM_PROT_GPU_EX) && (va_pages > gpu_pc_pages_max)) { |
| dev_warn(dev, KBASE_MSG_PRE "BASE_MEM_PROT_GPU_EX and va_pages==%lld larger than GPU PC range %lld", |
| (unsigned long long)va_pages, |
| (unsigned long long)gpu_pc_pages_max); |
| |
| return -EINVAL; |
| } |
| |
| if ((flags & BASE_MEM_GROW_ON_GPF) && (test_reg.extension == 0)) { |
| dev_warn(dev, KBASE_MSG_PRE |
| "BASE_MEM_GROW_ON_GPF but extension == 0\n"); |
| return -EINVAL; |
| } |
| |
| #if !MALI_USE_CSF |
| if ((flags & BASE_MEM_TILER_ALIGN_TOP) && (test_reg.extension == 0)) { |
| dev_warn(dev, KBASE_MSG_PRE |
| "BASE_MEM_TILER_ALIGN_TOP but extension == 0\n"); |
| return -EINVAL; |
| } |
| |
| if (!(flags & (BASE_MEM_GROW_ON_GPF | BASE_MEM_TILER_ALIGN_TOP)) && |
| test_reg.extension != 0) { |
| dev_warn( |
| dev, KBASE_MSG_PRE |
| "neither BASE_MEM_GROW_ON_GPF nor BASE_MEM_TILER_ALIGN_TOP set but extension != 0\n"); |
| return -EINVAL; |
| } |
| #else |
| if (!(flags & BASE_MEM_GROW_ON_GPF) && test_reg.extension != 0) { |
| dev_warn(dev, KBASE_MSG_PRE |
| "BASE_MEM_GROW_ON_GPF not set but extension != 0\n"); |
| return -EINVAL; |
| } |
| #endif /* !MALI_USE_CSF */ |
| |
| #if !MALI_USE_CSF |
| /* BASE_MEM_TILER_ALIGN_TOP memory has a number of restrictions */ |
| if (flags & BASE_MEM_TILER_ALIGN_TOP) { |
| #define KBASE_MSG_PRE_FLAG KBASE_MSG_PRE "BASE_MEM_TILER_ALIGN_TOP and " |
| unsigned long small_extension; |
| |
| if (large_extension > |
| BASE_MEM_TILER_ALIGN_TOP_EXTENSION_MAX_PAGES) { |
| dev_warn(dev, |
| KBASE_MSG_PRE_FLAG |
| "extension==%lld pages exceeds limit %lld", |
| (unsigned long long)large_extension, |
| BASE_MEM_TILER_ALIGN_TOP_EXTENSION_MAX_PAGES); |
| return -EINVAL; |
| } |
| /* For use with is_power_of_2, which takes unsigned long, so |
| * must ensure e.g. on 32-bit kernel it'll fit in that type |
| */ |
| small_extension = (unsigned long)large_extension; |
| |
| if (!is_power_of_2(small_extension)) { |
| dev_warn(dev, |
| KBASE_MSG_PRE_FLAG |
| "extension==%ld not a non-zero power of 2", |
| small_extension); |
| return -EINVAL; |
| } |
| |
| if (commit_pages > large_extension) { |
| dev_warn(dev, |
| KBASE_MSG_PRE_FLAG |
| "commit_pages==%ld exceeds extension==%ld", |
| (unsigned long)commit_pages, |
| (unsigned long)large_extension); |
| return -EINVAL; |
| } |
| #undef KBASE_MSG_PRE_FLAG |
| } |
| #endif /* !MALI_USE_CSF */ |
| |
| if ((flags & BASE_MEM_GPU_VA_SAME_4GB_PAGE) && |
| (va_pages > (BASE_MEM_PFN_MASK_4GB + 1))) { |
| dev_warn(dev, KBASE_MSG_PRE "BASE_MEM_GPU_VA_SAME_4GB_PAGE and va_pages==%lld greater than that needed for 4GB space", |
| (unsigned long long)va_pages); |
| return -EINVAL; |
| } |
| |
| return 0; |
| #undef KBASE_MSG_PRE |
| } |
| |
| /** |
| * Acquire the per-context region list lock |
| * @kctx: KBase context |
| */ |
| void kbase_gpu_vm_lock(struct kbase_context *kctx) |
| { |
| KBASE_DEBUG_ASSERT(kctx != NULL); |
| mutex_lock(&kctx->reg_lock); |
| } |
| |
| KBASE_EXPORT_TEST_API(kbase_gpu_vm_lock); |
| |
| /** |
| * Release the per-context region list lock |
| * @kctx: KBase context |
| */ |
| void kbase_gpu_vm_unlock(struct kbase_context *kctx) |
| { |
| KBASE_DEBUG_ASSERT(kctx != NULL); |
| mutex_unlock(&kctx->reg_lock); |
| } |
| |
| KBASE_EXPORT_TEST_API(kbase_gpu_vm_unlock); |
| |
| #if IS_ENABLED(CONFIG_DEBUG_FS) |
| struct kbase_jit_debugfs_data { |
| int (*func)(struct kbase_jit_debugfs_data *); |
| struct mutex lock; |
| struct kbase_context *kctx; |
| u64 active_value; |
| u64 pool_value; |
| u64 destroy_value; |
| char buffer[50]; |
| }; |
| |
| static int kbase_jit_debugfs_common_open(struct inode *inode, |
| struct file *file, int (*func)(struct kbase_jit_debugfs_data *)) |
| { |
| struct kbase_jit_debugfs_data *data; |
| |
| data = kzalloc(sizeof(*data), GFP_KERNEL); |
| if (!data) |
| return -ENOMEM; |
| |
| data->func = func; |
| mutex_init(&data->lock); |
| data->kctx = (struct kbase_context *) inode->i_private; |
| |
| file->private_data = data; |
| |
| return nonseekable_open(inode, file); |
| } |
| |
| static ssize_t kbase_jit_debugfs_common_read(struct file *file, |
|