blob: 7c34a1ba48bba38176d164693450882aa2034b0c [file] [log] [blame]
// SPDX-License-Identifier: GPL-2.0
/*
* DMA BUF page pool system
*
* Copyright (C) 2020 Linaro Ltd.
*
* Based on the ION page pool code
* Copyright (C) 2011 Google, Inc.
*/
#include <linux/freezer.h>
#include <linux/list.h>
#include <linux/slab.h>
#include <linux/swap.h>
#include <linux/sched/signal.h>
#include "page_pool.h"
static LIST_HEAD(pool_list);
static DEFINE_MUTEX(pool_list_lock);
static inline
struct page *dmabuf_page_pool_alloc_pages(struct dmabuf_page_pool *pool)
{
if (fatal_signal_pending(current))
return NULL;
return alloc_pages(pool->gfp_mask, pool->order);
}
static inline void dmabuf_page_pool_free_pages(struct dmabuf_page_pool *pool,
struct page *page)
{
__free_pages(page, pool->order);
}
static void dmabuf_page_pool_add(struct dmabuf_page_pool *pool, struct page *page)
{
int index;
if (PageHighMem(page))
index = POOL_HIGHPAGE;
else
index = POOL_LOWPAGE;
mutex_lock(&pool->mutex);
list_add_tail(&page->lru, &pool->items[index]);
pool->count[index]++;
mutex_unlock(&pool->mutex);
mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE,
1 << pool->order);
}
static struct page *dmabuf_page_pool_remove(struct dmabuf_page_pool *pool, int index)
{
struct page *page;
mutex_lock(&pool->mutex);
page = list_first_entry_or_null(&pool->items[index], struct page, lru);
if (page) {
pool->count[index]--;
list_del(&page->lru);
mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE,
-(1 << pool->order));
}
mutex_unlock(&pool->mutex);
return page;
}
static struct page *dmabuf_page_pool_fetch(struct dmabuf_page_pool *pool)
{
struct page *page = NULL;
page = dmabuf_page_pool_remove(pool, POOL_HIGHPAGE);
if (!page)
page = dmabuf_page_pool_remove(pool, POOL_LOWPAGE);
return page;
}
struct page *dmabuf_page_pool_alloc(struct dmabuf_page_pool *pool)
{
struct page *page = NULL;
if (WARN_ON(!pool))
return NULL;
page = dmabuf_page_pool_fetch(pool);
if (!page)
page = dmabuf_page_pool_alloc_pages(pool);
return page;
}
EXPORT_SYMBOL_GPL(dmabuf_page_pool_alloc);
void dmabuf_page_pool_free(struct dmabuf_page_pool *pool, struct page *page)
{
if (WARN_ON(pool->order != compound_order(page)))
return;
dmabuf_page_pool_add(pool, page);
}
EXPORT_SYMBOL_GPL(dmabuf_page_pool_free);
static int dmabuf_page_pool_total(struct dmabuf_page_pool *pool, bool high)
{
int count = pool->count[POOL_LOWPAGE];
if (high)
count += pool->count[POOL_HIGHPAGE];
return count << pool->order;
}
struct dmabuf_page_pool *dmabuf_page_pool_create(gfp_t gfp_mask, unsigned int order)
{
struct dmabuf_page_pool *pool = kmalloc(sizeof(*pool), GFP_KERNEL);
int i;
if (!pool)
return NULL;
for (i = 0; i < POOL_TYPE_SIZE; i++) {
pool->count[i] = 0;
INIT_LIST_HEAD(&pool->items[i]);
}
pool->gfp_mask = gfp_mask | __GFP_COMP;
pool->order = order;
mutex_init(&pool->mutex);
mutex_lock(&pool_list_lock);
list_add(&pool->list, &pool_list);
mutex_unlock(&pool_list_lock);
return pool;
}
EXPORT_SYMBOL_GPL(dmabuf_page_pool_create);
void dmabuf_page_pool_destroy(struct dmabuf_page_pool *pool)
{
struct page *page;
int i;
/* Remove us from the pool list */
mutex_lock(&pool_list_lock);
list_del(&pool->list);
mutex_unlock(&pool_list_lock);
/* Free any remaining pages in the pool */
for (i = 0; i < POOL_TYPE_SIZE; i++) {
while ((page = dmabuf_page_pool_remove(pool, i)))
dmabuf_page_pool_free_pages(pool, page);
}
kfree(pool);
}
EXPORT_SYMBOL_GPL(dmabuf_page_pool_destroy);
static int dmabuf_page_pool_do_shrink(struct dmabuf_page_pool *pool, gfp_t gfp_mask,
int nr_to_scan)
{
int freed = 0;
bool high;
if (current_is_kswapd())
high = true;
else
high = !!(gfp_mask & __GFP_HIGHMEM);
if (nr_to_scan == 0)
return dmabuf_page_pool_total(pool, high);
while (freed < nr_to_scan) {
struct page *page;
/* Try to free low pages first */
page = dmabuf_page_pool_remove(pool, POOL_LOWPAGE);
if (!page)
page = dmabuf_page_pool_remove(pool, POOL_HIGHPAGE);
if (!page)
break;
dmabuf_page_pool_free_pages(pool, page);
freed += (1 << pool->order);
}
return freed;
}
static int dmabuf_page_pool_shrink(gfp_t gfp_mask, int nr_to_scan)
{
struct dmabuf_page_pool *pool;
int nr_total = 0;
int nr_freed;
int only_scan = 0;
if (!nr_to_scan)
only_scan = 1;
mutex_lock(&pool_list_lock);
list_for_each_entry(pool, &pool_list, list) {
if (only_scan) {
nr_total += dmabuf_page_pool_do_shrink(pool,
gfp_mask,
nr_to_scan);
} else {
nr_freed = dmabuf_page_pool_do_shrink(pool,
gfp_mask,
nr_to_scan);
nr_to_scan -= nr_freed;
nr_total += nr_freed;
if (nr_to_scan <= 0)
break;
}
}
mutex_unlock(&pool_list_lock);
return nr_total;
}
static unsigned long dmabuf_page_pool_shrink_count(struct shrinker *shrinker,
struct shrink_control *sc)
{
return dmabuf_page_pool_shrink(sc->gfp_mask, 0);
}
static unsigned long dmabuf_page_pool_shrink_scan(struct shrinker *shrinker,
struct shrink_control *sc)
{
if (sc->nr_to_scan == 0)
return 0;
return dmabuf_page_pool_shrink(sc->gfp_mask, sc->nr_to_scan);
}
struct shrinker pool_shrinker = {
.count_objects = dmabuf_page_pool_shrink_count,
.scan_objects = dmabuf_page_pool_shrink_scan,
.seeks = DEFAULT_SEEKS,
.batch = 0,
};
static int dmabuf_page_pool_init_shrinker(void)
{
return register_shrinker(&pool_shrinker);
}
module_init(dmabuf_page_pool_init_shrinker);
MODULE_LICENSE("GPL v2");