ANDROID: cma: Add restrict_cma_redirect boot parameter

Commit "mm,page_alloc,cma: conditionally prefer cma pageblocks for
movable allocations" (16867664936e) introduced balancing of movable
allocations between CMA and normal areas.

Commit "ANDROID: cma: redirect page allocation to CMA" (f60c5572d20a)
removes it, making allocations go in CMA area first.

1. Reintroduce the condition, so that CMA and normal area are used
in a balanced way(as it used to be), so it prevents depleting of CMA
region;

2. Back-port a command line option(from 6.6), "restrict_cma_redirect",
that can be used if only MOVABLE allocations marked as __GFP_CMA are
eligible to be redirected to CMA region. By default it is true.

The purpose of this change is to keep using CMA for movable allocations,
but at the same time, to have enough free CMA pages for critical system
areas such as modem initialization, GPU initialization and so on.

Bug: 381168812
Signed-off-by: Sebastian Achim <sebastian.1.achim@sony.com>
Signed-off-by: Uladzislau Rezki <uladzislau.rezki@sony.com>
Signed-off-by: Oleksiy Avramchenko <oleksiy.avramchenko@sony.com>
Change-Id: I5fd6d022340715e27754c687189c5ea0e56d9ee6

mm/page_alloc.c

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index f33cb1c..28004c5 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -211,6 +211,38 @@ EXPORT_PER_CPU_SYMBOL(numa_node);
 
 DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key);
 
+/*
+ * By default, restrict_cma_redirect is set to true, so only MOVABLE allocations
+ * marked __GFP_CMA are eligible to be redirected to CMA region. These allocations
+ * are redirected if *any* free space is available in the CMA region.
+ * When restrict_cma_redirect is false, all movable allocations
+ * are eligible for redirection to CMA region (i.e movable allocations are
+ * not restricted from CMA region), when there is sufficient space there.
+ * (see __rmqueue()).
+ *
+ */
+DEFINE_STATIC_KEY_TRUE(restrict_cma_redirect);
+
+static int __init restrict_cma_redirect_setup(char *str)
+{
+#ifdef CONFIG_CMA
+	bool res;
+	int ret;
+	ret = kstrtobool(str, &res);
+	if (!ret && res == false)
+		static_branch_disable(&restrict_cma_redirect);
+#else
+	pr_warn("CONFIG_CMA not set. Ignoring restrict_cma_redirect option\n");
+#endif
+	return 1;
+}
+__setup("restrict_cma_redirect=", restrict_cma_redirect_setup);
+
+static inline bool cma_redirect_restricted(void)
+{
+	return static_branch_likely(&restrict_cma_redirect);
+}
+
 #ifdef CONFIG_HAVE_MEMORYLESS_NODES
 /*
  * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
@@ -3177,6 +3209,21 @@ __rmqueue(struct zone *zone, unsigned int order, int migratetype,
 	if (page)
 		return page;
 
+	if (IS_ENABLED(CONFIG_CMA)) {
+		/*
+		 * Balance movable allocations between regular and CMA areas by
+		 * allocating from CMA when over half of the zone's free memory
+		 * is in the CMA area.
+		 */
+		if (!cma_redirect_restricted() && alloc_flags & ALLOC_CMA &&
+			zone_page_state(zone, NR_FREE_CMA_PAGES) >
+			zone_page_state(zone, NR_FREE_PAGES) / 2) {
+				page = __rmqueue_cma_fallback(zone, order);
+				if (page)
+					return page;
+		}
+	}
+
 retry:
 	page = __rmqueue_smallest(zone, order, migratetype);
 
@@ -3186,6 +3233,9 @@ __rmqueue(struct zone *zone, unsigned int order, int migratetype,
 	if (unlikely(!page) && (migratetype == MIGRATE_MOVABLE))
 		trace_android_vh_rmqueue_cma_fallback(zone, order, &page);
 
+	if (!cma_redirect_restricted() && !page && alloc_flags & ALLOC_CMA)
+		page = __rmqueue_cma_fallback(zone, order);
+
 	if (unlikely(!page) && __rmqueue_fallback(zone, order, migratetype,
 						  alloc_flags))
 		goto retry;
@@ -3231,7 +3281,12 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
 	for (i = 0; i < count; ++i) {
 		struct page *page;
 
-		if (is_migrate_cma(migratetype))
+		/*
+		 * If CMA redirect is restricted, use CMA region only for
+		 * MIGRATE_CMA pages. cma_rediret_restricted() is false
+		 * if CONFIG_CMA is not set.
+		 */
+		if (cma_redirect_restricted() && is_migrate_cma(migratetype))
 			page = __rmqueue_cma(zone, order, migratetype,
 					     alloc_flags);
 		else
@@ -3880,7 +3935,8 @@ struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone,
 		if (alloc_flags & ALLOC_HIGHATOMIC)
 			page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);
 		if (!page) {
-			if (alloc_flags & ALLOC_CMA && migratetype == MIGRATE_MOVABLE)
+			if (cma_redirect_restricted() && alloc_flags & ALLOC_CMA &&
+					migratetype == MIGRATE_MOVABLE)
 				page = __rmqueue_cma(zone, order, migratetype,
 						     alloc_flags);
 			else
@@ -3924,7 +3980,8 @@ struct page *__rmqueue_pcplist(struct zone *zone, unsigned int order,
 
 	do {
 		/* First try to get CMA pages */
-		if (migratetype == MIGRATE_MOVABLE && alloc_flags & ALLOC_CMA)
+		if (cma_redirect_restricted() && migratetype == MIGRATE_MOVABLE &&
+				alloc_flags & ALLOC_CMA)
 			list = get_populated_pcp_list(zone, order, pcp, get_cma_migrate_type(),
 						      alloc_flags);
 		if (list == NULL) {
@@ -4344,7 +4401,12 @@ static inline unsigned int gfp_to_alloc_flags_cma(gfp_t gfp_mask,
 						  unsigned int alloc_flags)
 {
 #ifdef CONFIG_CMA
-	if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE && gfp_mask & __GFP_CMA)
+	/*
+	 * If cma_redirect_restricted is true, set ALLOC_CMA only for
+	 * movable allocations that have __GFP_CMA.
+	 */
+	if ((!cma_redirect_restricted() || gfp_mask & __GFP_CMA) &&
+			gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE)
 		alloc_flags |= ALLOC_CMA;
 	trace_android_vh_alloc_flags_cma_adjust(gfp_mask, &alloc_flags);
 #endif