kernel/sched/boost.c

/* Copyright (c) 2012-2019, The Linux Foundation. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 and
 * only version 2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */

#include "sched.h"
#include "walt.h"
#include <linux/of.h>
#include <linux/sched/core_ctl.h>
#include <trace/events/sched.h>

/*
 * Scheduler boost is a mechanism to temporarily place tasks on CPUs
 * with higher capacity than those where a task would have normally
 * ended up with their load characteristics. Any entity enabling
 * boost is responsible for disabling it as well.
 */

unsigned int sysctl_sched_boost; /* To/from userspace */
unsigned int sched_boost_type; /* currently activated sched boost */
enum sched_boost_policy boost_policy;

static enum sched_boost_policy boost_policy_dt = SCHED_BOOST_NONE;
static DEFINE_MUTEX(boost_mutex);

/*
 * Scheduler boost type and boost policy might at first seem unrelated,
 * however, there exists a connection between them that will allow us
 * to use them interchangeably during placement decisions. We'll explain
 * the connection here in one possible way so that the implications are
 * clear when looking at placement policies.
 *
 * When policy = SCHED_BOOST_NONE, type is either none or RESTRAINED
 * When policy = SCHED_BOOST_ON_ALL or SCHED_BOOST_ON_BIG, type can
 * neither be none nor RESTRAINED.
 */
static void set_boost_policy(int type)
{
	if (type == NO_BOOST || type == RESTRAINED_BOOST) {
		boost_policy = SCHED_BOOST_NONE;
		return;
	}

	if (boost_policy_dt) {
		boost_policy = boost_policy_dt;
		return;
	}

	if (min_possible_efficiency != max_possible_efficiency) {
		boost_policy = SCHED_BOOST_ON_BIG;
		return;
	}

	boost_policy = SCHED_BOOST_ON_ALL;
}

static bool verify_boost_params(int type)
{
	return type >= RESTRAINED_BOOST_DISABLE && type <= RESTRAINED_BOOST;
}

static void sched_no_boost_nop(void)
{
}

static void sched_full_throttle_boost_enter(void)
{
	core_ctl_set_boost(true);
	walt_enable_frequency_aggregation(true);
}

static void sched_full_throttle_boost_exit(void)
{
	core_ctl_set_boost(false);
	walt_enable_frequency_aggregation(false);
}

static void sched_conservative_boost_enter(void)
{
	update_cgroup_boost_settings();
}

static void sched_conservative_boost_exit(void)
{
	restore_cgroup_boost_settings();
}

static void sched_restrained_boost_enter(void)
{
	walt_enable_frequency_aggregation(true);
}

static void sched_restrained_boost_exit(void)
{
	walt_enable_frequency_aggregation(false);
}

struct sched_boost_data {
	int refcount;
	void (*enter)(void);
	void (*exit)(void);
};

static struct sched_boost_data sched_boosts[] = {
	[NO_BOOST] = {
		.refcount = 0,
		.enter = sched_no_boost_nop,
		.exit = sched_no_boost_nop,
	},
	[FULL_THROTTLE_BOOST] = {
		.refcount = 0,
		.enter = sched_full_throttle_boost_enter,
		.exit = sched_full_throttle_boost_exit,
	},
	[CONSERVATIVE_BOOST] = {
		.refcount = 0,
		.enter = sched_conservative_boost_enter,
		.exit = sched_conservative_boost_exit,
	},
	[RESTRAINED_BOOST] = {
		.refcount = 0,
		.enter = sched_restrained_boost_enter,
		.exit = sched_restrained_boost_exit,
	},
};

#define SCHED_BOOST_START FULL_THROTTLE_BOOST
#define SCHED_BOOST_END (RESTRAINED_BOOST + 1)

static int sched_effective_boost(void)
{
	int i;

	/*
	 * The boosts are sorted in descending order by
	 * priority.
	 */
	for (i = SCHED_BOOST_START; i < SCHED_BOOST_END; i++) {
		if (sched_boosts[i].refcount >= 1)
			return i;
	}

	return NO_BOOST;
}

static void sched_boost_disable(int type)
{
	struct sched_boost_data *sb = &sched_boosts[type];
	int next_boost;

	if (sb->refcount <= 0)
		return;

	sb->refcount--;

	if (sb->refcount)
		return;

	/*
	 * This boost's refcount becomes zero, so it must
	 * be disabled. Disable it first and then apply
	 * the next boost.
	 */
	sb->exit();

	next_boost = sched_effective_boost();
	sched_boosts[next_boost].enter();
}

static void sched_boost_enable(int type)
{
	struct sched_boost_data *sb = &sched_boosts[type];
	int next_boost, prev_boost = sched_boost_type;

	sb->refcount++;

	if (sb->refcount != 1)
		return;

	/*
	 * This boost enable request did not come before.
	 * Take this new request and find the next boost
	 * by aggregating all the enabled boosts. If there
	 * is a change, disable the previous boost and enable
	 * the next boost.
	 */

	next_boost = sched_effective_boost();
	if (next_boost == prev_boost)
		return;

	sched_boosts[prev_boost].exit();
	sched_boosts[next_boost].enter();
}

static void sched_boost_disable_all(void)
{
	int i;

	for (i = SCHED_BOOST_START; i < SCHED_BOOST_END; i++) {
		if (sched_boosts[i].refcount > 0) {
			sched_boosts[i].exit();
			sched_boosts[i].refcount = 0;
		}
	}
}

static void _sched_set_boost(int type)
{
	if (type == 0)
		sched_boost_disable_all();
	else if (type > 0)
		sched_boost_enable(type);
	else
		sched_boost_disable(-type);

	/*
	 * sysctl_sched_boost holds the boost request from
	 * user space which could be different from the
	 * effectively enabled boost. Update the effective
	 * boost here.
	 */

	sched_boost_type = sched_effective_boost();
	sysctl_sched_boost = sched_boost_type;
	set_boost_policy(sysctl_sched_boost);
	trace_sched_set_boost(sysctl_sched_boost);
}

void sched_boost_parse_dt(void)
{
	struct device_node *sn;
	const char *boost_policy;

	sn = of_find_node_by_path("/sched-hmp");
	if (!sn)
		return;

	if (!of_property_read_string(sn, "boost-policy", &boost_policy)) {
		if (!strcmp(boost_policy, "boost-on-big"))
			boost_policy_dt = SCHED_BOOST_ON_BIG;
		else if (!strcmp(boost_policy, "boost-on-all"))
			boost_policy_dt = SCHED_BOOST_ON_ALL;
	}
}

int sched_set_boost(int type)
{
	int ret = 0;

	mutex_lock(&boost_mutex);
	if (verify_boost_params(type))
		_sched_set_boost(type);
	else
		ret = -EINVAL;
	mutex_unlock(&boost_mutex);
	return ret;
}

int sched_boost_handler(struct ctl_table *table, int write,
		void __user *buffer, size_t *lenp,
		loff_t *ppos)
{
	int ret;
	unsigned int *data = (unsigned int *)table->data;

	mutex_lock(&boost_mutex);

	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);

	if (ret || !write)
		goto done;

	if (verify_boost_params(*data))
		_sched_set_boost(*data);
	else
		ret = -EINVAL;

done:
	mutex_unlock(&boost_mutex);
	return ret;
}