| /* |
| * arch/arm64/kernel/topology.c |
| * |
| * Copyright (C) 2011,2013,2014 Linaro Limited. |
| * |
| * Based on the arm32 version written by Vincent Guittot in turn based on |
| * arch/sh/kernel/topology.c |
| * |
| * This file is subject to the terms and conditions of the GNU General Public |
| * License. See the file "COPYING" in the main directory of this archive |
| * for more details. |
| */ |
| |
| #include <linux/cpu.h> |
| #include <linux/cpumask.h> |
| #include <linux/init.h> |
| #include <linux/percpu.h> |
| #include <linux/node.h> |
| #include <linux/nodemask.h> |
| #include <linux/of.h> |
| #include <linux/sched.h> |
| #include <linux/slab.h> |
| |
| #include <asm/cputype.h> |
| #include <asm/smp_plat.h> |
| #include <asm/topology.h> |
| |
| /* |
| * cpu power table |
| * This per cpu data structure describes the relative capacity of each core. |
| * On a heteregenous system, cores don't have the same computation capacity |
| * and we reflect that difference in the cpu_power field so the scheduler can |
| * take this difference into account during load balance. A per cpu structure |
| * is preferred because each CPU updates its own cpu_power field during the |
| * load balance except for idle cores. One idle core is selected to run the |
| * rebalance_domains for all idle cores and the cpu_power can be updated |
| * during this sequence. |
| */ |
| static DEFINE_PER_CPU(unsigned long, cpu_scale); |
| |
| unsigned long arch_scale_freq_power(struct sched_domain *sd, int cpu) |
| { |
| return per_cpu(cpu_scale, cpu); |
| } |
| |
| static void set_power_scale(unsigned int cpu, unsigned long power) |
| { |
| per_cpu(cpu_scale, cpu) = power; |
| } |
| |
| static int __init get_cpu_for_node(struct device_node *node) |
| { |
| struct device_node *cpu_node; |
| int cpu; |
| |
| cpu_node = of_parse_phandle(node, "cpu", 0); |
| if (!cpu_node) |
| return -1; |
| |
| for_each_possible_cpu(cpu) { |
| if (of_get_cpu_node(cpu, NULL) == cpu_node) { |
| of_node_put(cpu_node); |
| return cpu; |
| } |
| } |
| |
| pr_crit("Unable to find CPU node for %s\n", cpu_node->full_name); |
| |
| of_node_put(cpu_node); |
| return -1; |
| } |
| |
| static int __init parse_core(struct device_node *core, int cluster_id, |
| int core_id) |
| { |
| char name[10]; |
| bool leaf = true; |
| int i = 0; |
| int cpu; |
| struct device_node *t; |
| |
| do { |
| snprintf(name, sizeof(name), "thread%d", i); |
| t = of_get_child_by_name(core, name); |
| if (t) { |
| leaf = false; |
| cpu = get_cpu_for_node(t); |
| if (cpu >= 0) { |
| cpu_topology[cpu].cluster_id = cluster_id; |
| cpu_topology[cpu].core_id = core_id; |
| cpu_topology[cpu].thread_id = i; |
| } else { |
| pr_err("%s: Can't get CPU for thread\n", |
| t->full_name); |
| of_node_put(t); |
| return -EINVAL; |
| } |
| of_node_put(t); |
| } |
| i++; |
| } while (t); |
| |
| cpu = get_cpu_for_node(core); |
| if (cpu >= 0) { |
| if (!leaf) { |
| pr_err("%s: Core has both threads and CPU\n", |
| core->full_name); |
| return -EINVAL; |
| } |
| |
| cpu_topology[cpu].cluster_id = cluster_id; |
| cpu_topology[cpu].core_id = core_id; |
| } else if (leaf) { |
| pr_err("%s: Can't get CPU for leaf core\n", core->full_name); |
| return -EINVAL; |
| } |
| |
| return 0; |
| } |
| |
| static int __init parse_cluster(struct device_node *cluster, int depth) |
| { |
| char name[10]; |
| bool leaf = true; |
| bool has_cores = false; |
| struct device_node *c; |
| static int cluster_id __initdata; |
| int core_id = 0; |
| int i, ret; |
| |
| /* |
| * First check for child clusters; we currently ignore any |
| * information about the nesting of clusters and present the |
| * scheduler with a flat list of them. |
| */ |
| i = 0; |
| do { |
| snprintf(name, sizeof(name), "cluster%d", i); |
| c = of_get_child_by_name(cluster, name); |
| if (c) { |
| leaf = false; |
| ret = parse_cluster(c, depth + 1); |
| of_node_put(c); |
| if (ret != 0) |
| return ret; |
| } |
| i++; |
| } while (c); |
| |
| /* Now check for cores */ |
| i = 0; |
| do { |
| snprintf(name, sizeof(name), "core%d", i); |
| c = of_get_child_by_name(cluster, name); |
| if (c) { |
| has_cores = true; |
| |
| if (depth == 0) { |
| pr_err("%s: cpu-map children should be clusters\n", |
| c->full_name); |
| of_node_put(c); |
| return -EINVAL; |
| } |
| |
| if (leaf) { |
| ret = parse_core(c, cluster_id, core_id++); |
| } else { |
| pr_err("%s: Non-leaf cluster with core %s\n", |
| cluster->full_name, name); |
| ret = -EINVAL; |
| } |
| |
| of_node_put(c); |
| if (ret != 0) |
| return ret; |
| } |
| i++; |
| } while (c); |
| |
| if (leaf && !has_cores) |
| pr_warn("%s: empty cluster\n", cluster->full_name); |
| |
| if (leaf) |
| cluster_id++; |
| |
| return 0; |
| } |
| |
| struct cpu_efficiency { |
| const char *compatible; |
| unsigned long efficiency; |
| }; |
| |
| /* |
| * Table of relative efficiency of each processors |
| * The efficiency value must fit in 20bit and the final |
| * cpu_scale value must be in the range |
| * 0 < cpu_scale < 3*SCHED_POWER_SCALE/2 |
| * in order to return at most 1 when DIV_ROUND_CLOSEST |
| * is used to compute the capacity of a CPU. |
| * Processors that are not defined in the table, |
| * use the default SCHED_POWER_SCALE value for cpu_scale. |
| */ |
| static const struct cpu_efficiency table_efficiency[] = { |
| { "arm,cortex-a57", 3891 }, |
| { "arm,cortex-a53", 2048 }, |
| { NULL, }, |
| }; |
| |
| static unsigned long *__cpu_capacity; |
| #define cpu_capacity(cpu) __cpu_capacity[cpu] |
| |
| static unsigned long middle_capacity = 1; |
| |
| static DEFINE_PER_CPU(unsigned long, cpu_efficiency) = SCHED_POWER_SCALE; |
| |
| unsigned long arch_get_cpu_efficiency(int cpu) |
| { |
| return per_cpu(cpu_efficiency, cpu); |
| } |
| |
| /* |
| * Iterate all CPUs' descriptor in DT and compute the efficiency |
| * (as per table_efficiency). Also calculate a middle efficiency |
| * as close as possible to (max{eff_i} - min{eff_i}) / 2 |
| * This is later used to scale the cpu_power field such that an |
| * 'average' CPU is of middle power. Also see the comments near |
| * table_efficiency[] and update_cpu_power(). |
| */ |
| static int __init parse_dt_topology(void) |
| { |
| struct device_node *cn, *map; |
| int ret = 0; |
| int cpu; |
| |
| cn = of_find_node_by_path("/cpus"); |
| if (!cn) { |
| pr_err("No CPU information found in DT\n"); |
| return 0; |
| } |
| |
| /* |
| * When topology is provided cpu-map is essentially a root |
| * cluster with restricted subnodes. |
| */ |
| map = of_get_child_by_name(cn, "cpu-map"); |
| if (!map) |
| goto out; |
| |
| ret = parse_cluster(map, 0); |
| if (ret != 0) |
| goto out_map; |
| |
| /* |
| * Check that all cores are in the topology; the SMP code will |
| * only mark cores described in the DT as possible. |
| */ |
| for_each_possible_cpu(cpu) { |
| if (cpu_topology[cpu].cluster_id == -1) { |
| pr_err("CPU%d: No topology information specified\n", |
| cpu); |
| ret = -EINVAL; |
| } |
| } |
| |
| out_map: |
| of_node_put(map); |
| out: |
| of_node_put(cn); |
| return ret; |
| } |
| |
| static void __init parse_dt_cpu_power(void) |
| { |
| const struct cpu_efficiency *cpu_eff; |
| struct device_node *cn; |
| unsigned long min_capacity = ULONG_MAX; |
| unsigned long max_capacity = 0; |
| unsigned long capacity = 0; |
| int cpu; |
| |
| __cpu_capacity = kcalloc(nr_cpu_ids, sizeof(*__cpu_capacity), |
| GFP_NOWAIT); |
| |
| for_each_possible_cpu(cpu) { |
| const u32 *rate; |
| int len; |
| |
| /* Too early to use cpu->of_node */ |
| cn = of_get_cpu_node(cpu, NULL); |
| if (!cn) { |
| pr_err("Missing device node for CPU %d\n", cpu); |
| continue; |
| } |
| |
| for (cpu_eff = table_efficiency; cpu_eff->compatible; cpu_eff++) |
| if (of_device_is_compatible(cn, cpu_eff->compatible)) |
| break; |
| |
| if (cpu_eff->compatible == NULL) { |
| pr_warn("%s: Unknown CPU type\n", cn->full_name); |
| continue; |
| } |
| |
| per_cpu(cpu_efficiency, cpu) = cpu_eff->efficiency; |
| |
| rate = of_get_property(cn, "clock-frequency", &len); |
| if (!rate || len != 4) { |
| pr_err("%s: Missing clock-frequency property\n", |
| cn->full_name); |
| continue; |
| } |
| |
| capacity = ((be32_to_cpup(rate)) >> 20) * cpu_eff->efficiency; |
| |
| /* Save min capacity of the system */ |
| if (capacity < min_capacity) |
| min_capacity = capacity; |
| |
| /* Save max capacity of the system */ |
| if (capacity > max_capacity) |
| max_capacity = capacity; |
| |
| cpu_capacity(cpu) = capacity; |
| } |
| |
| /* If min and max capacities are equal we bypass the update of the |
| * cpu_scale because all CPUs have the same capacity. Otherwise, we |
| * compute a middle_capacity factor that will ensure that the capacity |
| * of an 'average' CPU of the system will be as close as possible to |
| * SCHED_POWER_SCALE, which is the default value, but with the |
| * constraint explained near table_efficiency[]. |
| */ |
| if (min_capacity == max_capacity) |
| return; |
| else if (4 * max_capacity < (3 * (max_capacity + min_capacity))) |
| middle_capacity = (min_capacity + max_capacity) |
| >> (SCHED_POWER_SHIFT+1); |
| else |
| middle_capacity = ((max_capacity / 3) |
| >> (SCHED_POWER_SHIFT-1)) + 1; |
| } |
| |
| /* |
| * Look for a customed capacity of a CPU in the cpu_topo_data table during the |
| * boot. The update of all CPUs is in O(n^2) for heteregeneous system but the |
| * function returns directly for SMP system. |
| */ |
| static void update_cpu_power(unsigned int cpu) |
| { |
| if (!cpu_capacity(cpu)) |
| return; |
| |
| set_power_scale(cpu, cpu_capacity(cpu) / middle_capacity); |
| |
| pr_info("CPU%u: update cpu_power %lu\n", |
| cpu, arch_scale_freq_power(NULL, cpu)); |
| } |
| |
| /* |
| * cpu topology table |
| */ |
| struct cpu_topology cpu_topology[NR_CPUS]; |
| EXPORT_SYMBOL_GPL(cpu_topology); |
| |
| const struct cpumask *cpu_coregroup_mask(int cpu) |
| { |
| return &cpu_topology[cpu].core_sibling; |
| } |
| |
| static void update_siblings_masks(unsigned int cpuid) |
| { |
| struct cpu_topology *cpu_topo, *cpuid_topo = &cpu_topology[cpuid]; |
| int cpu; |
| |
| if (cpuid_topo->cluster_id == -1) { |
| /* No topology information for this cpu ?! */ |
| pr_err("CPU%u: No topology information configured\n", cpuid); |
| return; |
| } |
| |
| /* update core and thread sibling masks */ |
| for_each_possible_cpu(cpu) { |
| cpu_topo = &cpu_topology[cpu]; |
| |
| if (cpuid_topo->cluster_id != cpu_topo->cluster_id) |
| continue; |
| |
| cpumask_set_cpu(cpuid, &cpu_topo->core_sibling); |
| if (cpu != cpuid) |
| cpumask_set_cpu(cpu, &cpuid_topo->core_sibling); |
| |
| if (cpuid_topo->core_id != cpu_topo->core_id) |
| continue; |
| |
| cpumask_set_cpu(cpuid, &cpu_topo->thread_sibling); |
| if (cpu != cpuid) |
| cpumask_set_cpu(cpu, &cpuid_topo->thread_sibling); |
| } |
| } |
| |
| void store_cpu_topology(unsigned int cpuid) |
| { |
| struct cpu_topology *cpuid_topo = &cpu_topology[cpuid]; |
| u64 mpidr; |
| |
| if (cpuid_topo->cluster_id != -1) |
| goto topology_populated; |
| |
| mpidr = read_cpuid_mpidr(); |
| |
| /* Create cpu topology mapping based on MPIDR. */ |
| if (mpidr & MPIDR_UP_BITMASK) { |
| /* Uniprocessor system */ |
| cpuid_topo->thread_id = -1; |
| cpuid_topo->core_id = MPIDR_AFFINITY_LEVEL(mpidr, 0); |
| cpuid_topo->cluster_id = 0; |
| } else if (mpidr & MPIDR_MT_BITMASK) { |
| /* Multiprocessor system : Multi-threads per core */ |
| cpuid_topo->thread_id = MPIDR_AFFINITY_LEVEL(mpidr, 0); |
| cpuid_topo->core_id = MPIDR_AFFINITY_LEVEL(mpidr, 1); |
| cpuid_topo->cluster_id = |
| ((mpidr & MPIDR_AFF_MASK(2)) >> mpidr_hash.shift_aff[2] | |
| (mpidr & MPIDR_AFF_MASK(3)) >> mpidr_hash.shift_aff[3]) |
| >> mpidr_hash.shift_aff[1] >> mpidr_hash.shift_aff[0]; |
| } else { |
| /* Multiprocessor system : Single-thread per core */ |
| cpuid_topo->thread_id = -1; |
| cpuid_topo->core_id = MPIDR_AFFINITY_LEVEL(mpidr, 0); |
| cpuid_topo->cluster_id = |
| ((mpidr & MPIDR_AFF_MASK(1)) >> mpidr_hash.shift_aff[1] | |
| (mpidr & MPIDR_AFF_MASK(2)) >> mpidr_hash.shift_aff[2] | |
| (mpidr & MPIDR_AFF_MASK(3)) >> mpidr_hash.shift_aff[3]) |
| >> mpidr_hash.shift_aff[0]; |
| } |
| |
| pr_debug("CPU%u: cluster %d core %d thread %d mpidr %llx\n", |
| cpuid, cpuid_topo->cluster_id, cpuid_topo->core_id, |
| cpuid_topo->thread_id, mpidr); |
| |
| topology_populated: |
| update_siblings_masks(cpuid); |
| update_cpu_power(cpuid); |
| } |
| |
| static void __init reset_cpu_topology(void) |
| { |
| unsigned int cpu; |
| |
| for_each_possible_cpu(cpu) { |
| struct cpu_topology *cpu_topo = &cpu_topology[cpu]; |
| |
| cpu_topo->thread_id = -1; |
| cpu_topo->core_id = 0; |
| cpu_topo->cluster_id = -1; |
| |
| cpumask_clear(&cpu_topo->core_sibling); |
| cpumask_set_cpu(cpu, &cpu_topo->core_sibling); |
| cpumask_clear(&cpu_topo->thread_sibling); |
| cpumask_set_cpu(cpu, &cpu_topo->thread_sibling); |
| } |
| } |
| |
| static void __init reset_cpu_power(void) |
| { |
| unsigned int cpu; |
| |
| for_each_possible_cpu(cpu) |
| set_power_scale(cpu, SCHED_POWER_SCALE); |
| } |
| |
| void __init init_cpu_topology(void) |
| { |
| reset_cpu_topology(); |
| |
| /* |
| * Discard anything that was parsed if we hit an error so we |
| * don't use partial information. |
| */ |
| if (parse_dt_topology()) |
| reset_cpu_topology(); |
| |
| reset_cpu_power(); |
| parse_dt_cpu_power(); |
| } |