| From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 |
| From: Sami Tolvanen <samitolvanen@google.com> |
| Date: Fri, 8 Dec 2017 15:49:24 -0800 |
| Subject: ANDROID: init: ensure initcall ordering with LTO |
| |
| With LTO, the compiler doesn't necessarily obey link order for |
| initcalls, and the initcall variables need to be globally unique |
| to avoid naming collisions. |
| |
| In order to preserve the intended order, this change moves each |
| initcall variable into its own section and generates a linker |
| script (in scripts/link-vmlinux.sh) to define the correct order |
| for these sections. We also add a __COUNTER__ prefix to the name, |
| so we can retain the order of initcalls within each compilation |
| unit, and __LINE__ to help ensure uniqueness. |
| |
| Bug: 145210207 |
| Change-Id: I602038783853497790c5a2941343c546e380c525 |
| Signed-off-by: Sami Tolvanen <samitolvanen@google.com> |
| --- |
| include/linux/init.h | 26 ++- |
| scripts/generate_initcall_order.pl | 250 +++++++++++++++++++++++++++++ |
| scripts/link-vmlinux.sh | 36 +++-- |
| 3 files changed, 293 insertions(+), 19 deletions(-) |
| create mode 100755 scripts/generate_initcall_order.pl |
| |
| diff --git a/include/linux/init.h b/include/linux/init.h |
| index 212fc9e2f691..0a16e48e6753 100644 |
| --- a/include/linux/init.h |
| +++ b/include/linux/init.h |
| @@ -192,10 +192,32 @@ extern bool initcall_debug; |
| ".long " #fn " - . \n" \ |
| ".previous \n"); |
| #else |
| -#define ___define_initcall(fn, id, __sec) \ |
| +#ifdef CONFIG_LTO_CLANG |
| + /* |
| + * With LTO, the compiler doesn't necessarily obey link order for |
| + * initcalls, and the initcall variable needs to be globally unique |
| + * to avoid naming collisions. In order to preserve the correct |
| + * order, we add each variable into its own section and generate a |
| + * linker script (in scripts/link-vmlinux.sh) to ensure the order |
| + * remains correct. We also add a __COUNTER__ prefix to the name, |
| + * so we can retain the order of initcalls within each compilation |
| + * unit, and __LINE__ to make the names more unique. |
| + */ |
| + #define ___lto_initcall(c, l, fn, id, __sec) \ |
| + static initcall_t __initcall_##c##_##l##_##fn##id __used \ |
| + __attribute__((__section__( #__sec \ |
| + __stringify(.init..##c##_##l##_##fn)))) = fn; |
| + #define __lto_initcall(c, l, fn, id, __sec) \ |
| + ___lto_initcall(c, l, fn, id, __sec) |
| + |
| + #define ___define_initcall(fn, id, __sec) \ |
| + __lto_initcall(__COUNTER__, __LINE__, fn, id, __sec) |
| +#else |
| + #define ___define_initcall(fn, id, __sec) \ |
| static initcall_t __initcall_##fn##id __used \ |
| __attribute__((__section__(#__sec ".init"))) = fn; |
| #endif |
| +#endif |
| |
| #define __define_initcall(fn, id) ___define_initcall(fn, id, .initcall##id) |
| |
| @@ -236,7 +258,7 @@ extern bool initcall_debug; |
| #define __exitcall(fn) \ |
| static exitcall_t __exitcall_##fn __exit_call = fn |
| |
| -#define console_initcall(fn) ___define_initcall(fn,, .con_initcall) |
| +#define console_initcall(fn) ___define_initcall(fn, con, .con_initcall) |
| |
| struct obs_kernel_param { |
| const char *str; |
| diff --git a/scripts/generate_initcall_order.pl b/scripts/generate_initcall_order.pl |
| new file mode 100755 |
| index 000000000000..f772b4a01caa |
| --- /dev/null |
| +++ b/scripts/generate_initcall_order.pl |
| @@ -0,0 +1,250 @@ |
| +#!/usr/bin/env perl |
| +# SPDX-License-Identifier: GPL-2.0 |
| +# |
| +# Generates a linker script that specifies the correct initcall order. |
| +# |
| +# Copyright (C) 2019 Google LLC |
| + |
| +use strict; |
| +use warnings; |
| +use IO::Handle; |
| + |
| +my $nm = $ENV{'LLVM_NM'} || "llvm-nm"; |
| +my $ar = $ENV{'AR'} || "llvm-ar"; |
| +my $objtree = $ENV{'objtree'} || "."; |
| + |
| +## list of all object files to process, in link order |
| +my @objects; |
| +## currently active child processes |
| +my $jobs = {}; # child process pid -> file handle |
| +## results from child processes |
| +my $results = {}; # object index -> { level, function } |
| + |
| +## reads _NPROCESSORS_ONLN to determine the number of processes to start |
| +sub get_online_processors { |
| + open(my $fh, "getconf _NPROCESSORS_ONLN 2>/dev/null |") |
| + or die "$0: failed to execute getconf: $!"; |
| + my $procs = <$fh>; |
| + close($fh); |
| + |
| + if (!($procs =~ /^\d+$/)) { |
| + return 1; |
| + } |
| + |
| + return int($procs); |
| +} |
| + |
| +## finds initcalls defined in an object file, parses level and function name, |
| +## and prints it out to the parent process |
| +sub find_initcalls { |
| + my ($object) = @_; |
| + |
| + die "$0: object file $object doesn't exist?" if (! -f $object); |
| + |
| + open(my $fh, "\"$nm\" -just-symbol-name -defined-only \"$object\" 2>/dev/null |") |
| + or die "$0: failed to execute \"$nm\": $!"; |
| + |
| + my $initcalls = {}; |
| + |
| + while (<$fh>) { |
| + chomp; |
| + |
| + my ($counter, $line, $symbol) = $_ =~ /^__initcall_(\d+)_(\d+)_(.*)$/; |
| + |
| + if (!defined($counter) || !defined($line) || !defined($symbol)) { |
| + next; |
| + } |
| + |
| + my ($function, $level) = $symbol =~ |
| + /^(.*)((early|rootfs|con|security|[0-9])s?)$/; |
| + |
| + die "$0: duplicate initcall counter value in object $object: $_" |
| + if exists($initcalls->{$counter}); |
| + |
| + $initcalls->{$counter} = { |
| + 'level' => $level, |
| + 'line' => $line, |
| + 'function' => $function |
| + }; |
| + } |
| + |
| + close($fh); |
| + |
| + # sort initcalls in each object file numerically by the counter value |
| + # to ensure they are in the order they were defined |
| + foreach my $counter (sort { $a <=> $b } keys(%{$initcalls})) { |
| + print $initcalls->{$counter}->{"level"} . " " . |
| + $counter . " " . |
| + $initcalls->{$counter}->{"line"} . " " . |
| + $initcalls->{$counter}->{"function"} . "\n"; |
| + } |
| +} |
| + |
| +## waits for any child process to complete, reads the results, and adds them to |
| +## the $results array for later processing |
| +sub wait_for_results { |
| + my $pid = wait(); |
| + if ($pid > 0) { |
| + my $fh = $jobs->{$pid}; |
| + |
| + # the child process prints out results in the following format: |
| + # line 1: <object file index> |
| + # line 2..n: <level> <counter> <line> <function> |
| + |
| + my $index = <$fh>; |
| + chomp($index); |
| + |
| + if (!($index =~ /^\d+$/)) { |
| + die "$0: child $pid returned an invalid index: $index"; |
| + } |
| + $index = int($index); |
| + |
| + while (<$fh>) { |
| + chomp; |
| + my ($level, $counter, $line, $function) = $_ =~ |
| + /^([^\ ]+)\ (\d+)\ (\d+)\ (.*)$/; |
| + |
| + if (!defined($level) || |
| + !defined($counter) || |
| + !defined($line) || |
| + !defined($function)) { |
| + die "$0: child $pid returned invalid data"; |
| + } |
| + |
| + if (!exists($results->{$index})) { |
| + $results->{$index} = []; |
| + } |
| + |
| + push (@{$results->{$index}}, { |
| + 'level' => $level, |
| + 'counter' => $counter, |
| + 'line' => $line, |
| + 'function' => $function |
| + }); |
| + } |
| + |
| + close($fh); |
| + delete($jobs->{$pid}); |
| + } |
| +} |
| + |
| +## launches child processes to find initcalls from the object files, waits for |
| +## each process to complete and collects the results |
| +sub process_objects { |
| + my $index = 0; # link order index of the object file |
| + my $njobs = get_online_processors(); |
| + |
| + while (scalar(@objects) > 0) { |
| + my $object = shift(@objects); |
| + |
| + # fork a child process and read it's stdout |
| + my $pid = open(my $fh, '-|'); |
| + |
| + if (!defined($pid)) { |
| + die "$0: failed to fork: $!"; |
| + } elsif ($pid) { |
| + # save the child process pid and the file handle |
| + $jobs->{$pid} = $fh; |
| + } else { |
| + STDOUT->autoflush(1); |
| + print "$index\n"; |
| + find_initcalls("$objtree/$object"); |
| + exit; |
| + } |
| + |
| + $index++; |
| + |
| + # if we reached the maximum number of processes, wait for one |
| + # to complete before launching new ones |
| + if (scalar(keys(%{$jobs})) >= $njobs && scalar(@objects) > 0) { |
| + wait_for_results(); |
| + } |
| + } |
| + |
| + # wait for the remaining children to complete |
| + while (scalar(keys(%{$jobs})) > 0) { |
| + wait_for_results(); |
| + } |
| +} |
| + |
| +## gets a list of actual object files from thin archives, and adds them to |
| +## @objects in link order |
| +sub find_objects { |
| + while (my $file = shift(@ARGV)) { |
| + my $pid = open (my $fh, "\"$ar\" t \"$file\" 2>/dev/null |") |
| + or die "$0: failed to execute $ar: $!"; |
| + |
| + my @output; |
| + |
| + while (<$fh>) { |
| + chomp; |
| + push(@output, $_); |
| + } |
| + |
| + close($fh); |
| + |
| + # if $ar failed, assume we have an object file |
| + if ($? != 0) { |
| + push(@objects, $file); |
| + next; |
| + } |
| + |
| + # if $ar succeeded, read the list of object files |
| + foreach (@output) { |
| + push(@objects, $_); |
| + } |
| + } |
| +} |
| + |
| +## START |
| +find_objects(); |
| +process_objects(); |
| + |
| +## process results and add them to $sections in the correct order |
| +my $sections = {}; |
| + |
| +foreach my $index (sort { $a <=> $b } keys(%{$results})) { |
| + foreach my $result (@{$results->{$index}}) { |
| + my $level = $result->{'level'}; |
| + |
| + if (!exists($sections->{$level})) { |
| + $sections->{$level} = []; |
| + } |
| + |
| + my $fsname = $result->{'counter'} . '_' . |
| + $result->{'line'} . '_' . |
| + $result->{'function'}; |
| + |
| + push(@{$sections->{$level}}, $fsname); |
| + } |
| +} |
| + |
| +if (!keys(%{$sections})) { |
| + exit(0); # no initcalls...? |
| +} |
| + |
| +## print out a linker script that defines the order of initcalls for each |
| +## level |
| +print "SECTIONS {\n"; |
| + |
| +foreach my $level (sort(keys(%{$sections}))) { |
| + my $section; |
| + |
| + if ($level eq 'con') { |
| + $section = '.con_initcall.init'; |
| + } elsif ($level eq 'security') { |
| + $section = '.security_initcall.init'; |
| + } else { |
| + $section = ".initcall${level}.init"; |
| + } |
| + |
| + print "\t${section} : {\n"; |
| + |
| + foreach my $fsname (@{$sections->{$level}}) { |
| + print "\t\t*(${section}..${fsname}) ;\n" |
| + } |
| + |
| + print "\t}\n"; |
| +} |
| + |
| +print "}\n"; |
| diff --git a/scripts/link-vmlinux.sh b/scripts/link-vmlinux.sh |
| index 370446faad0b..a99698fadc6c 100755 |
| --- a/scripts/link-vmlinux.sh |
| +++ b/scripts/link-vmlinux.sh |
| @@ -39,28 +39,30 @@ info() |
| fi |
| } |
| |
| -# If CONFIG_LTO_CLANG is selected, collect generated symbol versions into |
| -# .tmp_symversions |
| -modversions() |
| +# If CONFIG_LTO_CLANG is selected, generate a linker script to ensure correct |
| +# ordering of initcalls, and with CONFIG_MODVERSIONS also enabled, collect the |
| +# previously generated symbol versions into the same script. |
| +lto_lds() |
| { |
| if [ -z "${CONFIG_LTO_CLANG}" ]; then |
| return |
| fi |
| - if [ -z "${CONFIG_MODVERSIONS}" ]; then |
| - return |
| - fi |
| |
| - rm -f .tmp_symversions |
| - |
| - for a in ${KBUILD_VMLINUX_OBJS} ${KBUILD_VMLINUX_LIBS}; do |
| - for o in $(${AR} t $a 2>/dev/null); do |
| - if [ -f ${o}.symversions ]; then |
| - cat ${o}.symversions >> .tmp_symversions |
| - fi |
| + ${srctree}/scripts/generate_initcall_order.pl \ |
| + ${KBUILD_VMLINUX_OBJS} ${KBUILD_VMLINUX_LIBS} \ |
| + > .tmp_lto.lds |
| + |
| + if [ -n "${CONFIG_MODVERSIONS}" ]; then |
| + for a in ${KBUILD_VMLINUX_OBJS} ${KBUILD_VMLINUX_LIBS}; do |
| + for o in $(${AR} t $a 2>/dev/null); do |
| + if [ -f ${o}.symversions ]; then |
| + cat ${o}.symversions >> .tmp_lto.lds |
| + fi |
| + done |
| done |
| - done |
| + fi |
| |
| - echo "-T .tmp_symversions" |
| + echo "-T .tmp_lto.lds" |
| } |
| |
| # Link of vmlinux.o used for section mismatch analysis |
| @@ -84,7 +86,7 @@ modpost_link() |
| info LD ${1} |
| fi |
| |
| - ${LD} ${KBUILD_LDFLAGS} -r -o ${1} $(modversions) ${objects} |
| + ${LD} ${KBUILD_LDFLAGS} -r -o ${1} $(lto_lds) ${objects} |
| } |
| |
| # If CONFIG_LTO_CLANG is selected, we postpone running recordmcount until |
| @@ -243,7 +245,7 @@ cleanup() |
| rm -f .btf.* |
| rm -f .tmp_System.map |
| rm -f .tmp_kallsyms* |
| - rm -f .tmp_symversions |
| + rm -f .tmp_lto.lds |
| rm -f .tmp_vmlinux* |
| rm -f System.map |
| rm -f vmlinux |