blob: 69079e854002029a9980f9c2214dc6cdfa5a6ef0 [file] [log] [blame]
#!/usr/bin/env python
# Copyright (C) 2017 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This tool translates a collection of BUILD.gn files into a mostly equivalent
# Android.bp file for the Android Soong build system. The input to the tool is a
# JSON description of the GN build definition generated with the following
# command:
#
# gn desc out --format=json --all-toolchains "//*" > desc.json
#
# The tool is then given a list of GN labels for which to generate Android.bp
# build rules. The dependencies for the GN labels are squashed to the generated
# Android.bp target, except for actions which get their own genrule. Some
# libraries are also mapped to their Android equivalents -- see |builtin_deps|.
import argparse
import errno
import json
import os
import re
import shutil
import subprocess
import sys
# Default targets to translate to the blueprint file.
default_targets = [
'//:libperfetto',
'//:libperfetto_android_internal',
'//:perfetto_integrationtests',
'//:perfetto_trace_protos',
'//:perfetto_unittests',
'//:perfetto',
'//:traced',
'//:traced_probes',
'//:trace_to_text',
'//:heapprofd_client',
'//:heapprofd',
'//:trigger_perfetto',
]
# Defines a custom init_rc argument to be applied to the corresponding output
# blueprint target.
target_initrc = {
'//:traced': 'perfetto.rc',
'//:heapprofd': 'heapprofd.rc',
}
target_host_supported = [
'//:perfetto_trace_protos',
]
target_host_only = [
'//:trace_to_text',
]
# Arguments for the GN output directory.
gn_args = 'target_os="android" target_cpu="arm" is_debug=false perfetto_build_with_android=true'
# All module names are prefixed with this string to avoid collisions.
module_prefix = 'perfetto_'
# Shared libraries which are directly translated to Android system equivalents.
library_whitelist = [
"android.hardware.atrace@1.0",
'android.hardware.health@2.0',
"android.hardware.power.stats@1.0",
'android',
'base',
'binder',
'hidlbase',
'hidltransport',
'hwbinder',
'incident',
'log',
'services',
'utils',
]
# Name of the module which settings such as compiler flags for all other
# modules.
defaults_module = module_prefix + 'defaults'
# Location of the project in the Android source tree.
tree_path = 'external/perfetto'
# Compiler flags which are passed through to the blueprint.
cflag_whitelist = r'^-DPERFETTO.*$'
# Compiler defines which are passed through to the blueprint.
define_whitelist = r'^(GOOGLE_PROTO.*)|(PERFETTO_BUILD_WITH_ANDROID)|(ZLIB_.*)|(USE_MMAP)|(HAVE_HIDDEN)$'
# Shared libraries which are not in PDK.
library_not_in_pdk = {
'libandroid',
'libservices',
}
# Additional arguments to apply to Android.bp rules.
additional_args = {
'heapprofd_client': [
('include_dirs', ['bionic/libc']),
('static_libs', ['libasync_safe']),
],
'traced_probes': [
('required', ['libperfetto_android_internal', 'trigger_perfetto']),
],
'libperfetto_android_internal': [
('static_libs', ['libhealthhalutils']),
],
}
def enable_gmock(module):
module.static_libs.append('libgmock')
def enable_gtest_prod(module):
module.static_libs.append('libgtest_prod')
def enable_gtest(module):
assert module.type == 'cc_test'
def enable_protobuf_full(module):
module.shared_libs.append('libprotobuf-cpp-full')
def enable_protobuf_lite(module):
module.shared_libs.append('libprotobuf-cpp-lite')
def enable_protoc_lib(module):
module.shared_libs.append('libprotoc')
def enable_libunwindstack(module):
module.shared_libs.append('libunwindstack')
module.shared_libs.append('libprocinfo')
module.shared_libs.append('libbase')
def enable_libunwind(module):
# libunwind is disabled on Darwin so we cannot depend on it.
pass
def enable_sqlite(module):
module.static_libs.append('libsqlite')
def enable_zlib(module):
module.shared_libs.append('libz')
# Android equivalents for third-party libraries that the upstream project
# depends on.
builtin_deps = {
'//buildtools:gmock': enable_gmock,
'//buildtools:gtest': enable_gtest,
'//gn:gtest_prod_config': enable_gtest_prod,
'//buildtools:gtest_main': enable_gtest,
'//buildtools:libunwind': enable_libunwind,
'//buildtools:protobuf_full': enable_protobuf_full,
'//buildtools:protobuf_lite': enable_protobuf_lite,
'//buildtools:protoc_lib': enable_protoc_lib,
'//buildtools:libunwindstack': enable_libunwindstack,
'//buildtools:sqlite': enable_sqlite,
'//buildtools:zlib': enable_zlib,
}
# ----------------------------------------------------------------------------
# End of configuration.
# ----------------------------------------------------------------------------
class Error(Exception):
pass
class ThrowingArgumentParser(argparse.ArgumentParser):
def __init__(self, context):
super(ThrowingArgumentParser, self).__init__()
self.context = context
def error(self, message):
raise Error('%s: %s' % (self.context, message))
class Module(object):
"""A single module (e.g., cc_binary, cc_test) in a blueprint."""
def __init__(self, mod_type, name):
self.type = mod_type
self.name = name
self.srcs = []
self.comment = None
self.shared_libs = []
self.static_libs = []
self.tools = []
self.cmd = None
self.host_supported = False
self.init_rc = []
self.out = []
self.export_include_dirs = []
self.generated_headers = []
self.export_generated_headers = []
self.defaults = []
self.cflags = set()
self.local_include_dirs = []
self.include_dirs = []
self.required = []
self.user_debug_flag = False
self.tool_files = None
def to_string(self, output):
if self.comment:
output.append('// %s' % self.comment)
output.append('%s {' % self.type)
self._output_field(output, 'name')
self._output_field(output, 'srcs')
self._output_field(output, 'shared_libs')
self._output_field(output, 'static_libs')
self._output_field(output, 'tools')
self._output_field(output, 'cmd', sort=False)
self._output_field(output, 'host_supported')
self._output_field(output, 'init_rc')
self._output_field(output, 'out')
self._output_field(output, 'export_include_dirs')
self._output_field(output, 'generated_headers')
self._output_field(output, 'export_generated_headers')
self._output_field(output, 'defaults')
self._output_field(output, 'cflags')
self._output_field(output, 'local_include_dirs')
self._output_field(output, 'include_dirs')
self._output_field(output, 'required')
self._output_field(output, 'tool_files')
disable_pdk = any(name in library_not_in_pdk for name in self.shared_libs)
if self.user_debug_flag or disable_pdk:
output.append(' product_variables: {')
if disable_pdk:
output.append(' pdk: {')
output.append(' enabled: false,')
output.append(' },')
if self.user_debug_flag:
output.append(' debuggable: {')
output.append(' cflags: ["-DPERFETTO_BUILD_WITH_ANDROID_USERDEBUG"],')
output.append(' },')
output.append(' },')
output.append('}')
output.append('')
def _output_field(self, output, name, sort=True):
value = getattr(self, name)
return self._write_value(output, name, value, sort)
def _write_value(self, output, name, value, sort=True):
if not value:
return
if isinstance(value, set):
value = sorted(value)
if isinstance(value, list):
output.append(' %s: [' % name)
for item in sorted(value) if sort else value:
output.append(' "%s",' % item)
output.append(' ],')
return
if isinstance(value, bool):
output.append(' %s: true,' % name)
return
output.append(' %s: "%s",' % (name, value))
class Blueprint(object):
"""In-memory representation of an Android.bp file."""
def __init__(self):
self.modules = {}
def add_module(self, module):
"""Adds a new module to the blueprint, replacing any existing module
with the same name.
Args:
module: Module instance.
"""
self.modules[module.name] = module
def to_string(self, output):
for m in sorted(self.modules.itervalues(), key=lambda m: m.name):
m.to_string(output)
def label_to_path(label):
"""Turn a GN output label (e.g., //some_dir/file.cc) into a path."""
assert label.startswith('//')
return label[2:]
def label_to_module_name(label):
"""Turn a GN label (e.g., //:perfetto_tests) into a module name."""
module = re.sub(r'^//:?', '', label)
module = re.sub(r'[^a-zA-Z0-9_]', '_', module)
if not module.startswith(module_prefix) and label not in default_targets:
return module_prefix + module
return module
def label_without_toolchain(label):
"""Strips the toolchain from a GN label.
Return a GN label (e.g //buildtools:protobuf(//gn/standalone/toolchain:
gcc_like_host) without the parenthesised toolchain part.
"""
return label.split('(')[0]
def is_supported_source_file(name):
"""Returns True if |name| can appear in a 'srcs' list."""
return os.path.splitext(name)[1] in ['.c', '.cc', '.proto']
def is_generated_by_action(desc, label):
"""Checks if a label is generated by an action.
Returns True if a GN output label |label| is an output for any action,
i.e., the file is generated dynamically.
"""
for target in desc.itervalues():
if target['type'] == 'action' and label in target['outputs']:
return True
return False
def apply_module_dependency(blueprint, desc, module, dep_name):
"""Recursively collect dependencies for a given module.
Walk the transitive dependencies for a GN target and apply them to a given
module. This effectively flattens the dependency tree so that |module|
directly contains all the sources, libraries, etc. in the corresponding GN
dependency tree.
Args:
blueprint: Blueprint instance which is being generated.
desc: JSON GN description.
module: Module to which dependencies should be added.
dep_name: GN target of the dependency.
"""
# If the dependency refers to a library which we can replace with an Android
# equivalent, stop recursing and patch the dependency in.
if label_without_toolchain(dep_name) in builtin_deps:
builtin_deps[label_without_toolchain(dep_name)](module)
return
# Similarly some shared libraries are directly mapped to Android
# equivalents.
target = desc[dep_name]
for lib in target.get('libs', []):
# Generally library names sould be mangled as 'libXXX', unless they are
# HAL libraries (e.g., android.hardware.health@2.0).
android_lib = lib if '@' in lib else 'lib' + lib
if lib in library_whitelist and not android_lib in module.shared_libs:
module.shared_libs.append(android_lib)
type = target['type']
if type == 'action':
if "gen_merged_sql_metrics" in dep_name:
dep_mod = create_merged_sql_metrics_target(blueprint, desc, dep_name)
module.generated_headers.append(dep_mod.name)
else:
create_modules_from_target(blueprint, desc, dep_name)
# Depend both on the generated sources and headers -- see
# make_genrules_for_action.
module.srcs.append(':' + label_to_module_name(dep_name))
module.generated_headers.append(
label_to_module_name(dep_name) + '_headers')
elif type == 'static_library' and label_to_module_name(
dep_name) != module.name:
create_modules_from_target(blueprint, desc, dep_name)
module.static_libs.append(label_to_module_name(dep_name))
elif type == 'shared_library' and label_to_module_name(
dep_name) != module.name:
module.shared_libs.append(label_to_module_name(dep_name))
elif type in ['group', 'source_set', 'executable', 'static_library'
] and 'sources' in target:
# Ignore source files that are generated by actions since they will be
# implicitly added by the genrule dependencies.
module.srcs.extend(
label_to_path(src) for src in target['sources']
if is_supported_source_file(src)
and not is_generated_by_action(desc, src))
module.cflags |= _get_cflags(target)
def make_genrules_for_action(blueprint, desc, target_name):
"""Generate genrules for a GN action.
GN actions are used to dynamically generate files during the build. The
Soong equivalent is a genrule. This function turns a specific kind of
genrule which turns .proto files into source and header files into a pair
equivalent genrules.
Args:
blueprint: Blueprint instance which is being generated.
desc: JSON GN description.
target_name: GN target for genrule generation.
Returns:
A (source_genrule, header_genrule) module tuple.
"""
target = desc[target_name]
# We only support genrules which call protoc (with or without a plugin) to
# turn .proto files into header and source files.
args = target['args']
if not args[0].endswith('/protoc'):
raise Error('Unsupported action in target %s: %s' % (target_name,
target['args']))
parser = ThrowingArgumentParser('Action in target %s (%s)' %
(target_name, ' '.join(target['args'])))
parser.add_argument('--proto_path')
parser.add_argument('--cpp_out')
parser.add_argument('--plugin')
parser.add_argument('--plugin_out')
parser.add_argument('--descriptor_set_out')
parser.add_argument('--include_imports', action='store_true')
parser.add_argument('protos', nargs=argparse.REMAINDER)
args = parser.parse_args(args[1:])
# Depending on whether we are using the default protoc C++ generator or the
# protozero plugin, the output dir is passed as:
# --cpp_out=gen/xxx or
# --plugin_out=:gen/xxx or
# --plugin_out=wrapper_namespace=pbzero:gen/xxx
gen_dir = args.cpp_out if args.cpp_out else args.plugin_out.split(':')[1]
assert gen_dir.startswith('gen/')
gen_dir = gen_dir[4:]
cpp_out_dir = ('$(genDir)/%s/%s' % (tree_path, gen_dir)).rstrip('/')
# TODO(skyostil): Is there a way to avoid hardcoding the tree path here?
# TODO(skyostil): Find a way to avoid creating the directory.
cmd = [
'mkdir -p %s &&' % cpp_out_dir,
'$(location aprotoc)',
'--cpp_out=%s' % cpp_out_dir
]
# We create two genrules for each action: one for the protobuf headers and
# another for the sources. This is because the module that depends on the
# generated files needs to declare two different types of dependencies --
# source files in 'srcs' and headers in 'generated_headers' -- and it's not
# valid to generate .h files from a source dependency and vice versa.
source_module = Module('genrule', label_to_module_name(target_name))
source_module.srcs.extend(label_to_path(src) for src in target['sources'])
source_module.tools = ['aprotoc']
header_module = Module('genrule',
label_to_module_name(target_name) + '_headers')
header_module.srcs = source_module.srcs[:]
header_module.tools = source_module.tools[:]
header_module.export_include_dirs = [gen_dir or '.']
# In GN builds the proto path is always relative to the output directory
# (out/tmp.xxx).
assert args.proto_path.startswith('../../')
cmd += [ '--proto_path=%s/%s' % (tree_path, args.proto_path[6:])]
namespaces = ['pb']
if args.plugin:
_, plugin = os.path.split(args.plugin)
# TODO(skyostil): Can we detect this some other way?
if plugin == 'ipc_plugin':
namespaces.append('ipc')
elif plugin == 'protoc_plugin':
namespaces = ['pbzero']
for dep in target['deps']:
if desc[dep]['type'] != 'executable':
continue
_, executable = os.path.split(desc[dep]['outputs'][0])
if executable == plugin:
cmd += [
'--plugin=protoc-gen-plugin=$(location %s)' %
label_to_module_name(dep)
]
source_module.tools.append(label_to_module_name(dep))
# Also make sure the module for the tool is generated.
create_modules_from_target(blueprint, desc, dep)
break
else:
raise Error('Unrecognized protoc plugin in target %s: %s' %
(target_name, args[i]))
if args.plugin_out:
plugin_args = args.plugin_out.split(':')[0]
cmd += ['--plugin_out=%s:%s' % (plugin_args, cpp_out_dir)]
cmd += ['$(in)']
source_module.cmd = ' '.join(cmd)
header_module.cmd = source_module.cmd
header_module.tools = source_module.tools[:]
for ns in namespaces:
source_module.out += [
'%s/%s' % (tree_path, src.replace('.proto', '.%s.cc' % ns))
for src in source_module.srcs
]
header_module.out += [
'%s/%s' % (tree_path, src.replace('.proto', '.%s.h' % ns))
for src in header_module.srcs
]
return source_module, header_module
def create_merged_sql_metrics_target(blueprint, desc, gn_target_name):
target_desc = desc[gn_target_name]
module = Module(
'genrule',
'gen_merged_sql_metrics',
)
module.tool_files = [
'tools/gen_merged_sql_metrics.py',
]
module.cmd = ' '.join([
'$(location tools/gen_merged_sql_metrics.py)',
'--cpp_out=$(out)',
'$(in)',
])
module.out = set(
src[src.index('gen/') + len('gen/'):]
for src in target_desc.get('outputs', [])
)
module.srcs.extend(
label_to_path(src)
for src in target_desc.get('inputs', [])
)
blueprint.add_module(module)
return module
def _get_cflags(target):
cflags = set(flag for flag in target.get('cflags', [])
if re.match(cflag_whitelist, flag))
cflags |= set("-D%s" % define for define in target.get('defines', [])
if re.match(define_whitelist, define))
return cflags
def create_modules_from_target(blueprint, desc, target_name):
"""Generate module(s) for a given GN target.
Given a GN target name, generate one or more corresponding modules into a
blueprint.
Args:
blueprint: Blueprint instance which is being generated.
desc: JSON GN description.
target_name: GN target for module generation.
"""
target = desc[target_name]
if target['type'] == 'executable':
if 'host' in target['toolchain'] or target_name in target_host_only:
module_type = 'cc_binary_host'
elif target.get('testonly'):
module_type = 'cc_test'
else:
module_type = 'cc_binary'
modules = [Module(module_type, label_to_module_name(target_name))]
elif target['type'] == 'action':
modules = make_genrules_for_action(blueprint, desc, target_name)
elif target['type'] == 'static_library':
module = Module('cc_library_static', label_to_module_name(target_name))
module.export_include_dirs = ['include']
modules = [module]
elif target['type'] == 'shared_library':
modules = [
Module('cc_library_shared', label_to_module_name(target_name))
]
else:
raise Error('Unknown target type: %s' % target['type'])
for module in modules:
module.comment = 'GN target: %s' % target_name
if target_name in target_initrc:
module.init_rc = [target_initrc[target_name]]
if target_name in target_host_supported:
module.host_supported = True
# Don't try to inject library/source dependencies into genrules because
# they are not compiled in the traditional sense.
if module.type != 'genrule':
module.defaults = [defaults_module]
apply_module_dependency(blueprint, desc, module, target_name)
for dep in resolve_dependencies(desc, target_name):
apply_module_dependency(blueprint, desc, module, dep)
# If the module is a static library, export all the generated headers.
if module.type == 'cc_library_static':
module.export_generated_headers = module.generated_headers
# Merge in additional hardcoded arguments.
for key, add_val in additional_args.get(module.name, []):
curr = getattr(module, key)
if add_val and isinstance(add_val, list) and isinstance(curr, list):
curr.extend(add_val)
else:
raise Error('Unimplemented type of additional_args')
blueprint.add_module(module)
def resolve_dependencies(desc, target_name):
"""Return the transitive set of dependent-on targets for a GN target.
Args:
blueprint: Blueprint instance which is being generated.
desc: JSON GN description.
Returns:
A set of transitive dependencies in the form of GN targets.
"""
if label_without_toolchain(target_name) in builtin_deps:
return set()
target = desc[target_name]
resolved_deps = set()
for dep in target.get('deps', []):
resolved_deps.add(dep)
# Ignore the transitive dependencies of actions because they are
# explicitly converted to genrules.
if desc[dep]['type'] == 'action':
continue
# Dependencies on shared libraries shouldn't propagate any transitive
# dependencies but only depend on the shared library target
if desc[dep]['type'] == 'shared_library':
continue
resolved_deps.update(resolve_dependencies(desc, dep))
return resolved_deps
def create_blueprint_for_targets(desc, targets):
"""Generate a blueprint for a list of GN targets."""
blueprint = Blueprint()
# Default settings used by all modules.
defaults = Module('cc_defaults', defaults_module)
defaults.local_include_dirs = ['include']
defaults.cflags = [
'-Wno-error=return-type',
'-Wno-sign-compare',
'-Wno-sign-promo',
'-Wno-unused-parameter',
'-fvisibility=hidden',
'-Oz',
]
defaults.user_debug_flag = True
blueprint.add_module(defaults)
for target in targets:
create_modules_from_target(blueprint, desc, target)
return blueprint
def repo_root():
"""Returns an absolute path to the repository root."""
return os.path.join(
os.path.realpath(os.path.dirname(__file__)), os.path.pardir)
def create_build_description():
"""Creates the JSON build description by running GN."""
out = os.path.join(repo_root(), 'out', 'tmp.gen_android_bp')
try:
try:
os.makedirs(out)
except OSError as e:
if e.errno != errno.EEXIST:
raise
subprocess.check_output(
['gn', 'gen', out, '--args=%s' % gn_args], cwd=repo_root())
desc = subprocess.check_output(
['gn', 'desc', out, '--format=json', '--all-toolchains', '//*'],
cwd=repo_root())
return json.loads(desc)
finally:
shutil.rmtree(out)
def main():
parser = argparse.ArgumentParser(
description='Generate Android.bp from a GN description.')
parser.add_argument(
'--desc',
help=
'GN description (e.g., gn desc out --format=json --all-toolchains "//*"'
)
parser.add_argument(
'--extras',
help='Extra targets to include at the end of the Blueprint file',
default=os.path.join(repo_root(), 'Android.bp.extras'),
)
parser.add_argument(
'--output',
help='Blueprint file to create',
default=os.path.join(repo_root(), 'Android.bp'),
)
parser.add_argument(
'targets',
nargs=argparse.REMAINDER,
help='Targets to include in the blueprint (e.g., "//:perfetto_tests")')
args = parser.parse_args()
if args.desc:
with open(args.desc) as f:
desc = json.load(f)
else:
desc = create_build_description()
blueprint = create_blueprint_for_targets(desc, args.targets or
default_targets)
output = [
"""// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// This file is automatically generated by %s. Do not edit.
""" % (__file__)
]
blueprint.to_string(output)
with open(args.extras, 'r') as r:
for line in r:
output.append(line.rstrip("\n\r"))
with open(args.output, 'w') as f:
f.write('\n'.join(output))
if __name__ == '__main__':
sys.exit(main())