tools/gen_build

#!/usr/bin/env python
# Copyright (C) 2018 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This tool translates a collection of BUILD.gn files into a mostly equivalent
# BUILD file for the Bazel build system. The input to the tool is a
# JSON description of the GN build definition generated with the following
# command:
#
#   gn desc out --format=json --all-toolchains "//*" > desc.json
#
# The tool is then given a list of GN labels for which to generate Bazel
# build rules.

from __future__ import print_function
import argparse
import errno
import functools
import json
import os
import re
import shutil
import subprocess
import sys
import textwrap

# Copyright header for generated code.
header = """# Copyright (C) 2019 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This file is automatically generated by {}. Do not edit.
""".format(__file__)

# Arguments for the GN output directory.
# host_os="linux" is to generate the right build files from Mac OS.
gn_args = 'target_os="linux" is_debug=false host_os="linux"'

# Default targets to translate to the blueprint file.
default_targets = [
  '//src/protozero:libprotozero',
  '//src/trace_processor:trace_processor',
  '//src/trace_processor:trace_processor_shell_host(//gn/standalone/toolchain:gcc_like_host)',
  '//tools/trace_to_text:trace_to_text_host(//gn/standalone/toolchain:gcc_like_host)',
  '//protos/perfetto/config:merged_config_gen',
  '//protos/perfetto/trace:merged_trace_gen',
]

# Aliases to add to the BUILD file
alias_targets = {
  '//src/protozero:libprotozero': 'libprotozero',
  '//src/trace_processor:trace_processor': 'trace_processor',
  '//src/trace_processor:trace_processor_shell_host': 'trace_processor_shell',
  '//tools/trace_to_text:trace_to_text_host': 'trace_to_text',
}


def enable_sqlite(module):
  module.deps.add(Label('//third_party/sqlite'))
  module.deps.add(Label('//third_party/sqlite:sqlite_ext_percentile'))


def enable_jsoncpp(module):
  module.deps.add(Label('//third_party/perfetto/google:jsoncpp'))


def enable_linenoise(module):
  module.deps.add(Label('//third_party/perfetto/google:linenoise'))


def enable_gtest_prod(module):
  module.deps.add(Label('//third_party/perfetto/google:gtest_prod'))


def enable_protobuf_full(module):
  module.deps.add(Label('//third_party/protobuf:libprotoc'))
  module.deps.add(Label('//third_party/protobuf'))


def enable_perfetto_version(module):
  module.deps.add(Label('//third_party/perfetto/google:perfetto_version'))


def disable_module(module):
  pass


# Internal equivalents for third-party libraries that the upstream project
# depends on.
builtin_deps = {
    '//gn:jsoncpp_deps': enable_jsoncpp,
    '//buildtools:linenoise': enable_linenoise,
    '//buildtools:protobuf_lite': disable_module,
    '//buildtools:protobuf_full': enable_protobuf_full,
    '//buildtools:protoc': disable_module,
    '//buildtools:sqlite': enable_sqlite,
    '//gn:default_deps': disable_module,
    '//gn:gtest_prod_config': enable_gtest_prod,
    '//gn:protoc_lib_deps': enable_protobuf_full,
    '//gn/standalone:gen_git_revision': enable_perfetto_version,
}

# ----------------------------------------------------------------------------
# End of configuration.
# ----------------------------------------------------------------------------


def check_output(cmd, cwd):
  try:
    output = subprocess.check_output(
        cmd, stderr=subprocess.STDOUT, cwd=cwd)
  except subprocess.CalledProcessError as e:
    print('Cmd "{}" failed in {}:'.format(
        ' '.join(cmd), cwd), file=sys.stderr)
    print(e.output)
    exit(1)
  else:
    return output


class Error(Exception):
  pass


def repo_root():
  """Returns an absolute path to the repository root."""
  return os.path.join(
      os.path.realpath(os.path.dirname(__file__)), os.path.pardir)


def create_build_description(repo_root):
  """Creates the JSON build description by running GN."""

  out = os.path.join(repo_root, 'out', 'tmp.gen_build')
  try:
    try:
      os.makedirs(out)
    except OSError as e:
      if e.errno != errno.EEXIST:
        raise
    check_output(
        ['gn', 'gen', out, '--args=%s' % gn_args], repo_root)
    desc = check_output(
        ['gn', 'desc', out, '--format=json', '--all-toolchains', '//*'],
        repo_root)
    return json.loads(desc)
  finally:
    shutil.rmtree(out)


def label_to_path(label):
  """Turn a GN output label (e.g., //some_dir/file.cc) into a path."""
  assert label.startswith('//')
  return label[2:]


def label_to_target_name_with_path(label):
  """
  Turn a GN label into a target name involving the full path.
  e.g., //src/perfetto:tests -> src_perfetto_tests
  """
  name = re.sub(r'^//:?', '', label)
  name = re.sub(r'[^a-zA-Z0-9_]', '_', name)
  return name


def label_without_toolchain(label):
  """Strips the toolchain from a GN label.

  Return a GN label (e.g //buildtools:protobuf(//gn/standalone/toolchain:
  gcc_like_host) without the parenthesised toolchain part.
  """
  return label.split('(')[0]


def is_public_header(label):
  """
  Returns if this is a c++ header file that is part of the API.
  Args:
      label: Label to evaluate
  """
  return label.endswith('.h') and label.startswith('//include/perfetto/')


@functools.total_ordering
class Label(object):
  """Represents a label in BUILD file terminology. This class wraps a string
  label to allow for correct comparision of labels for sorting.

  Args:
      label: The string rerepsentation of the label.
  """

  def __init__(self, label):
    self.label = label

  def is_absolute(self):
    return self.label.startswith('//')

  def dirname(self):
    return self.label.split(':')[0] if ':' in self.label else self.label

  def basename(self):
    return self.label.split(':')[1] if ':' in self.label else ''

  def __eq__(self, other):
    return self.label == other.label

  def __lt__(self, other):
    return (
        self.is_absolute(),
        self.dirname(),
        self.basename()
    ) < (
        other.is_absolute(),
        other.dirname(),
        other.basename()
    )

  def __str__(self):
    return self.label

  def __hash__(self):
    return hash(self.label)


class Writer(object):
  def __init__(self, output, width=79):
    self.output = output
    self.width = width

  def comment(self, text):
    for line in textwrap.wrap(text,
                              self.width - 2,
                              break_long_words=False,
                              break_on_hyphens=False):
      self.output.write('# {}\n'.format(line))

  def newline(self):
    self.output.write('\n')

  def line(self, s, indent=0):
    self.output.write('    ' * indent + s + '\n')

  def variable(self, key, value, sort=True):
    if value is None:
      return
    if isinstance(value, set) or isinstance(value, list):
      if len(value) == 0:
        return
      self.line('{} = ['.format(key), indent=1)
      for v in sorted(list(value)) if sort else value:
        self.line('"{}",'.format(v), indent=2)
      self.line('],', indent=1)
    elif isinstance(value, basestring):
      self.line('{} = "{}",'.format(key, value), indent=1)
    else:
      self.line('{} = {},'.format(key, value), indent=1)

  def header(self):
    self.output.write(header)


class Target(object):
  """In-memory representation of a BUILD target."""

  def __init__(self, type, name, gn_name=None):
    assert type in ('cc_binary', 'cc_library', 'cc_proto_library',
                    'proto_library', 'filegroup', 'alias',
                    'pbzero_cc_proto_library', 'genrule', )
    self.type = type
    self.name = name
    self.srcs = set()
    self.hdrs = set()
    self.deps = set()
    self.visibility = set()
    self.gn_name = gn_name
    self.is_pbzero = False
    self.src_proto_library = None
    self.outs = set()
    self.cmd = None
    self.tools = set()

  def write(self, writer):
    if self.gn_name:
      writer.comment('GN target: {}'.format(self.gn_name))

    writer.line('{}('.format(self.type))
    writer.variable('name', self.name)
    writer.variable('srcs', self.srcs)
    writer.variable('hdrs', self.hdrs)

    if self.type == 'proto_library' and not self.is_pbzero:
      if self.srcs:
        writer.variable('has_services', 1)
      writer.variable('cc_api_version', 2)
      if self.srcs:
        writer.variable('cc_generic_services', 1)

    writer.variable('src_proto_library', self.src_proto_library)

    writer.variable('outs', self.outs)
    writer.variable('cmd', self.cmd)
    writer.variable('tools', self.tools)

    # Keep visibility and deps last.
    writer.variable('visibility', self.visibility)

    if type != 'filegroup':
      writer.variable('deps', self.deps)

    writer.line(')')


class Build(object):
  """In-memory representation of a BUILD file."""

  def __init__(self, public, header_lines=[]):
    self.targets = {}
    self.public = public
    self.header_lines = header_lines

  def add_target(self, target):
    self.targets[target.name] = target

  def write(self, writer):
    writer.header()
    writer.newline()
    for line in self.header_lines:
      writer.line(line)
    if self.header_lines:
      writer.newline()
    if self.public:
      writer.line(
          'package(default_visibility = ["//visibility:public"])')
    else:
      writer.line(
          'package(default_visibility = ["//third_party/perfetto:__subpackages__"])')
    writer.newline()
    writer.line('licenses(["notice"])  # Apache 2.0')
    writer.newline()
    writer.line('exports_files(["LICENSE"])')
    writer.newline()

    sorted_targets = sorted(
        self.targets.itervalues(), key=lambda m: m.name)
    for target in sorted_targets[:-1]:
      target.write(writer)
      writer.newline()

    # BUILD files shouldn't have a trailing new line.
    sorted_targets[-1].write(writer)


class BuildGenerator(object):
  def __init__(self, desc):
    self.desc = desc
    self.action_generated_files = set()

    for target in self.desc.itervalues():
      if target['type'] == 'action':
        self.action_generated_files.update(target['outputs'])


  def create_build_for_targets(self, targets):
    """Generate a BUILD for a list of GN targets and aliases."""
    self.build = Build(public=True)

    proto_cc_import = 'load("//tools/build_defs/proto/cpp:cc_proto_library.bzl", "cc_proto_library")'
    pbzero_cc_import = 'load("//third_party/perfetto/google:build_defs.bzl", "pbzero_cc_proto_library")'
    self.proto_build = Build(public=False, header_lines=[
                        proto_cc_import, pbzero_cc_import])

    for target in targets:
      self.create_target(target)

    return (self.build, self.proto_build)


  def resolve_dependencies(self, target_name):
    """Return the set of direct dependent-on targets for a GN target.

    Args:
        desc: JSON GN description.
        target_name: Name of target

    Returns:
        A set of transitive dependencies in the form of GN targets.
    """

    if label_without_toolchain(target_name) in builtin_deps:
      return set()
    target = self.desc[target_name]
    resolved_deps = set()
    for dep in target.get('deps', []):
      resolved_deps.add(dep)
    return resolved_deps


  def apply_module_sources_to_target(self, target, module_desc):
    """
    Args:
        target: Module to which dependencies should be added.
        module_desc: JSON GN description of the module.
        visibility: Whether the module is visible with respect to the target.
    """
    for src in module_desc['sources']:
      label = Label(label_to_path(src))
      if target.type == 'cc_library' and is_public_header(src):
        target.hdrs.add(label)
      else:
        target.srcs.add(label)


  def apply_module_dependency(self, target, dep_name):
    """
    Args:
        build: BUILD instance which is being generated.
        proto_build: BUILD instance which is being generated to hold protos.
        desc: JSON GN description.
        target: Module to which dependencies should be added.
        dep_name: GN target of the dependency.
    """
    # If the dependency refers to a library which we can replace with an internal
    # equivalent, stop recursing and patch the dependency in.
    dep_name_no_toolchain = label_without_toolchain(dep_name)
    if dep_name_no_toolchain in builtin_deps:
      builtin_deps[dep_name_no_toolchain](target)
      return

    dep_desc = self.desc[dep_name]
    if dep_desc['type'] == 'source_set':
      for inner_name in self.resolve_dependencies(dep_name):
        self.apply_module_dependency(target, inner_name)

      # Any source set which has a source generated by an action doesn't need
      # to be depended on as we will depend on the action directly.
      if any(src in self.action_generated_files for src in dep_desc['sources']):
        return

      self.apply_module_sources_to_target(target, dep_desc)
    elif dep_desc['type'] == 'action':
      args = dep_desc['args']
      if "gen_merged_sql_metrics" in dep_name:
        dep_target = self.create_merged_sql_metrics_target(dep_name)
        target.deps.add(Label("//third_party/perfetto:" + dep_target.name))

        if target.type == 'cc_library' or target.type == 'cc_binary':
          target.srcs.update(dep_target.outs)
      elif args[0].endswith('/protoc'):
        (proto_target, cc_target) = self.create_proto_target(dep_name)
        if target.type == 'proto_library':
          dep_target_name = proto_target.name
        else:
          dep_target_name = cc_target.name
        target.deps.add(
            Label("//third_party/perfetto/protos:" + dep_target_name))
      else:
        raise Error('Unsupported action in target %s: %s' % (dep_target_name,
                                                            args))
    elif dep_desc['type'] == 'static_library':
      dep_target = self.create_target(dep_name)
      target.deps.add(Label("//third_party/perfetto:" + dep_target.name))
    elif dep_desc['type'] == 'group':
      for inner_name in self.resolve_dependencies(dep_name):
        self.apply_module_dependency(target, inner_name)
    elif dep_desc['type'] == 'executable':
      # Just create the dep target but don't add it as a dep because it's an
      # executable.
      self.create_target(dep_name)
    else:
      raise Error('Unknown target name %s with type: %s' %
                  (dep_name, dep_desc['type']))

  def create_merged_sql_metrics_target(self, gn_target_name):
    target_desc = self.desc[gn_target_name]
    gn_target_name_no_toolchain = label_without_toolchain(gn_target_name)
    target = Target(
      'genrule',
      'gen_merged_sql_metrics',
      gn_name=gn_target_name_no_toolchain,
    )
    target.outs.update(
      Label(src[src.index('gen/') + len('gen/'):])
      for src in target_desc.get('outputs', [])
    )
    target.cmd = '$(location gen_merged_sql_metrics_py) --cpp_out=$@ $(SRCS)'
    target.tools.update([
      'gen_merged_sql_metrics_py',
    ])
    target.srcs.update(
      Label(label_to_path(src))
      for src in target_desc.get('inputs', [])
      if src not in self.action_generated_files
    )
    self.build.add_target(target)
    return target

  def create_proto_target(self, gn_target_name):
    target_desc = self.desc[gn_target_name]
    args = target_desc['args']

    gn_target_name_no_toolchain = label_without_toolchain(gn_target_name)
    stripped_path = gn_target_name_no_toolchain.replace("protos/perfetto/", "")
    pretty_target_name = label_to_target_name_with_path(stripped_path)
    pretty_target_name = pretty_target_name.replace("_lite_gen", "")
    pretty_target_name = pretty_target_name.replace("_zero_gen", "_zero")

    proto_target = Target(
      'proto_library',
      pretty_target_name,
      gn_name=gn_target_name_no_toolchain
    )
    proto_target.is_pbzero = any("pbzero" in arg for arg in args)
    proto_target.srcs.update([
      Label(label_to_path(src).replace('protos/', ''))
      for src in target_desc.get('sources', [])
    ])
    if not proto_target.is_pbzero:
      proto_target.visibility.add("//visibility:public")
    self.proto_build.add_target(proto_target)

    for dep_name in self.resolve_dependencies(gn_target_name):
      self.apply_module_dependency(proto_target, dep_name)

    if proto_target.is_pbzero:
      # Remove all the protozero srcs from the proto_library.
      proto_target.srcs.difference_update(
          [src for src in proto_target.srcs if not src.label.endswith('.proto')])

      # Remove all the non-proto deps from the proto_library and add to the cc
      # library.
      cc_deps = [
        dep for dep in proto_target.deps
        if not dep.label.startswith('//third_party/perfetto/protos')
      ]
      proto_target.deps.difference_update(cc_deps)

      cc_target_name = proto_target.name + "_cc_proto"
      cc_target = Target('pbzero_cc_proto_library', cc_target_name,
                         gn_name=gn_target_name_no_toolchain)

      cc_target.deps.add(Label('//third_party/perfetto:libprotozero'))
      cc_target.deps.update(cc_deps)

      # Add the proto_library to the cc_target.
      cc_target.src_proto_library = \
          "//third_party/perfetto/protos:" + proto_target.name

      self.proto_build.add_target(cc_target)
    else:
      cc_target_name = proto_target.name + "_cc_proto"
      cc_target = Target('cc_proto_library',
                        cc_target_name, gn_name=gn_target_name_no_toolchain)
      cc_target.visibility.add("//visibility:public")
      cc_target.deps.add(
          Label("//third_party/perfetto/protos:" + proto_target.name))
      self.proto_build.add_target(cc_target)

    return (proto_target, cc_target)


  def create_target(self, gn_target_name):
    """Generate module(s) for a given GN target.

    Given a GN target name, generate one or more corresponding modules into a
    build file.

    Args:
        build: Build instance which is being generated.
        desc: JSON GN description.
        gn_target_name: GN target name for module generation.
    """

    target_desc = self.desc[gn_target_name]
    if target_desc['type'] == 'action':
      args = target_desc['args']
      if args[0].endswith('/protoc'):
        return self.create_proto_target(gn_target_name)
      else:
        raise Error('Unsupported action in target %s: %s' % (gn_target_name,
                                                            args))
    elif target_desc['type'] == 'executable':
      target_type = 'cc_binary'
    elif target_desc['type'] == 'static_library':
      target_type = 'cc_library'
    elif target_desc['type'] == 'source_set':
      target_type = 'filegroup'
    else:
      raise Error('Unknown target type: %s' % target_desc['type'])

    label_no_toolchain = label_without_toolchain(gn_target_name)
    target_name_path = label_to_target_name_with_path(label_no_toolchain)
    target_name = alias_targets.get(label_no_toolchain, target_name_path)
    target = Target(target_type, target_name, gn_name=label_no_toolchain)
    target.srcs.update(
        Label(label_to_path(src))
        for src in target_desc.get('sources', [])
        if src not in self.action_generated_files
    )

    for dep_name in self.resolve_dependencies(gn_target_name):
      self.apply_module_dependency(target, dep_name)

    self.build.add_target(target)
    return target

def main():
  parser = argparse.ArgumentParser(
      description='Generate BUILD from a GN description.')
  parser.add_argument(
      '--desc',
      help='GN description (e.g., gn desc out --format=json --all-toolchains "//*"'
  )
  parser.add_argument(
      '--repo-root',
      help='Standalone Perfetto repository to generate a GN description',
      default=repo_root(),
  )
  parser.add_argument(
      '--extras',
      help='Extra targets to include at the end of the BUILD file',
      default=os.path.join(repo_root(), 'BUILD.extras'),
  )
  parser.add_argument(
      '--output',
      help='BUILD file to create',
      default=os.path.join(repo_root(), 'BUILD'),
  )
  parser.add_argument(
      '--output-proto',
      help='Proto BUILD file to create',
      default=os.path.join(repo_root(), 'protos', 'BUILD'),
  )
  parser.add_argument(
      'targets',
      nargs=argparse.REMAINDER,
      help='Targets to include in the BUILD file (e.g., "//:perfetto_tests")')
  args = parser.parse_args()

  if args.desc:
    with open(args.desc) as f:
      desc = json.load(f)
  else:
    desc = create_build_description(args.repo_root)

  build_generator = BuildGenerator(desc)
  build, proto_build = build_generator.create_build_for_targets(
      args.targets or default_targets)
  with open(args.output, 'w') as f:
    writer = Writer(f)
    build.write(writer)
    writer.newline()

    with open(args.extras, 'r') as r:
      for line in r:
        writer.line(line.rstrip("\n\r"))

  with open(args.output_proto, 'w') as f:
    proto_build.write(Writer(f))

  return 0


if __name__ == '__main__':
  sys.exit(main())