test/etc/default-build

#!/usr/bin/env python3
#
# Copyright (C) 2021 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This is the default build script for run-tests.

It can be overwrite by specific run-tests if needed.
It is used from soong build and not intended to be called directly.
"""

import argparse
import functools
import glob
import os
from os import path
import shlex
import shutil
import subprocess
import tempfile
import zipfile

if not os.sys.argv:
  print(
      'Error: default-build should have the parameters from the "build" script forwarded to it'
  )
  print('Error: An example of how do it correctly is ./default-build "$@"')
  os.sys.exit(1)


def parse_bool(text):
  return {"true": True, "false": False}[text.lower()]


TEST_NAME = os.environ["TEST_NAME"]
ART_TEST_RUN_TEST_BOOTCLASSPATH = os.environ["ART_TEST_RUN_TEST_BOOTCLASSPATH"]
NEED_DEX = parse_bool(os.environ["NEED_DEX"])

# Set default values for directories.
HAS_SMALI = path.exists("smali")
HAS_JASMIN = path.exists("jasmin")
HAS_SRC = path.exists("src")
HAS_SRC_ART = path.exists("src-art")
HAS_SRC2 = path.exists("src2")
HAS_SRC_MULTIDEX = path.exists("src-multidex")
HAS_SMALI_MULTIDEX = path.exists("smali-multidex")
HAS_JASMIN_MULTIDEX = path.exists("jasmin-multidex")
HAS_SMALI_EX = path.exists("smali-ex")
HAS_SRC_EX = path.exists("src-ex")
HAS_SRC_EX2 = path.exists("src-ex2")
HAS_SRC_DEX2OAT_UNRESOLVED = path.exists("src-dex2oat-unresolved")
HAS_HIDDENAPI_SPEC = path.exists("hiddenapi-flags.csv")

# USE_HIDDENAPI=false run-test... will disable hiddenapi.
USE_HIDDENAPI = parse_bool(os.environ.get("USE_HIDDENAPI", "true"))

# USE_DESUGAR=false run-test... will disable desugaring.
USE_DESUGAR = parse_bool(os.environ.get("USE_DESUGAR", "true"))

JAVAC_ARGS = shlex.split(os.environ.get("JAVAC_ARGS", ""))
SMALI_ARGS = shlex.split(os.environ.get("SMALI_ARGS", ""))
D8_FLAGS = shlex.split(os.environ.get("D8_FLAGS", ""))

# Allow overriding ZIP_COMPRESSION_METHOD with e.g. 'store'
ZIP_COMPRESSION_METHOD = "deflate"
# Align every ZIP file made by calling $ZIPALIGN command?
ZIP_ALIGN_BYTES = None

DEV_MODE = False
BUILD_MODE = "target"
API_LEVEL = None
DEFAULT_EXPERIMENT = "no-experiment"
EXPERIMENTAL = DEFAULT_EXPERIMENT

# Setup experimental API level mappings in a bash associative array.
EXPERIMENTAL_API_LEVEL = {}
EXPERIMENTAL_API_LEVEL[DEFAULT_EXPERIMENT] = "26"
EXPERIMENTAL_API_LEVEL["default-methods"] = "24"
EXPERIMENTAL_API_LEVEL["parameter-annotations"] = "25"
EXPERIMENTAL_API_LEVEL["agents"] = "26"
EXPERIMENTAL_API_LEVEL["method-handles"] = "26"
EXPERIMENTAL_API_LEVEL["var-handles"] = "28"

# Parse command line arguments.
opt_bool = argparse.BooleanOptionalAction  # Bool also accepts the --no- prefix.
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--src", dest="HAS_SRC", action=opt_bool)
parser.add_argument("--src2", dest="HAS_SRC2", action=opt_bool)
parser.add_argument("--src-multidex", dest="HAS_SRC_MULTIDEX", action=opt_bool)
parser.add_argument(
    "--smali-multidex", dest="HAS_SMALI_MULTIDEX", action=opt_bool)
parser.add_argument("--src-ex", dest="HAS_SRC_EX", action=opt_bool)
parser.add_argument("--src-ex2", dest="HAS_SRC_EX2", action=opt_bool)
parser.add_argument("--smali", dest="HAS_SMALI", action=opt_bool)
parser.add_argument("--jasmin", dest="HAS_JASMIN", action=opt_bool)
parser.add_argument("--api-level", dest="API_LEVEL", type=int)
parser.add_argument(
    "--experimental", dest="EXPERIMENTAL", type=str)
parser.add_argument(
    "--zip-compression-method", dest="ZIP_COMPRESSION_METHOD", type=str)
parser.add_argument("--zip-align", dest="ZIP_ALIGN_BYTES", type=int)
parser.add_argument(
    "--host", dest="BUILD_MODE", action="store_const", const="host")
parser.add_argument(
    "--target", dest="BUILD_MODE", action="store_const", const="target")
parser.add_argument(
    "--jvm", dest="BUILD_MODE", action="store_const", const="jvm")
parser.add_argument("--dev", dest="DEV_MODE", action=opt_bool)
# Update variables with command line arguments that were set.
globals().update(
    {k: v for k, v in parser.parse_args().__dict__.items() if v is not None})

if BUILD_MODE == "jvm":
  # No desugaring on jvm because it supports the latest functionality.
  USE_DESUGAR = False
  # Do not attempt to build src-art directories on jvm,
  # since it would fail without libcore.
  HAS_SRC_ART = False

# Set API level for smali and d8.
if not API_LEVEL:
  API_LEVEL = EXPERIMENTAL_API_LEVEL[EXPERIMENTAL]

# Add API level arguments to smali and dx
SMALI_ARGS.extend(["--api", str(API_LEVEL)])
D8_FLAGS.extend(["--min-api", str(API_LEVEL)])


def run(executable, args):
  cmd = shlex.split(executable) + args
  if executable.endswith(".sh"):
    cmd = ["/bin/bash"] + cmd
  if DEV_MODE:
    print("Run:", " ".join(cmd))
  p = subprocess.run(cmd, check=True)
  if p.returncode != 0:
    raise Exception("Failed command: " + " ".join(cmd))


# Helper functions to execute tools.
soong_zip = functools.partial(run, os.environ["SOONG_ZIP"])
zipalign = functools.partial(run, os.environ["ZIPALIGN"])
javac = functools.partial(run, os.environ["JAVAC"])
jasmin = functools.partial(run, os.environ["JASMIN"])
smali = functools.partial(run, os.environ["SMALI"])
d8 = functools.partial(run, os.environ["D8"])
hiddenapi = functools.partial(run, os.environ["HIDDENAPI"])


def find(root, name):
  return sorted(glob.glob(path.join(root, "**", name), recursive=True))


def zip(zip_target, *files):
  zip_args = ["-o", zip_target]
  if ZIP_COMPRESSION_METHOD == "store":
    zip_args.extend(["-L", "0"])
  for f in files:
    zip_args.extend(["-f", f])
  soong_zip(zip_args)

  if ZIP_ALIGN_BYTES:
    # zipalign does not operate in-place, so write results to a temp file.
    with tempfile.TemporaryDirectory(dir=".") as tmp_dir:
      tmp_file = path.join(tmp_dir, "aligned.zip")
      zipalign(["-f", str(ZIP_ALIGN_BYTES), zip_target, tmp_file])
      # replace original zip target with our temp file.
      os.rename(tmp_file, zip_target)


def make_jasmin(out_directory, jasmin_sources):
  os.makedirs(out_directory, exist_ok=True)
  jasmin(["-d", out_directory] + sorted(jasmin_sources))


# Like regular javac but may include libcore on the bootclasspath.
def javac_with_bootclasspath(args):
  flags = JAVAC_ARGS + ["-encoding", "utf8"]
  if BUILD_MODE != "jvm":
    flags.extend(["-bootclasspath", ART_TEST_RUN_TEST_BOOTCLASSPATH])
  javac(flags + args)


# Make a "dex" file given a directory of classes. This will be
# packaged in a jar file.
def make_dex(name):
  d8_inputs = find(name, "*.class")
  d8_output = name + ".jar"
  dex_output = name + ".dex"
  if USE_DESUGAR:
    flags = ["--lib", ART_TEST_RUN_TEST_BOOTCLASSPATH]
  else:
    flags = ["--no-desugaring"]
  assert d8_inputs
  d8(D8_FLAGS + flags + ["--output", d8_output] + d8_inputs)

  # D8 outputs to JAR files today rather than DEX files as DX used
  # to. To compensate, we extract the DEX from d8's output to meet the
  # expectations of make_dex callers.
  with tempfile.TemporaryDirectory(dir=".") as tmp_dir:
    zipfile.ZipFile(d8_output, "r").extractall(tmp_dir)
    os.rename(path.join(tmp_dir, "classes.dex"), dex_output)


# Merge all the dex files.
# Skip non-existing files, but at least 1 file must exist.
def make_dexmerge(*dex_files_to_merge):
  # Dex file that acts as the destination.
  dst_file = dex_files_to_merge[0]

  # Skip any non-existing files.
  dex_files_to_merge = list(filter(path.exists, dex_files_to_merge))

  # NB: We merge even if there is just single input.
  # It is useful to normalize non-deterministic smali output.

  with tempfile.TemporaryDirectory(dir=".") as tmp_dir:
    d8(["--min-api", API_LEVEL, "--output", tmp_dir] + dex_files_to_merge)
    assert not path.exists(path.join(tmp_dir, "classes2.dex"))
    for input_dex in dex_files_to_merge:
      os.remove(input_dex)
    os.rename(path.join(tmp_dir, "classes.dex"), dst_file)


def make_hiddenapi(*dex_files):
  args = ["encode"]
  for dex_file in dex_files:
    args.extend(["--input-dex=" + dex_file, "--output-dex=" + dex_file])
  args.append("--api-flags=hiddenapi-flags.csv")
  args.append("--no-force-assign-all")
  hiddenapi(args)


if path.exists("classes.dex"):
  zip(TEST_NAME + ".jar", "classes.dex")
  os.sys.exit(0)


def has_multidex():
  return HAS_SRC_MULTIDEX or HAS_JASMIN_MULTIDEX or HAS_SMALI_MULTIDEX


if HAS_SRC_DEX2OAT_UNRESOLVED:
  os.makedirs("classes", exist_ok=True)
  os.makedirs("classes-ex")
  javac_with_bootclasspath([
      "-implicit:none", "-sourcepath", "src-dex2oat-unresolved", "-d", "classes"
  ] + find("src", "*.java"))
  javac_with_bootclasspath(
      ["-implicit:none", "-sourcepath", "src", "-d", "classes-ex"] +
      find("src-dex2oat-unresolved", "*.java"))
  if NEED_DEX:
    make_dex("classes-ex")
    # rename it so it shows up as "classes.dex" in the zip file.
    os.rename("classes-ex.dex", "classes.dex")
    zip(TEST_NAME + "-ex.jar", "classes.dex")
    make_dex("classes")
else:
  src_tmp_all = []
  if HAS_SRC and HAS_SRC_MULTIDEX:
    # To allow circular references, compile src/ and src-multidex/ together
    # and pass the output as class path argument. Replacement sources
    # in src-art/ can replace symbols used by src-multidex but everything
    # needed to compile src-multidex should be present in src/.
    os.makedirs("classes-tmp-all")
    javac_with_bootclasspath(["-implicit:none", "-d", "classes-tmp-all"] +
                             find("src", "*.java") +
                             find("src-multidex", "*.java"))
    src_tmp_all = ["-cp", "classes-tmp-all"]

  if HAS_SRC:
    os.makedirs("classes", exist_ok=True)
    javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
                             ["-d", "classes"] + find("src", "*.java"))

  if HAS_SRC_ART:
    os.makedirs("classes", exist_ok=True)
    javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
                             ["-d", "classes"] + find("src-art", "*.java"))

  if HAS_SRC_MULTIDEX:
    os.makedirs("classes2")
    javac_with_bootclasspath(["-implicit:none"] + src_tmp_all +
                             ["-d", "classes2"] +
                             find("src-multidex", "*.java"))
    if NEED_DEX:
      make_dex("classes2")

  if HAS_SRC2:
    os.makedirs("classes", exist_ok=True)
    javac_with_bootclasspath(["-classpath", "classes", "-d", "classes"] +
                             find("src2", "*.java"))

  # If the classes directory is not-empty, package classes in a DEX file.
  # NB: some tests provide classes rather than java files.
  if find("classes", "*"):
    if NEED_DEX:
      make_dex("classes")

if HAS_JASMIN:
  # Compile Jasmin classes as if they were part of the classes.dex file.
  make_jasmin("jasmin_classes", find("jasmin", "*.j"))
  if NEED_DEX:
    make_dex("jasmin_classes")
    make_dexmerge("classes.dex", "jasmin_classes.dex")
  else:
    # Move jasmin classes into classes directory so that they are picked up
    # with -cp classes.
    os.makedirs("classes", exist_ok=True)
    shutil.copytree("jasmin_classes", "classes", dirs_exist_ok=True)

if HAS_SMALI and NEED_DEX:
  # Compile Smali classes
  smali(["-JXmx512m", "assemble"] + SMALI_ARGS +
        ["--output", "smali_classes.dex"] + find("smali", "*.smali"))
  assert path.exists("smali_classes.dex")
  # Merge smali files into classes.dex,
  # this takes priority over any jasmin files.
  make_dexmerge("classes.dex", "smali_classes.dex")

# Compile Jasmin classes in jasmin-multidex as if they were part of
# the classes2.jar
if HAS_JASMIN_MULTIDEX:
  make_jasmin("jasmin_classes2", find("jasmin-multidex", "*.j"))

  if NEED_DEX:
    make_dex("jasmin_classes2")
    make_dexmerge("classes2.dex", "jasmin_classes2.dex")
  else:
    # Move jasmin classes into classes2 directory so that
    # they are picked up with -cp classes2.
    os.makedirs("classes2", exist_ok=True)
    shutil.copytree("jasmin_classes2", "classes2", dirs_exist_ok=True)
    shutil.rmtree("jasmin_classes2")

if HAS_SMALI_MULTIDEX and NEED_DEX:
  # Compile Smali classes
  smali(["-JXmx512m", "assemble"] + SMALI_ARGS +
        ["--output", "smali_classes2.dex"] + find("smali-multidex", "*.smali"))

  # Merge smali_classes2.dex into classes2.dex
  make_dexmerge("classes2.dex", "smali_classes2.dex")

if HAS_SRC_EX or HAS_SRC_EX2:
  # Build src-ex into classes-ex.
  # Includes 'src', 'src-art' and 'jasmin' source when compiling classes-ex,
  # but exclude their .class files.
  src_tmp_for_ex = []
  if HAS_SRC or HAS_SRC_ART or HAS_JASMIN:
    os.makedirs("classes-tmp-for-ex", exist_ok=True)
    src_tmp_for_ex = ["-cp", "classes-tmp-for-ex"]
  if HAS_SRC and HAS_SRC_MULTIDEX:
    javac_with_bootclasspath(["-d", "classes-tmp-for-ex"] +
                             find("src", "*.java") +
                             find("src-multidex", "*.java"))
  elif HAS_SRC:
    javac_with_bootclasspath(["-d", "classes-tmp-for-ex"] +
                             find("src", "*.java"))
  elif HAS_SRC_MULTIDEX:
    javac_with_bootclasspath(["-d", "classes-tmp-for-ex"] +
                             find("src-multidex", "*.java"))
  if HAS_SRC_ART:
    javac_with_bootclasspath(["-d", "classes-tmp-for-ex"] +
                             find("src-art", "*.java"))
  if HAS_JASMIN:
    make_jasmin("classes-tmp-for-ex", find("jasmin", "*.j"))
  os.makedirs("classes-ex", exist_ok=True)
  if HAS_SRC_EX:
    javac_with_bootclasspath(["-d", "classes-ex"] + src_tmp_for_ex +
                             find("src-ex", "*.java"))
    if not src_tmp_for_ex:
      src_tmp_for_ex = ["-cp", "classes-ex"]
    else:
      src_tmp_for_ex[-1] += ":classes-ex"
  if HAS_SRC_EX2:
    javac_with_bootclasspath(["-d", "classes-ex"] + src_tmp_for_ex +
                             find("src-ex2", "*.java"))

if path.exists("classes-ex") and NEED_DEX:
  make_dex("classes-ex")

if HAS_SMALI_EX and NEED_DEX:
  # Compile Smali classes
  smali(["-JXmx512m", "assemble"] + SMALI_ARGS +
        ["--output", "smali_classes-ex.dex"] + find("smali-ex", "*.smali"))
  assert path.exists("smali_classes-ex.dex")
  # Merge smali files into classes-ex.dex.
  make_dexmerge("classes-ex.dex", "smali_classes-ex.dex")

if path.exists("classes-ex.dex"):
  # Apply hiddenapi on the dex files if the test has API list file(s).
  if USE_HIDDENAPI and HAS_HIDDENAPI_SPEC:
    make_hiddenapi("classes-ex.dex")

  # quick shuffle so that the stored name is "classes.dex"
  os.rename("classes.dex", "classes-1.dex")
  os.rename("classes-ex.dex", "classes.dex")
  zip(TEST_NAME + "-ex.jar", "classes.dex")
  os.rename("classes.dex", "classes-ex.dex")
  os.rename("classes-1.dex", "classes.dex")

# Apply hiddenapi on the dex files if the test has API list file(s).
if NEED_DEX and USE_HIDDENAPI and HAS_HIDDENAPI_SPEC:
  if has_multidex():
    make_hiddenapi("classes.dex", "classes2.dex")
  else:
    make_hiddenapi("classes.dex")

# Create a single dex jar with two dex files for multidex.
if NEED_DEX:
  if path.exists("classes2.dex"):
    zip(TEST_NAME + ".jar", "classes.dex", "classes2.dex")
  else:
    zip(TEST_NAME + ".jar", "classes.dex")