test/utils/regen-test-files

#! /usr/bin/env python3
#
# Copyright 2020 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Regenerate ART run-tests Blueprint files.

# This script handles only a subset of ART run-tests at the moment; additional
# cases will be added later.

import collections
import json
import logging
import os
import re
import sys
import textwrap

me = os.path.basename(sys.argv[0])

# Relative path to ART's tests directory from ART's source directory.
TESTS_DIR = "test"

# Indentation unit.
INDENT = "  "

def reindent(str, indent_level = 0):
  """Reindent literal string while removing common leading spaces."""
  return textwrap.indent(textwrap.dedent(str), INDENT * indent_level)

# Known failing ART run-tests.
# TODO(rpl): Investigate and address the causes of failures.
known_failing_tests = frozenset([
  "004-SignalTest",
  "004-UnsafeTest",
  "030-bad-finalizer",
  "034-call-null",
  "038-inner-null",
  "044-proxy",
  "051-thread",
  "054-uncaught",
  "086-null-super",
  "087-gc-after-link",
  "096-array-copy-concurrent-gc",
  "115-native-bridge",
  "116-nodex2oat",
  "1336-short-finalizer-timeout",
  "1337-gc-coverage",
  "1339-dead-reference-safe",
  "134-nodex2oat-nofallback",
  "136-daemon-jni-shutdown",
  "139-register-natives",
  "148-multithread-gc-annotations",
  "149-suspend-all-stress",
  "150-loadlibrary",
  "154-gc-loop",
  "158-app-image-class-table",
  "169-threadgroup-jni",
  "172-app-image-twice",
  "177-visibly-initialized-deadlock",
  "178-app-image-native-method",
  "179-nonvirtual-jni",
  "1900-track-alloc",
  "1901-get-bytecodes",
  "1902-suspend",
  "1903-suspend-self",
  "1904-double-suspend",
  "1905-suspend-native",
  "1906-suspend-list-me-first",
  "1907-suspend-list-self-twice",
  "1908-suspend-native-resume-self",
  "1909-per-agent-tls",
  "1910-transform-with-default",
  "1911-get-local-var-table",
  "1912-get-set-local-primitive",
  "1913-get-set-local-objects",
  "1914-get-local-instance",
  "1915-get-set-local-current-thread",
  "1916-get-set-current-frame",
  "1917-get-stack-frame",
  "1919-vminit-thread-start-timing",
  "1920-suspend-native-monitor",
  "1921-suspend-native-recursive-monitor",
  "1922-owned-monitors-info",
  "1923-frame-pop",
  "1924-frame-pop-toggle",
  "1925-self-frame-pop",
  "1926-missed-frame-pop",
  "1927-exception-event",
  "1928-exception-event-exception",
  "1930-monitor-info",
  "1931-monitor-events",
  "1932-monitor-events-misc",
  "1933-monitor-current-contended",
  "1934-jvmti-signal-thread",
  "1935-get-set-current-frame-jit",
  "1936-thread-end-events",
  "1937-transform-soft-fail",
  "1938-transform-abstract-single-impl",
  "1939-proxy-frames",
  "1941-dispose-stress",
  "1942-suspend-raw-monitor-exit",
  "1943-suspend-raw-monitor-wait",
  "1945-proxy-method-arguments",
  "1947-breakpoint-redefine-deopt",
  "1949-short-dex-file",
  "1951-monitor-enter-no-suspend",
  "1953-pop-frame",
  "1954-pop-frame-jit",
  "1955-pop-frame-jit-called",
  "1956-pop-frame-jit-calling",
  "1957-error-ext",
  "1958-transform-try-jit",
  "1959-redefine-object-instrument",
  "1960-obsolete-jit-multithread-native",
  "1961-obsolete-jit-multithread",
  "1962-multi-thread-events",
  "1963-add-to-dex-classloader-in-memory",
  "1967-get-set-local-bad-slot",
  "1968-force-early-return",
  "1969-force-early-return-void",
  "1970-force-early-return-long",
  "1971-multi-force-early-return",
  "1972-jni-id-swap-indices",
  "1973-jni-id-swap-pointer",
  "1974-resize-array",
  "1975-hello-structural-transformation",
  "1976-hello-structural-static-methods",
  "1977-hello-structural-obsolescence",
  "1978-regular-obsolete-then-structural-obsolescence",
  "1979-threaded-structural-transformation",
  "1980-obsolete-object-cleared",
  "1982-no-virtuals-structural-redefinition",
  "1984-structural-redefine-field-trace",
  "1985-structural-redefine-stack-scope",
  "1986-structural-redefine-multi-thread-stack-scope",
  "1987-structural-redefine-recursive-stack-scope",
  "1988-multi-structural-redefine",
  "1989-transform-bad-monitor",
  "1990-structural-bad-verify",
  "1991-hello-structural-retransform",
  "1992-retransform-no-such-field",
  "1993-fallback-non-structural",
  "1994-final-virtual-structural",
  "1995-final-virtual-structural-multithread",
  "1996-final-override-virtual-structural",
  "1997-structural-shadow-method",
  "1998-structural-shadow-field",
  "1999-virtual-structural",
  "2003-double-virtual-structural",
  "2004-double-virtual-structural-abstract",
  "2005-pause-all-redefine-multithreaded",
  "2008-redefine-then-old-reflect-field",
  "2011-stack-walk-concurrent-instrument",
  "203-multi-checkpoint",
  "2031-zygote-compiled-frame-deopt",
  "2033-shutdown-mechanics",
  "2036-jni-filechannel",
  "2037-thread-name-inherit",
  "305-other-fault-handler",
  "449-checker-bce",
  "454-get-vreg",
  "461-get-reference-vreg",
  "466-get-live-vreg",
  "497-inlining-and-class-loader",
  "530-regression-lse",
  "555-UnsafeGetLong-regression",
  "566-polymorphic-inlining",
  "595-profile-saving",
  "597-deopt-busy-loop",
  "597-deopt-invoke-stub",
  "597-deopt-new-string",
  "602-deoptimizeable",
  "604-hot-static-interface",
  "616-cha-abstract",
  "616-cha-interface",
  "616-cha-miranda",
  "616-cha-native",
  "616-cha-regression-proxy-method",
  "616-cha",
  "623-checker-loop-regressions",
  "626-set-resolved-string",
  "629-vdex-speed",
  "638-checker-inline-cache-intrinsic",
  "642-fp-callees",
  "647-jni-get-field-id",
  "652-deopt-intrinsic",
  "655-jit-clinit",
  "656-loop-deopt",
  "660-clinit",
  "661-oat-writer-layout",
  "664-aget-verifier",
  "667-jit-jni-stub",
  "674-hotness-compiled",
  "679-locks",
  "680-checker-deopt-dex-pc-0",
  "685-deoptimizeable",
  "687-deopt",
  "689-zygote-jit-deopt",
  "693-vdex-inmem-loader-evict",
  "707-checker-invalid-profile",
  "708-jit-cache-churn",
  "717-integer-value-of",
  "720-thread-priority",
  "728-imt-conflict-zygote",  # Custom `run` script + dependency on `libarttest`.
  "813-fp-args",              # Dependency on `libarttest`.
  "900-hello-plugin",
  "901-hello-ti-agent",
  "902-hello-transformation",
  "903-hello-tagging",
  "904-object-allocation",
  "905-object-free",
  "906-iterate-heap",
  "907-get-loaded-classes",
  "908-gc-start-finish",
  "910-methods",
  "911-get-stack-trace",
  "913-heaps",
  "914-hello-obsolescence",
  "915-obsolete-2",
  "916-obsolete-jit",
  "917-fields-transformation",
  "918-fields",
  "919-obsolete-fields",
  "920-objects",
  "921-hello-failure",
  "922-properties",
  "923-monitors",
  "924-threads",
  "925-threadgroups",
  "926-multi-obsolescence",
  "927-timers",
  "928-jni-table",
  "930-hello-retransform",
  "931-agent-thread",
  "932-transform-saves",
  "933-misc-events",
  "937-hello-retransform-package",
  "939-hello-transformation-bcp",
  "940-recursive-obsolete",
  "941-recursive-obsolete-jit",
  "942-private-recursive",
  "943-private-recursive-jit",
  "944-transform-classloaders",
  "945-obsolete-native",
  "946-obsolete-throw",
  "947-reflect-method",
  "949-in-memory-transform",
  "950-redefine-intrinsic",
  "951-threaded-obsolete",
  "982-ok-no-retransform",
  "983-source-transform-verify",
  "984-obsolete-invoke",
  "985-re-obsolete",
  "986-native-method-bind",
  "987-agent-bind",
  "988-method-trace",
  "989-method-trace-throw",
  "990-field-trace",
  "991-field-trace-2",
  "992-source-data",
  "993-breakpoints",
  "994-breakpoint-line",
  "995-breakpoints-throw",
  "996-breakpoint-obsolete",
  "997-single-step",
])

# Percentage of ART run-tests (among the ones expected to succeed) to include in
# the `presubmit` test group in `TEST_MAPPING` file -- the rest will be included
# in `postsubmit` test group.
# This value has to be a number between 0 and 100.
presubmit_tests_percentage = 25


# Is `run_test` a Checker test (i.e. a test containing Checker
# assertions)?
def is_checker_test(run_test):
  return re.match("^[0-9]+-checker-", run_test)

# Is `run_test` expected to succeed?
#
# Also temporarily consider Checker tests as known failing tests, as they
# currently break some test runs (see b/169852871).
# TODO(b/162408889): Complete Checker integration and re-include Checker
# tests in test mapping.
def is_expected_succeeding(run_test):
  return run_test not in known_failing_tests and not is_checker_test(run_test)


class Generator:
  def __init__(self, art_dir):
    self.art_dir = art_dir
    self.art_test_dir = os.path.join(art_dir, TESTS_DIR)

  def enumerate_run_tests(self):
    return sorted(
        [run_test for \
         run_test in os.listdir(self.art_test_dir) if re.match("^[0-9]{3,}-", run_test)])

  # Is building `run_test` supported?
  # TODO(b/147814778): Add build support for more tests.
  def is_buildable(self, run_test):
    run_test_path = os.path.join(self.art_test_dir, run_test)

    # Ignore tests with non-default build rules.
    if os.path.isfile(os.path.join(run_test_path, "build")):
      return False
    # Ignore tests with no `src` directory.
    if not os.path.isdir(os.path.join(run_test_path, "src")):
      return False
    # Ignore tests with sources outside the `src` directory.
    for subdir in ["jasmin",
                   "jasmin-multidex",
                   "smali",
                   "smali-ex",
                   "smali-multidex",
                   "src-art",
                   "src-dex2oat-unresolved",
                   "src-ex",
                   "src-ex2",
                   "src-multidex",
                   "src2"]:
      if os.path.isdir(os.path.join(run_test_path, subdir)):
        return False
    # Ignore test with a copy of `sun.misc.Unsafe`.
    if os.path.isfile(os.path.join(run_test_path, "src", "sun", "misc", "Unsafe.java")):
      return False
    # Ignore tests with Hidden API specs.
    if os.path.isfile(os.path.join(run_test_path, "hiddenapi-flags.csv")):
      return False
    # All other tests are considered buildable.
    return True

  def regen_bp_files(self, run_tests, buildable_tests):
    for run_test in run_tests:
      # Remove any previously generated file.
      bp_file = os.path.join(self.art_test_dir, run_test, "Android.bp")
      if os.path.exists(bp_file):
        os.remove(bp_file)

    for run_test in buildable_tests:
      self.regen_bp_file(run_test)

  def regen_bp_file(self, run_test):
    """Regenerate Blueprint file for an ART run-test."""

    bp_file = os.path.join(self.art_test_dir, run_test, "Android.bp")

    run_test_module_name = "art-run-test-" + run_test

    if is_expected_succeeding(run_test):
      test_config_template = "art-run-test-target-template"
    else:
      test_config_template = "art-run-test-target-no-test-suite-tag-template"

    if is_checker_test(run_test):
      include_src = """\

          // Include the Java source files in the test's artifacts, to make Checker assertions
          // available to the TradeFed test runner.
          include_srcs: true,"""
    else:
      include_src = ""
    with open(bp_file, "w") as f:
      f.write(reindent(f"""\
      // Generated by `{me}`. Do not edit manually.

      // Build rules for ART run-test `{run_test}`.

      // Test's Dex code.
      java_test {{
          name: "{run_test_module_name}",
          defaults: ["art-run-test-defaults"],
          test_config_template: ":{test_config_template}",
          srcs: ["src/**/*.java"],
          data: [
              ":{run_test_module_name}-expected-stdout",
              ":{run_test_module_name}-expected-stderr",
          ],{include_src}
      }}

      // Test's expected standard output.
      genrule {{
          name: "{run_test_module_name}-expected-stdout",
          out: ["{run_test_module_name}-expected-stdout.txt"],
          srcs: ["expected-stdout.txt"],
          cmd: "cp -f $(in) $(out)",
      }}

      // Test's expected standard error.
      genrule {{
          name: "{run_test_module_name}-expected-stderr",
          out: ["{run_test_module_name}-expected-stderr.txt"],
          srcs: ["expected-stderr.txt"],
          cmd: "cp -f $(in) $(out)",
      }}
      """))

  def regen_test_mapping_file(self, art_run_tests, num_presubmit_run_tests):
    """Regenerate ART's `TEST_MAPPING`."""

    run_test_module_names = list(map(lambda t: "art-run-test-" + t, art_run_tests))

    # Mainline presubmits.
    # TODO(rpl): Progressively add more tests to this test group.
    mainline_presubmit_tests = [
        "art-run-test-001-HelloWorld[com.google.android.art.apex]",
    ]
    mainline_presubmit_tests_dict = [{"name": t} for t in mainline_presubmit_tests]

    # Presubmits.
    other_presubmit_tests = [
        "CtsJdwpTestCases",
        "BootImageProfileTest",
        "ArtServiceTests",
    ]
    art_gtests = [
        "ArtGtestsTarget",
    ]
    presubmit_run_tests = run_test_module_names[0:num_presubmit_run_tests]
    presubmit_tests = other_presubmit_tests + art_gtests + presubmit_run_tests
    presubmit_tests_dict = [{"name": t} for t in presubmit_tests]

    # Postsubmits.
    postsubmit_run_tests = run_test_module_names[num_presubmit_run_tests:]
    postsubmit_tests_dict = [{"name": t} for t in postsubmit_run_tests]

    # Use an `OrderedDict` container to preserve the order in which items are inserted.
    test_mapping_dict = collections.OrderedDict([
        ("mainline-presubmit", mainline_presubmit_tests_dict),
        ("presubmit", presubmit_tests_dict),
        ("postsubmit", postsubmit_tests_dict),
    ])
    test_mapping_contents = json.dumps(test_mapping_dict, indent = INDENT)

    test_mapping_file = os.path.join(self.art_dir, "TEST_MAPPING")
    with open(test_mapping_file, "w") as f:
      f.write(f"// Generated by `{me}`. Do not edit manually.\n")
      f.write(test_mapping_contents)
      f.write("\n")

  def regen_test_files(self):
    run_tests = self.enumerate_run_tests()

    # Create a list of the tests that can currently be built, and for
    # which a Blueprint file is to be generated.
    buildable_tests = list(filter(self.is_buildable, run_tests))

    # Create a list of the tests that can be built and are expected to
    # succeed. These tests are to be added to ART's `TEST_MAPPING`
    # file and also tagged as part of TradeFed's `art-target-run-test`
    # test suite via the `test-suite-tag` option in their
    # configuration file.
    expected_succeeding_tests = list(filter(is_expected_succeeding, buildable_tests))

    # Regenerate Blueprint files.
    # ---------------------------

    self.regen_bp_files(run_tests, buildable_tests)

    buildable_tests_percentage = int(len(buildable_tests) * 100 / len(run_tests))

    print(f"Generated Blueprint files for {len(buildable_tests)} ART run-tests out of"
          f" {len(run_tests)} ({buildable_tests_percentage}%).")

    # Regenerate `TEST_MAPPING` file.
    # -------------------------------

    # Note: We only include ART run-tests expected to succeed for now.

    # Note: We only include a (growing) fraction of the supported ART
    # run-tests (see `presubmit_tests_percentage`) into the
    # `presubmit` test group (the other ART run-tests are added to the
    # `postsubmit` test group), as we initially had issues with
    # Android presubmits when the whole set of supported ART run-tests
    # was included in one go (b/169310621). This progressive rollout
    # allows us to better monitor future potential presubmit failures.
    num_presubmit_run_tests = int(len(expected_succeeding_tests) * presubmit_tests_percentage / 100)
    self.regen_test_mapping_file(expected_succeeding_tests, num_presubmit_run_tests)

    expected_succeeding_tests_percentage = int(len(expected_succeeding_tests) * 100 /
                                               len(run_tests))

    num_postsubmit_tests = len(expected_succeeding_tests) - num_presubmit_run_tests
    postsubmit_tests_percentage = 100 - presubmit_tests_percentage

    print(f"Generated TEST_MAPPING entries for {len(expected_succeeding_tests)} ART run-tests out"
          f" of {len(run_tests)} ({expected_succeeding_tests_percentage}%):")
    print(f"  {num_presubmit_run_tests} tests ({presubmit_tests_percentage}%) in `presubmit` test"
          f" group;")
    print(f"  {num_postsubmit_tests} tests ({postsubmit_tests_percentage}%) in `postsubmit` test"
          f" group.")


def main():
  if "ANDROID_BUILD_TOP" not in os.environ:
    logging.error("ANDROID_BUILD_TOP environment variable is empty; did you forget to run `lunch`?")
    sys.exit(1)

  generator = Generator(os.path.join(os.environ["ANDROID_BUILD_TOP"], "art"))
  generator.regen_test_files()


if __name__ == '__main__':
  main()