scripts/update_crate_tests.py - platform/development.git - Git at Google

 #!/usr/bin/env python3
 #
 # Copyright (C) 2020 The Android Open Source Project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #      http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Add or update tests to TEST_MAPPING.

 This script uses Bazel to find reverse dependencies on a crate and generates a
 TEST_MAPPING file. It accepts the absolute path to a crate as argument. If no
 argument is provided, it assumes the crate is the current directory.

   Usage:
   $ . build/envsetup.sh
   $ lunch aosp_arm64-eng
   $ update_crate_tests.py $ANDROID_BUILD_TOP/external/rust/crates/libc

 This script is automatically called by external_updater.
 """

 import json
 import os
 import platform
 import subprocess
 import sys

 # Some tests requires specific options. Consider fixing the upstream crate
 # before updating this dictionary.
 TEST_OPTIONS = {
     "ring_device_test_tests_digest_tests": [{"test-timeout": "600000"}],
     "ring_device_test_src_lib": [{"test-timeout": "100000"}],
 }

 # Excluded tests. These tests will be ignored by this script.
 TEST_EXCLUDE = [
         "aidl_test_rust_client",
         "aidl_test_rust_service"
 ]

 # Excluded modules.
 EXCLUDE_PATHS = [
         "//external/adhd",
         "//external/crosvm",
         "//external/libchromeos-rs",
         "//external/vm_tools"
 ]


 class UpdaterException(Exception):
     """Exception generated by this script."""


 class Env(object):
     """Env captures the execution environment.

     It ensures this script is executed within an AOSP repository.

     Attributes:
       ANDROID_BUILD_TOP: A string representing the absolute path to the top
         of the repository.
     """
     def __init__(self):
         try:
             self.ANDROID_BUILD_TOP = os.environ['ANDROID_BUILD_TOP']
         except KeyError:
             raise UpdaterException('$ANDROID_BUILD_TOP is not defined; you '
                                    'must first source build/envsetup.sh and '
                                    'select a target.')


 class Bazel(object):
     """Bazel wrapper.

     The wrapper is used to call bazel queryview and generate the list of
     reverse dependencies.

     Attributes:
       path: The path to the bazel executable.
     """
     def __init__(self, env):
         """Constructor.

         Note that the current directory is changed to ANDROID_BUILD_TOP.

         Args:
           env: An instance of Env.

         Raises:
           UpdaterException: an error occurred while calling soong_ui.
         """
         if platform.system() != 'Linux':
             raise UpdaterException('This script has only been tested on Linux.')
         self.path = os.path.join(env.ANDROID_BUILD_TOP, "tools", "bazel")
         soong_ui = os.path.join(env.ANDROID_BUILD_TOP, "build", "soong", "soong_ui.bash")

         # soong_ui requires to be at the root of the repository.
         os.chdir(env.ANDROID_BUILD_TOP)
         print("Generating Bazel files...")
         cmd = [soong_ui, "--make-mode", "GENERATE_BAZEL_FILES=1", "nothing"]
         try:
             subprocess.check_output(cmd, stderr=subprocess.STDOUT, text=True)
         except subprocess.CalledProcessError as e:
             raise UpdaterException('Unable to generate bazel workspace: ' + e.output)

         print("Building Bazel Queryview. This can take a couple of minutes...")
         cmd = [soong_ui, "--build-mode", "--all-modules", "--dir=.", "queryview"]
         try:
             subprocess.check_output(cmd, stderr=subprocess.STDOUT, text=True)
         except subprocess.CalledProcessError as e:
             raise UpdaterException('Unable to update TEST_MAPPING: ' + e.output)

     def query_modules(self, path):
         """Returns all modules for a given path."""
         cmd = self.path + " query --config=queryview /" + path + ":all"
         out = subprocess.check_output(cmd, shell=True, stderr=subprocess.DEVNULL, text=True).strip().split("\n")
         modules = set()
         for line in out:
             # speed up by excluding unused modules.
             if "windows_x86" in line:
                 continue
             modules.add(line)
         return modules

     def query_rdeps(self, module):
         """Returns all reverse dependencies for a single module."""
         cmd = (self.path + " query --config=queryview \'rdeps(//..., " +
                 module + ")\' --output=label_kind")
         out = (subprocess.check_output(cmd, shell=True, stderr=subprocess.DEVNULL, text=True)
                 .strip().split("\n"))
         if '' in out:
             out.remove('')
         return out

     def exclude_module(self, module):
         for path in EXCLUDE_PATHS:
             if module.startswith(path):
                 return True
         return False

     def query_rdep_tests(self, modules):
         """Returns all reverse dependency tests for modules in this package."""
         rdep_tests = set()
         for module in modules:
             for rdep in self.query_rdeps(module):
                 rule_type, _, mod = rdep.split(" ")
                 if rule_type == "rust_test_" or rule_type == "rust_test":
                     if self.exclude_module(mod) == False:
                         rdep_tests.add(mod.split(":")[1].split("--")[0])
         return rdep_tests


 class Package(object):
     """A Bazel package.

     Attributes:
       dir: The absolute path to this package.
       dir_rel: The relative path to this package.
       rdep_tests: The list of computed reverse dependencies.
     """
     def __init__(self, path, env, bazel):
         """Constructor.

         Note that the current directory is changed to the package location when
         called.

         Args:
           path: Path to the package.
           env: An instance of Env.
           bazel: An instance of Bazel.

         Raises:
           UpdaterException: the package does not appear to belong to the
             current repository.
         """
         if path == None:
             self.dir = os.getcwd()
         else:
             self.dir = path
         try:
             self.dir_rel = self.dir.split(env.ANDROID_BUILD_TOP)[1]
         except IndexError:
             raise UpdaterException('The path ' + self.dir + ' is not under ' +
                             env.ANDROID_BUILD_TOP + '; You must be in the '
                             'directory of a crate or pass its absolute path '
                             'as first argument.')

         # Move to the package_directory.
         os.chdir(self.dir)
         modules = bazel.query_modules(self.dir_rel)
         self.rdep_tests = bazel.query_rdep_tests(modules)

     def get_rdep_tests(self):
         return self.rdep_tests


 class TestMapping(object):
     """A TEST_MAPPING file.

     Attributes:
       package: The package associated with this TEST_MAPPING file.
     """
     def __init__(self, path):
         """Constructor.

         Args:
           path: The absolute path to the package.
         """
         env = Env()
         bazel = Bazel(env)
         self.package = Package(path, env, bazel)

     def create(self):
         """Generates the TEST_MAPPING file."""
         tests = self.package.get_rdep_tests()
         if not bool(tests):
             return
         test_mapping = self.tests_to_mapping(tests)
         self.write_test_mapping(test_mapping)

     def tests_to_mapping(self, tests):
         """Translate the test list into a dictionary."""
         test_mapping = {"presubmit": []}
         for test in tests:
             if test in TEST_EXCLUDE:
                 continue
             if test in TEST_OPTIONS:
                 test_mapping["presubmit"].append({"name": test, "options": TEST_OPTIONS[test]})
             else:
                 test_mapping["presubmit"].append({"name": test})
         test_mapping["presubmit"] = sorted(test_mapping["presubmit"], key=lambda t: t["name"])
         return test_mapping

     def write_test_mapping(self, test_mapping):
         """Writes the TEST_MAPPING file."""
         with open("TEST_MAPPING", "w") as json_file:
             json_file.write("// Generated by update_crate_tests.py for tests that depend on this crate.\n")
             json.dump(test_mapping, json_file, indent=2, separators=(',', ': '), sort_keys=True)
             json_file.write("\n")
         print("TEST_MAPPING successfully updated!")


 def main():
     if len(sys.argv) == 2:
         path = sys.argv[1]
     else:
         path = None
     try:
         test_mapping = TestMapping(path)
     except UpdaterException as err:
         sys.exit("Error: " + str(err))
     test_mapping.create()

 if __name__ == '__main__':
   main()
	#!/usr/bin/env python3
	#
	# Copyright (C) 2020 The Android Open Source Project
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.
	"""Add or update tests to TEST_MAPPING.

	This script uses Bazel to find reverse dependencies on a crate and generates a
	TEST_MAPPING file. It accepts the absolute path to a crate as argument. If no
	argument is provided, it assumes the crate is the current directory.

	Usage:
	$ . build/envsetup.sh
	$ lunch aosp_arm64-eng
	$ update_crate_tests.py $ANDROID_BUILD_TOP/external/rust/crates/libc

	This script is automatically called by external_updater.
	"""

	import json
	import os
	import platform
	import subprocess
	import sys

	# Some tests requires specific options. Consider fixing the upstream crate
	# before updating this dictionary.
	TEST_OPTIONS = {
	"ring_device_test_tests_digest_tests": [{"test-timeout": "600000"}],
	"ring_device_test_src_lib": [{"test-timeout": "100000"}],
	}

	# Excluded tests. These tests will be ignored by this script.
	TEST_EXCLUDE = [
	"aidl_test_rust_client",
	"aidl_test_rust_service"
	]

	# Excluded modules.
	EXCLUDE_PATHS = [
	"//external/adhd",
	"//external/crosvm",
	"//external/libchromeos-rs",
	"//external/vm_tools"
	]


	class UpdaterException(Exception):
	"""Exception generated by this script."""


	class Env(object):
	"""Env captures the execution environment.

	It ensures this script is executed within an AOSP repository.

	Attributes:
	ANDROID_BUILD_TOP: A string representing the absolute path to the top
	of the repository.
	"""
	def __init__(self):
	try:
	self.ANDROID_BUILD_TOP = os.environ['ANDROID_BUILD_TOP']
	except KeyError:
	raise UpdaterException('$ANDROID_BUILD_TOP is not defined; you '
	'must first source build/envsetup.sh and '
	'select a target.')


	class Bazel(object):
	"""Bazel wrapper.

	The wrapper is used to call bazel queryview and generate the list of
	reverse dependencies.

	Attributes:
	path: The path to the bazel executable.
	"""
	def __init__(self, env):
	"""Constructor.

	Note that the current directory is changed to ANDROID_BUILD_TOP.

	Args:
	env: An instance of Env.

	Raises:
	UpdaterException: an error occurred while calling soong_ui.
	"""
	if platform.system() != 'Linux':
	raise UpdaterException('This script has only been tested on Linux.')
	self.path = os.path.join(env.ANDROID_BUILD_TOP, "tools", "bazel")
	soong_ui = os.path.join(env.ANDROID_BUILD_TOP, "build", "soong", "soong_ui.bash")

	# soong_ui requires to be at the root of the repository.
	os.chdir(env.ANDROID_BUILD_TOP)
	print("Generating Bazel files...")
	cmd = [soong_ui, "--make-mode", "GENERATE_BAZEL_FILES=1", "nothing"]
	try:
	subprocess.check_output(cmd, stderr=subprocess.STDOUT, text=True)
	except subprocess.CalledProcessError as e:
	raise UpdaterException('Unable to generate bazel workspace: ' + e.output)

	print("Building Bazel Queryview. This can take a couple of minutes...")
	cmd = [soong_ui, "--build-mode", "--all-modules", "--dir=.", "queryview"]
	try:
	subprocess.check_output(cmd, stderr=subprocess.STDOUT, text=True)
	except subprocess.CalledProcessError as e:
	raise UpdaterException('Unable to update TEST_MAPPING: ' + e.output)

	def query_modules(self, path):
	"""Returns all modules for a given path."""
	cmd = self.path + " query --config=queryview /" + path + ":all"
	out = subprocess.check_output(cmd, shell=True, stderr=subprocess.DEVNULL, text=True).strip().split("\n")
	modules = set()
	for line in out:
	# speed up by excluding unused modules.
	if "windows_x86" in line:
	continue
	modules.add(line)
	return modules

	def query_rdeps(self, module):
	"""Returns all reverse dependencies for a single module."""
	cmd = (self.path + " query --config=queryview \'rdeps(//..., " +
	module + ")\' --output=label_kind")
	out = (subprocess.check_output(cmd, shell=True, stderr=subprocess.DEVNULL, text=True)
	.strip().split("\n"))
	if '' in out:
	out.remove('')
	return out

	def exclude_module(self, module):
	for path in EXCLUDE_PATHS:
	if module.startswith(path):
	return True
	return False

	def query_rdep_tests(self, modules):
	"""Returns all reverse dependency tests for modules in this package."""
	rdep_tests = set()
	for module in modules:
	for rdep in self.query_rdeps(module):
	rule_type, _, mod = rdep.split(" ")
	if rule_type == "rust_test_" or rule_type == "rust_test":
	if self.exclude_module(mod) == False:
	rdep_tests.add(mod.split(":")[1].split("--")[0])
	return rdep_tests


	class Package(object):
	"""A Bazel package.

	Attributes:
	dir: The absolute path to this package.
	dir_rel: The relative path to this package.
	rdep_tests: The list of computed reverse dependencies.
	"""
	def __init__(self, path, env, bazel):
	"""Constructor.

	Note that the current directory is changed to the package location when
	called.

	Args:
	path: Path to the package.
	env: An instance of Env.
	bazel: An instance of Bazel.

	Raises:
	UpdaterException: the package does not appear to belong to the
	current repository.
	"""
	if path == None:
	self.dir = os.getcwd()
	else:
	self.dir = path
	try:
	self.dir_rel = self.dir.split(env.ANDROID_BUILD_TOP)[1]
	except IndexError:
	raise UpdaterException('The path ' + self.dir + ' is not under ' +
	env.ANDROID_BUILD_TOP + '; You must be in the '
	'directory of a crate or pass its absolute path '
	'as first argument.')

	# Move to the package_directory.
	os.chdir(self.dir)
	modules = bazel.query_modules(self.dir_rel)
	self.rdep_tests = bazel.query_rdep_tests(modules)

	def get_rdep_tests(self):
	return self.rdep_tests


	class TestMapping(object):
	"""A TEST_MAPPING file.

	Attributes:
	package: The package associated with this TEST_MAPPING file.
	"""
	def __init__(self, path):
	"""Constructor.

	Args:
	path: The absolute path to the package.
	"""
	env = Env()
	bazel = Bazel(env)
	self.package = Package(path, env, bazel)

	def create(self):
	"""Generates the TEST_MAPPING file."""
	tests = self.package.get_rdep_tests()
	if not bool(tests):
	return
	test_mapping = self.tests_to_mapping(tests)
	self.write_test_mapping(test_mapping)

	def tests_to_mapping(self, tests):
	"""Translate the test list into a dictionary."""
	test_mapping = {"presubmit": []}
	for test in tests:
	if test in TEST_EXCLUDE:
	continue
	if test in TEST_OPTIONS:
	test_mapping["presubmit"].append({"name": test, "options": TEST_OPTIONS[test]})
	else:
	test_mapping["presubmit"].append({"name": test})
	test_mapping["presubmit"] = sorted(test_mapping["presubmit"], key=lambda t: t["name"])
	return test_mapping

	def write_test_mapping(self, test_mapping):
	"""Writes the TEST_MAPPING file."""
	with open("TEST_MAPPING", "w") as json_file:
	json_file.write("// Generated by update_crate_tests.py for tests that depend on this crate.\n")
	json.dump(test_mapping, json_file, indent=2, separators=(',', ': '), sort_keys=True)
	json_file.write("\n")
	print("TEST_MAPPING successfully updated!")


	def main():
	if len(sys.argv) == 2:
	path = sys.argv[1]
	else:
	path = None
	try:
	test_mapping = TestMapping(path)
	except UpdaterException as err:
	sys.exit("Error: " + str(err))
	test_mapping.create()

	if __name__ == '__main__':
	main()