Create ModuleInfo by Loader
Bug: 281547254
Bug: 263199608
Test: m clean && m atest && atest-dev atest_unittests
Test: m clean && m atest && atest-dev HelloWorldTests
Test: m clean && m atest && atest-dev cts/tests/tests/media/misc/src/android/media/misc/cts/ResourceManagerTest.java
Test: m clean && m atest && atest-dev cts/tests/tests/gesture
Test: m clean && m atest && atest-dev --list-modules cts
Change-Id: Iee200ae235f99cd64f90f5b6a7643809d4ef53ec
diff --git a/atest/atest_main.py b/atest/atest_main.py
index 2097980..6f6d146 100755
--- a/atest/atest_main.py
+++ b/atest/atest_main.py
@@ -1058,15 +1058,8 @@
proc_idx = atest_utils.run_multi_proc(at.index_targets)
smart_rebuild = need_rebuild_module_info(args)
- mod_start = time.time()
- mod_info = module_info.load_from_file(force_build=smart_rebuild)
- mod_stop = time.time() - mod_start
- metrics.LocalDetectEvent(detect_type=DetectType.MODULE_INFO_INIT_MS,
- result=int(mod_stop * 1000))
- atest_utils.run_multi_proc(func=mod_info._save_module_info_timestamp)
- atest_utils.run_multi_proc(
- func=atest_utils.save_build_files_timestamp,
- )
+ mod_info = module_info.load_from_file(
+ force_build=smart_rebuild, save_timestamps=True)
translator = cli_translator.CLITranslator(
mod_info=mod_info,
diff --git a/atest/module_info.py b/atest/module_info.py
index a364efe..308fa6b 100644
--- a/atest/module_info.py
+++ b/atest/module_info.py
@@ -29,7 +29,7 @@
import time
from pathlib import Path
-from typing import Any, Dict, List, Set
+from typing import Any, Callable, Dict, List, Set
from atest import atest_utils
from atest import constants
@@ -55,33 +55,41 @@
def load_from_file(
module_file: Path = None,
force_build: bool = False,
+ save_timestamps: bool = False,
) -> ModuleInfo:
"""Factory method that initializes ModuleInfo from the build-generated
JSON file"""
- return ModuleInfo(module_file=module_file, force_build=force_build)
-
-
-def load_from_dict(name_to_module_info: Dict[str, Any]) -> ModuleInfo:
- """Factory method that initializes ModuleInfo from a dictionary."""
- with tempfile.NamedTemporaryFile(mode='w') as f:
- # TODO: Serialize the input dict to JSON.
- json.dump({}, f)
- mi = load_from_file(module_file=f.name)
-
- mi.loader.name_to_module_info.update(name_to_module_info)
+ mod_start = time.time()
+ loader = Loader(
+ module_file=module_file, force_build=force_build)
+ mi = loader.load(save_timestamps=save_timestamps)
+ mod_stop = time.time() - mod_start
+ metrics.LocalDetectEvent(
+ detect_type=DetectType.MODULE_INFO_INIT_MS, result=int(mod_stop * 1000))
return mi
+def load_from_dict(name_to_module_info: Dict[str, Any]) -> ModuleInfo:
+ """Factory method that initializes ModuleInfo from a dictionary."""
+ path_to_module_info = get_path_to_module_info(name_to_module_info)
+ return ModuleInfo(
+ name_to_module_info=name_to_module_info,
+ path_to_module_info=path_to_module_info,
+ get_testable_modules=lambda s: _get_testable_modules(
+ name_to_module_info, path_to_module_info, s),
+ )
+
+
def create_empty() -> ModuleInfo:
"""Factory method that initializes an empty ModuleInfo."""
- return ModuleInfo(no_generate=True)
+ return ModuleInfo()
class Loader:
"""Class that handles load and merge processes."""
- def __init__(self, module_file=None, force_build: bool=False, no_generate: bool=False):
+ def __init__(self, module_file: Path=None, force_build: bool=False):
self.java_dep_path = Path(
atest_utils.get_build_out_dir()).joinpath('soong', _JAVA_DEP_INFO)
self.cc_dep_path = Path(
@@ -99,12 +107,7 @@
# module_info_target stays None.
self.module_info_target = None
- if no_generate:
- self.name_to_module_info = {}
- return
-
- self.name_to_module_info, self.path_to_module_info = self.load(
- module_file)
+ self.name_to_module_info, self.path_to_module_info = self._load_module_info_file()
# Index and checksum files that will be used.
index_dir = Path(os.getenv(constants.ANDROID_HOST_OUT, '')).joinpath('indexes')
@@ -119,11 +122,21 @@
func=self._get_testable_modules,
kwargs={'index': True})
- def load(self, module_file):
- """Load the module file.
+ def load(self, save_timestamps: bool=False):
+ if save_timestamps:
+ atest_utils.run_multi_proc(func=self._save_module_info_timestamp)
+ atest_utils.run_multi_proc(func=atest_utils.save_build_files_timestamp)
- No matter whether passing module_file or not, ModuleInfo will load
- atest_merged_dep.json as module info eventually.
+ return ModuleInfo(
+ name_to_module_info=self.name_to_module_info,
+ path_to_module_info=self.path_to_module_info,
+ module_info_target=self.module_info_target,
+ mod_info_file_path=self.mod_info_file_path,
+ get_testable_modules=self.get_testable_modules,
+ )
+
+ def _load_module_info_file(self):
+ """Load the module file as ModuleInfo.
+--------------+ +----------------------------------+
| ModuleInfo() | | ModuleInfo(module_file=foo.json) |
@@ -144,15 +157,10 @@
| /atest_merged_dep.json |--> load as module info.
+============================+
- Args:
- module_file: String of path to file to load up. Used for testing.
- Note: if set, ModuleInfo will skip build process.
-
Returns:
Dict of module name to module info and dict of module path to module info.
"""
- file_path = module_file
- if not file_path:
+ if not self.mod_info_file_path:
self.module_info_target, file_path = _discover_mod_file_and_target(
self.force_build)
self.mod_info_file_path = Path(file_path)
@@ -344,29 +352,10 @@
Returns:
Set of all testable modules.
"""
- modules = set()
- begin = time.time()
- for _, info in self.name_to_module_info.items():
- if _is_testable_module(
- self.name_to_module_info, self.path_to_module_info, info):
-
- testable_module = info.get(constants.MODULE_NAME)
- if testable_module:
- modules.add(testable_module)
-
- logging.debug('Probing all testable modules took %ss',
- time.time() - begin)
+ modules = _get_testable_modules(
+ self.name_to_module_info, self.path_to_module_info, suite)
if index:
self._index_testable_modules(modules)
- if suite:
- _modules = set()
- for module_name in modules:
- info = self.name_to_module_info.get(module_name)
- if ModuleInfo.is_suite_in_compatibility_suites(suite, info):
- testable_module = info.get(constants.MODULE_NAME)
- if testable_module:
- _modules.add(testable_module)
- return _modules
return modules
def _index_testable_modules(self, content):
@@ -392,9 +381,12 @@
def __init__(
self,
- force_build=False,
- module_file=None,
- no_generate=False):
+ name_to_module_info: Dict[str, Any]=None,
+ path_to_module_info: Dict[str, Any]=None,
+ module_info_target: str=None,
+ mod_info_file_path: Path=None,
+ get_testable_modules: Callable=None,
+ ):
"""Initialize the ModuleInfo object.
Load up the module-info.json file and initialize the helper vars.
@@ -424,57 +416,20 @@
+============================+
Args:
- force_build: Boolean to indicate if we should rebuild the
- module_info file regardless if it's created or not.
- module_file: String of path to file to load up. Used for testing.
- no_generate: Boolean to indicate if we should populate module info
- from the soong artifacts; setting to true will
- leave module info empty.
+ name_to_module_info: Dict of name to module info.
+ path_to_module_info: Dict of path to module info.
+ module_info_target: Target name of module-info.json.
+ mod_info_file_path: Path of module-info.json.
+ get_testable_modules: Function to get all testable modules.
"""
self.root_dir = os.environ.get(constants.ANDROID_BUILD_TOP)
self.roboleaf_tests = {}
- self.loader = Loader(
- module_file=module_file,
- force_build=force_build,
- no_generate=no_generate,
- )
-
- @property
- def name_to_module_info(self):
- return self.loader.name_to_module_info
-
- @property
- def path_to_module_info(self):
- return self.loader.path_to_module_info
-
- @property
- def module_info_target(self):
- return self.loader.module_info_target
-
- @property
- def mod_info_file_path(self):
- return self.loader.mod_info_file_path
-
- @mod_info_file_path.setter
- def mod_info_file_path(self, value):
- self.loader.mod_info_file_path = value
-
- @property
- def java_dep_path(self):
- return self.loader.java_dep_path
-
- @property
- def cc_dep_path(self):
- return self.loader.cc_dep_path
-
- @property
- def merged_dep_path(self):
- return self.loader.merged_dep_path
-
- def _save_module_info_timestamp(self):
- """Caller of the same method in Loader class."""
- return self.loader._save_module_info_timestamp()
+ self.name_to_module_info = name_to_module_info or {}
+ self.path_to_module_info = path_to_module_info or {}
+ self.module_info_target = module_info_target
+ self.mod_info_file_path = mod_info_file_path
+ self._get_testable_modules = get_testable_modules
def is_module(self, name):
"""Return True if name is a module, False otherwise."""
@@ -527,7 +482,7 @@
constants.MODULE_COMPATIBILITY_SUITES, [])
def get_testable_modules(self, suite=None):
- return self.loader.get_testable_modules(suite)
+ return self._get_testable_modules(suite)
@staticmethod
def is_tradefed_testable_module(info: Dict[str, Any]) -> bool:
@@ -844,15 +799,6 @@
return True
return False
- def _merge_build_system_infos(self, name_to_module_info,
- java_bp_info_path=None, cc_bp_info_path=None):
- """Caller of the same method in Loader class."""
- return self.loader._merge_build_system_infos(
- name_to_module_info,
- java_bp_info_path,
- cc_bp_info_path,
- )
-
def get_filepath_from_module(self, module_name: str, filename: str) -> Path:
"""Return absolute path of the given module and filename."""
mod_path = self.get_paths(module_name)
@@ -1263,3 +1209,30 @@
if _is_legacy_robolectric_test(name_to_module_info, path_to_module_info, info):
return True
return False
+
+def _get_testable_modules(
+ name_to_module_info: Dict[str, Dict],
+ path_to_module_info: Dict[str, Dict],
+ suite: bool=None):
+
+ modules = set()
+ begin = time.time()
+ for _, info in name_to_module_info.items():
+ if _is_testable_module(name_to_module_info, path_to_module_info, info):
+ testable_module = info.get(constants.MODULE_NAME)
+ if testable_module:
+ modules.add(testable_module)
+
+ logging.debug('Probing all testable modules took %ss',
+ time.time() - begin)
+
+ if suite:
+ _modules = set()
+ for module_name in modules:
+ info = name_to_module_info.get(module_name)
+ if ModuleInfo.is_suite_in_compatibility_suites(suite, info):
+ testable_module = info.get(constants.MODULE_NAME)
+ if testable_module:
+ _modules.add(testable_module)
+ return _modules
+ return modules
diff --git a/atest/module_info_unittest.py b/atest/module_info_unittest.py
index bc4a493..b8535f0 100755
--- a/atest/module_info_unittest.py
+++ b/atest/module_info_unittest.py
@@ -124,7 +124,7 @@
default_out_dir_mod_targ = 'out/dir/here/module-info.json'
# Make sure module_info_target is what we think it is.
with mock.patch.dict('os.environ', os_environ_mock, clear=True):
- mod_info = module_info.ModuleInfo()
+ mod_info = module_info.load_from_file()
self.assertEqual(default_out_dir_mod_targ,
mod_info.module_info_target)
@@ -135,7 +135,7 @@
custom_out_dir_mod_targ = 'out2/dir/here/module-info.json'
# Make sure module_info_target is what we think it is.
with mock.patch.dict('os.environ', os_environ_mock, clear=True):
- mod_info = module_info.ModuleInfo()
+ mod_info = module_info.load_from_file()
self.assertEqual(custom_out_dir_mod_targ,
mod_info.module_info_target)
@@ -146,7 +146,7 @@
custom_abs_out_dir_mod_targ = '/tmp/out/dir/module-info.json'
# Make sure module_info_target is what we think it is.
with mock.patch.dict('os.environ', os_environ_mock, clear=True):
- mod_info = module_info.ModuleInfo()
+ mod_info = module_info.load_from_file()
self.assertEqual(custom_abs_out_dir_mod_targ,
mod_info.module_info_target)
@@ -279,7 +279,7 @@
MOD_INFO_DICT[MOD_NAME2] = is_not_auto_test_config
MOD_INFO_DICT[MOD_NAME3] = is_not_auto_test_config_again
MOD_INFO_DICT[MOD_NAME4] = {}
- mod_info.loader.name_to_module_info = MOD_INFO_DICT
+ mod_info.name_to_module_info = MOD_INFO_DICT
self.assertTrue(mod_info.is_auto_gen_test_config(MOD_NAME1))
self.assertFalse(mod_info.is_auto_gen_test_config(MOD_NAME2))
self.assertFalse(mod_info.is_auto_gen_test_config(MOD_NAME3))
@@ -287,12 +287,12 @@
def test_merge_build_system_infos(self):
"""Test _merge_build_system_infos."""
- mod_info = module_info.load_from_file(module_file=JSON_FILE_PATH)
+ loader = module_info.Loader(module_file=JSON_FILE_PATH)
mod_info_1 = {constants.MODULE_NAME: 'module_1',
constants.MODULE_DEPENDENCIES: []}
name_to_mod_info = {'module_1' : mod_info_1}
expect_deps = ['test_dep_level_1_1', 'test_dep_level_1_2']
- name_to_mod_info = mod_info._merge_build_system_infos(
+ name_to_mod_info = loader._merge_build_system_infos(
name_to_mod_info, java_bp_info_path=self.java_dep_path)
self.assertEqual(
name_to_mod_info['module_1'].get(constants.MODULE_DEPENDENCIES),
@@ -300,8 +300,8 @@
def test_merge_build_system_infos_missing_keys(self):
"""Test _merge_build_system_infos for keys missing from module-info.json."""
- mod_info = module_info.load_from_file(module_file=JSON_FILE_PATH)
- name_to_mod_info = mod_info._merge_build_system_infos(
+ loader = module_info.Loader(module_file=JSON_FILE_PATH)
+ name_to_mod_info = loader._merge_build_system_infos(
{}, java_bp_info_path=self.java_dep_path)
expect_deps = ['test_dep_level_1_1']
@@ -311,12 +311,12 @@
def test_merge_dependency_with_ori_dependency(self):
"""Test _merge_dependency."""
- mod_info = module_info.load_from_file(module_file=JSON_FILE_PATH)
+ loader = module_info.Loader(module_file=JSON_FILE_PATH)
mod_info_1 = {constants.MODULE_NAME: 'module_1',
constants.MODULE_DEPENDENCIES: ['ori_dep_1']}
name_to_mod_info = {'module_1' : mod_info_1}
expect_deps = ['ori_dep_1', 'test_dep_level_1_1', 'test_dep_level_1_2']
- name_to_mod_info = mod_info._merge_build_system_infos(
+ name_to_mod_info = loader._merge_build_system_infos(
name_to_mod_info, java_bp_info_path=self.java_dep_path)
self.assertEqual(
name_to_mod_info['module_1'].get(constants.MODULE_DEPENDENCIES),
@@ -391,34 +391,37 @@
def test_get_module_dependency(self):
"""Test get_module_dependency."""
- mod_info = module_info.load_from_file(module_file=JSON_FILE_PATH)
+ loader = module_info.Loader(module_file=JSON_FILE_PATH)
+ mod_info = loader.load()
expect_deps = {'test_dep_level_1_1', 'module_1', 'test_dep_level_1_2',
'test_dep_level_2_2', 'test_dep_level_2_1', 'module_2'}
- mod_info._merge_build_system_infos(mod_info.name_to_module_info,
- java_bp_info_path=self.java_dep_path)
+ loader._merge_build_system_infos(
+ loader.name_to_module_info, java_bp_info_path=self.java_dep_path)
self.assertEqual(
mod_info.get_module_dependency('dep_test_module'),
expect_deps)
def test_get_module_dependency_w_loop(self):
"""Test get_module_dependency with problem dep file."""
- mod_info = module_info.load_from_file(module_file=JSON_FILE_PATH)
+ loader = module_info.Loader(module_file=JSON_FILE_PATH)
+ mod_info = loader.load()
# Java dependency file with a endless loop define.
java_dep_file = os.path.join(uc.TEST_DATA_DIR,
'module_bp_java_loop_deps.json')
expect_deps = {'test_dep_level_1_1', 'module_1', 'test_dep_level_1_2',
'test_dep_level_2_2', 'test_dep_level_2_1', 'module_2'}
- mod_info._merge_build_system_infos(mod_info.name_to_module_info,
- java_bp_info_path=java_dep_file)
+ loader._merge_build_system_infos(
+ loader.name_to_module_info, java_bp_info_path=java_dep_file)
self.assertEqual(
mod_info.get_module_dependency('dep_test_module'),
expect_deps)
def test_get_install_module_dependency(self):
"""Test get_install_module_dependency."""
- mod_info = module_info.load_from_file(module_file=JSON_FILE_PATH)
+ loader = module_info.Loader(module_file=JSON_FILE_PATH)
+ mod_info = loader.load()
expect_deps = {'module_1', 'test_dep_level_2_1'}
- mod_info._merge_build_system_infos(mod_info.name_to_module_info,
+ loader._merge_build_system_infos(loader.name_to_module_info,
java_bp_info_path=self.java_dep_path)
self.assertEqual(
mod_info.get_install_module_dependency('dep_test_module'),
@@ -426,12 +429,12 @@
def test_cc_merge_build_system_infos(self):
"""Test _merge_build_system_infos for cc."""
- mod_info = module_info.load_from_file(module_file=JSON_FILE_PATH)
+ loader = module_info.Loader(module_file=JSON_FILE_PATH)
mod_info_1 = {constants.MODULE_NAME: 'module_cc_1',
constants.MODULE_DEPENDENCIES: []}
name_to_mod_info = {'module_cc_1' : mod_info_1}
expect_deps = ['test_cc_dep_level_1_1', 'test_cc_dep_level_1_2']
- name_to_mod_info = mod_info._merge_build_system_infos(
+ name_to_mod_info = loader._merge_build_system_infos(
name_to_mod_info, cc_bp_info_path=self.cc_dep_path)
self.assertEqual(
name_to_mod_info['module_cc_1'].get(constants.MODULE_DEPENDENCIES),
@@ -660,7 +663,7 @@
modules = modules or []
for m in modules:
- mod_info.loader.name_to_module_info[m['module_name']] = m
+ mod_info.name_to_module_info[m['module_name']] = m
for path in m['path']:
if path in mod_info.path_to_module_info:
mod_info.path_to_module_info[path].append(m)
diff --git a/atest/test_finders/module_finder_unittest.py b/atest/test_finders/module_finder_unittest.py
index 20fea6e..787ff89 100755
--- a/atest/test_finders/module_finder_unittest.py
+++ b/atest/test_finders/module_finder_unittest.py
@@ -212,18 +212,13 @@
return module_info.load_from_file(module_file=fake_temp_file_name)
def create_module_info(self, modules=None):
- mod_info = self.create_empty_module_info()
modules = modules or []
+ name_to_module_info = {}
for m in modules:
- mod_info.loader.name_to_module_info[m['module_name']] = m
- for path in m['path']:
- if path in mod_info.path_to_module_info:
- mod_info.path_to_module_info[path].append(m)
- else:
- mod_info.path_to_module_info[path] = [m]
+ name_to_module_info[m['module_name']] = m
- return mod_info
+ return module_info.load_from_dict(name_to_module_info=name_to_module_info)
# TODO: remove below mocks and hide unnecessary information.
@mock.patch.object(module_finder.ModuleFinder, '_get_test_info_filter')
@@ -1383,13 +1378,13 @@
def create_module_info(modules=None):
- mod_info = create_empty_module_info()
+ name_to_module_info = {}
modules = modules or []
for m in modules:
- mod_info.loader.name_to_module_info[m['module_name']] = m
+ name_to_module_info[m['module_name']] = m
- return mod_info
+ return module_info.load_from_dict(name_to_module_info)
# pylint: disable=too-many-arguments
@@ -1414,7 +1409,7 @@
m['module_name'] = name
m['class'] = classes
- m['path'] = [path or '']
+ m['path'] = path or ['']
m['installed'] = installed or []
m['is_unit_test'] = 'false'
m['auto_test_config'] = auto_test_config or []
diff --git a/atest/test_runner_handler_unittest.py b/atest/test_runner_handler_unittest.py
index 6686bbf..c868461 100755
--- a/atest/test_runner_handler_unittest.py
+++ b/atest/test_runner_handler_unittest.py
@@ -129,7 +129,7 @@
"""Test that the return value as we expected."""
results_dir = ""
extra_args = {}
- mod_info = module_info.ModuleInfo(
+ mod_info = module_info.load_from_file(
module_file=os.path.join(uc.TEST_DATA_DIR, uc.JSON_FILE))
# Tests both run_tests return 0
test_infos = [MODULE_INFO_A, MODULE_INFO_A_AGAIN]
