scripts: Support composite command in test maps
Some tests are side-effectful and must therefore be excecuted in
a particular order. Add a composite command element to group such
tests. Composite tests can contain port tests and and reboot
commands. The latter should no longer occur outside composite
tests.
Bug: 236898842
Change-Id: I28569a3ae4a819b3cdf2c39fea4c9125e33551ea
diff --git a/scripts/run_tests.py b/scripts/run_tests.py
index bc0f028..e8242aa 100755
--- a/scripts/run_tests.py
+++ b/scripts/run_tests.py
@@ -31,8 +31,12 @@
import subprocess
import sys
import time
+from typing import Optional
-import trusty_build_config
+from trusty_build_config import TrustyTest, TrustyCompositeTest
+from trusty_build_config import TrustyRebootCommand, TrustyHostTest
+from trusty_build_config import TrustyBuildConfig
+
class TestResults(object):
"""Stores test results.
@@ -144,7 +148,7 @@
return run
- def run_test(test):
+ def run_test(test, parent_test: Optional[TrustyCompositeTest] = None) -> int:
"""Execute a single test and print out helpful information"""
cmd = test.command[1:]
disable_rpmb = True if "--disable_rpmb" in cmd else None
@@ -154,14 +158,21 @@
test_start_time = time.time()
match test:
- case trusty_build_config.TrustyHostTest():
+ case TrustyHostTest():
# append nice and expand path to command
cmd = ["nice", f"{project_root}/{test.command[0]}"] + cmd
print("Command line:",
" ".join([s.replace(" ", "\\ ") for s in cmd]),
flush=True)
status = subprocess.call(cmd)
- case trusty_build_config.TrustyTest():
+ case TrustyCompositeTest():
+ status = 0
+ for subtest in test.sequence:
+ if status := run_test(subtest, test):
+ # fail the composite test with the same status code as
+ # the first failing subtest
+ break
+ case TrustyTest():
if hasattr(test, "shell_command"):
print("Command line:",
test.shell_command.replace(" ", "\\ "),
@@ -181,16 +192,21 @@
finally:
if test_env:
test_env.shutdown(test_runner)
- case trusty_build_config.TrustyRebootCommand():
+ case TrustyRebootCommand() if parent_test:
+ assert isinstance(parent_test, TrustyCompositeTest)
# ignore reboot commands since we are not (yet) batching
# tests such that they can share an emulator instance.
- return
+ return 0
+ case TrustyRebootCommand():
+ raise RuntimeError(
+ "Reboot may only be used inside compositetest")
case _:
raise NotImplementedError(f"Don't know how to run {test.name}")
elapsed = time.time() - test_start_time
print(f"{test.name:s} returned {status:d} after {elapsed:.3f} seconds")
test_results.add_result(test.name, status == 0)
+ return status
for test in project_config.tests:
if not test.enabled and not run_disabled_tests:
@@ -211,7 +227,7 @@
help="Project to test.")
args = parser.parse_args()
- build_config = trusty_build_config.TrustyBuildConfig()
+ build_config = TrustyBuildConfig()
test_results = run_tests(build_config, args.root, args.project)
test_results.print_results()
if not test_results.passed:
diff --git a/scripts/test-map b/scripts/test-map
index 34b2356..7873049 100644
--- a/scripts/test-map
+++ b/scripts/test-map
@@ -41,13 +41,27 @@
# tests=[
# # Run a program on the host
# hosttest("some_host_binary"),
-# # Use test-runner to activate a Trusty IPC port
-# boottest("port.under.test"),
-# # Use Android to activate a Trusty IPC port
-# androidport("port.under.test"),
+# # Run test on device or in emulator. Porttests run in one of two
+# # contexts:
+# # 1. In a minimal
+# # bootloader environment (when porttest is nested within
+# # in a boottests element), or
+# # 2. with a full Android userspace present (when nested within an
+# # androidporttests element).
+# porttest("port.under.test"),
# # Run a shell command inside Android
# androidtest(name="test_name", command="command to run"),
-# ...
+# # Run a sequence of test and commands in the given order
+# # Ensure that test environment is rebooted before second port test
+# compositetest(name="testname", sequence=[
+# hosttest("some_host_binary"),
+# porttest("port.under.test"),
+# # a reboot may only be requested inside composite tests
+# reboot(),
+# porttest("another.port.under.test"),
+# ...
+# ]
+# ...
# ],
# ),
# ...
diff --git a/scripts/trusty_build_config.py b/scripts/trusty_build_config.py
index dd7e2f5..ab7d04c 100755
--- a/scripts/trusty_build_config.py
+++ b/scripts/trusty_build_config.py
@@ -24,6 +24,7 @@
import argparse
import os
import re
+from typing import List
script_dir = os.path.dirname(os.path.abspath(__file__))
@@ -173,6 +174,37 @@
super().__init__("reboot command")
+class TrustyCompositeTest(TrustyTest):
+ """Stores a sequence of tests that must execute in order"""
+
+ def __init__(self, name: str,
+ sequence: List[TrustyPortTest | TrustyCommand],
+ enabled=True):
+ super().__init__(name, [], enabled)
+ self.sequence = sequence
+ flags = set()
+ for subtest in sequence:
+ flags.update(subtest.need.flags)
+ self.need = TrustyPortTestFlags(**{flag: True for flag in flags})
+
+ def needs(self, **need):
+ self.need.set(**need)
+ return self
+
+ def into_androidporttest(self, **kwargs):
+ # because the needs of the composite test is the union of the needs of
+ # its subtests, we do not need to filter out any subtests; all needs met
+ self.sequence = [subtest.into_androidporttest(**kwargs)
+ for subtest in self.sequence]
+ return self
+
+ def into_bootporttest(self):
+ # similarly to into_androidporttest, we do not need to filter out tests
+ self.sequence = [subtest.into_bootporttest()
+ for subtest in self.sequence]
+ return self
+
+
class TrustyBuildConfig(object):
"""Trusty build and test configuration file parser."""
@@ -309,6 +341,7 @@
"testmap": testmap,
"hosttest": hosttest,
"porttest": TrustyPortTest,
+ "compositetest": TrustyCompositeTest,
"porttestflags": TrustyPortTestFlags,
"hosttests": hosttests,
"boottests": boottests,
@@ -482,6 +515,32 @@
print("get_projects test passed")
reboot_seen = False
+
+ def check_test(i, test):
+ match test:
+ case TrustyTest():
+ host_m = re.match(r"host-test:self_test.*\.(\d+)",
+ test.name)
+ unit_m = re.match(r"boot-test:self_test.*\.(\d+)",
+ test.name)
+ if args.debug:
+ print(project, i, test.name)
+ m = host_m or unit_m
+ assert m
+ assert m.group(1) == str(i + 1)
+ case TrustyRebootCommand():
+ assert False, "Reboot outside composite command"
+ case _:
+ assert False, "Unexpected test type"
+
+ def check_subtest(i, test):
+ nonlocal reboot_seen
+ match test:
+ case TrustyRebootCommand():
+ reboot_seen = True
+ case _:
+ check_test(i, test)
+
for project_name in config.get_projects():
project = config.get_project(project_name)
if args.debug:
@@ -506,18 +565,15 @@
for i, test in enumerate(project.tests):
match test:
- case TrustyRebootCommand():
- reboot_seen = True
- case TrustyTest():
- host_m = re.match(r"host-test:self_test.*\.(\d+)",
- test.name)
- unit_m = re.match(r"boot-test:self_test.*\.(\d+)",
- test.name)
- if args.debug:
- print(project, i, test.name)
- m = host_m or unit_m
- assert m
- assert m.group(1) == str(i + 1)
+ case TrustyCompositeTest():
+ # because one of its subtest needs storage_boot,
+ # the composite test should similarly need it
+ assert "storage_boot" in test.need.flags
+ for subtest in test.sequence:
+ check_subtest(i, subtest)
+ case _:
+ check_test(i, test)
+
assert reboot_seen
print("get_tests test passed")
diff --git a/scripts/trusty_build_config_self_test_include1 b/scripts/trusty_build_config_self_test_include1
index c9bb0b8..55914c8 100644
--- a/scripts/trusty_build_config_self_test_include1
+++ b/scripts/trusty_build_config_self_test_include1
@@ -49,8 +49,14 @@
porttest("self_test.c.1"),
porttest("self_test.a.2"),
porttest("self_test.b.3"),
- hosttest("self_test.d.4"),
- reboot(),
+ hosttest("self_test.d.5"),
+ compositetest(
+ name="self_test.composite.a",
+ sequence=[
+ porttest("self_test.a.4").needs(storage_boot=True),
+ reboot(),
+ ],
+ ),
]),
],
),
