Cert: make test setup more robust to binary failures
* Add checks to see if a backing process or root-canal is alive
immediately after launching them to avoid making the whole
test stuck on a response from these dead processes
* Add methods to disable verity on test devices and push test files
to required directories on the Android device
Bug: 150321998
Test: source gd/cert/set_up_and_run_device_cert.sh
Change-Id: I44f2e73aa88e7924c7e594faad020f8199c76994
diff --git a/system/gd/cert/gd_base_test_facade_only.py b/system/gd/cert/gd_base_test_facade_only.py
index cdda85a..1412e64 100644
--- a/system/gd/cert/gd_base_test_facade_only.py
+++ b/system/gd/cert/gd_base_test_facade_only.py
@@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from acts import asserts
from acts.base_test import BaseTestClass
from facade import rootservice_pb2 as facade_rootservice
@@ -24,6 +25,14 @@
import subprocess
+def is_subprocess_alive(process, timeout_seconds=1):
+ try:
+ process.wait(timeout=timeout_seconds)
+ return False
+ except subprocess.TimeoutExpired as exp:
+ return True
+
+
class GdFacadeOnlyBaseTestClass(BaseTestClass):
def setup_class(self, dut_module, cert_module):
@@ -54,9 +63,14 @@
env=os.environ.copy(),
stdout=self.rootcanal_logs,
stderr=self.rootcanal_logs)
+ asserts.assert_true(
+ self.rootcanal_process,
+ msg="Cannot start root-canal at " + str(rootcanal))
+ asserts.assert_true(
+ is_subprocess_alive(self.rootcanal_process),
+ msg="root-canal stopped immediately after running")
for gd_device in gd_devices:
gd_device["rootcanal_port"] = rootcanal_hci_port
-
self.register_controller(
importlib.import_module('cert.gd_device'), builtin=True)
diff --git a/system/gd/cert/gd_device_base.py b/system/gd/cert/gd_device_base.py
index e18adaa..7c00cfc 100644
--- a/system/gd/cert/gd_device_base.py
+++ b/system/gd/cert/gd_device_base.py
@@ -23,18 +23,21 @@
import subprocess
import time
+from acts import asserts
from acts import context
-from acts.controllers.adb import AdbProxy
+from acts.controllers.adb import AdbProxy, AdbError
import grpc
from cert.environment_provider import PRODUCT_DEVICE
+from cert.gd_base_test_facade_only import is_subprocess_alive
ANDROID_PRODUCT_OUT = os.path.join(
os.getcwd(), "out/dist/bluetooth_cert_test/out/target/product",
PRODUCT_DEVICE)
WAIT_CHANNEL_READY_TIMEOUT = 10
+WAIT_FOR_DEVICE_TIMEOUT = 180
def replace_vars(string, config):
@@ -77,26 +80,31 @@
self.serial_number = serial_number
if self.serial_number:
- self.ad = AdbProxy(serial_number)
- self.ad.shell("date " + time.strftime("%m%d%H%M%Y.%S"))
- self.ad.tcp_forward(int(grpc_port), int(grpc_port))
- self.ad.tcp_forward(
+ self.adb = AdbProxy(self.serial_number)
+ self.ensure_verity_disabled()
+ asserts.assert_true(
+ self.adb.ensure_root(),
+ msg="device %s cannot run as root after enabling verity" %
+ self.serial_number)
+ self.adb.shell("date " + time.strftime("%m%d%H%M%Y.%S"))
+ self.adb.tcp_forward(int(grpc_port), int(grpc_port))
+ self.adb.tcp_forward(
int(grpc_root_server_port), int(grpc_root_server_port))
- self.ad.reverse("tcp:%s tcp:%s" % (signal_port, signal_port))
- self.ad.push(
+ self.adb.reverse("tcp:%s tcp:%s" % (signal_port, signal_port))
+ self.push_or_die(
os.path.join(ANDROID_PRODUCT_OUT,
"system/bin/bluetooth_stack_with_facade"),
"system/bin")
- self.ad.push(
+ self.push_or_die(
os.path.join(ANDROID_PRODUCT_OUT,
"system/lib64/libbluetooth_gd.so"), "system/lib64")
- self.ad.push(
+ self.push_or_die(
os.path.join(ANDROID_PRODUCT_OUT,
"system/lib64/libgrpc++_unsecure.so"),
"system/lib64")
- self.ad.shell("logcat -c")
- self.ad.shell("rm /data/misc/bluetooth/logs/btsnoop_hci.log")
- self.ad.shell("svc bluetooth disable")
+ self.ensure_no_output(self.adb.shell("logcat -c"))
+ self.adb.shell("rm /data/misc/bluetooth/logs/btsnoop_hci.log")
+ self.ensure_no_output(self.adb.shell("svc bluetooth disable"))
tester_signal_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tester_signal_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,
@@ -111,6 +119,13 @@
env=os.environ.copy(),
stdout=self.backing_process_logs,
stderr=self.backing_process_logs)
+ asserts.assert_true(
+ self.backing_process,
+ msg="Cannot start backing_process at " + " ".join(cmd))
+ asserts.assert_true(
+ is_subprocess_alive(self.backing_process),
+ msg="backing_process stopped immediately after running " +
+ " ".join(cmd))
tester_signal_socket.accept()
tester_signal_socket.close()
@@ -131,11 +146,11 @@
(self.label, backing_process_return_code))
if self.serial_number:
- self.ad.shell("logcat -d -f /data/misc/bluetooth/logs/system_log")
- self.ad.pull(
+ self.adb.shell("logcat -d -f /data/misc/bluetooth/logs/system_log")
+ self.adb.pull(
"/data/misc/bluetooth/logs/btsnoop_hci.log %s" % os.path.join(
self.log_path_base, "%s_btsnoop_hci.log" % self.label))
- self.ad.pull(
+ self.adb.pull(
"/data/misc/bluetooth/logs/system_log %s" % os.path.join(
self.log_path_base, "%s_system_log" % self.label))
@@ -145,3 +160,97 @@
future.result(timeout=WAIT_CHANNEL_READY_TIMEOUT)
except grpc.FutureTimeoutError:
logging.error("wait channel ready timeout")
+
+ def ensure_no_output(self, result):
+ """
+ Ensure a command has not output
+ """
+ asserts.assert_true(
+ result is None or len(result) == 0,
+ msg="command returned something when it shouldn't: %s" % result)
+
+ def push_or_die(self, src_file_path, dst_file_path, push_timeout=300):
+ """Pushes a file to the Android device
+
+ Args:
+ src_file_path: The path to the file to install.
+ dst_file_path: The destination of the file.
+ push_timeout: How long to wait for the push to finish in seconds
+ """
+ try:
+ self.adb.ensure_root()
+ self.ensure_verity_disabled()
+ out = self.adb.push(
+ '%s %s' % (src_file_path, dst_file_path), timeout=push_timeout)
+ if 'error' in out:
+ asserts.fail('Unable to push file %s to %s due to %s' %
+ (src_file_path, dst_file_path, out))
+ except Exception as e:
+ asserts.fail(
+ msg='Unable to push file %s to %s due to %s' %
+ (src_file_path, dst_file_path, e),
+ extras=e)
+
+ def ensure_verity_disabled(self):
+ """Ensures that verity is enabled.
+
+ If verity is not enabled, this call will reboot the phone. Note that
+ this only works on debuggable builds.
+ """
+ logging.debug("Disabling verity and remount for %s", self.serial_number)
+ asserts.assert_true(self.adb.ensure_root(),
+ "device %s cannot run as root", self.serial_number)
+ # The below properties will only exist if verity has been enabled.
+ system_verity = self.adb.getprop('partition.system.verified')
+ vendor_verity = self.adb.getprop('partition.vendor.verified')
+ if system_verity or vendor_verity:
+ self.adb.disable_verity()
+ self.reboot()
+ self.adb.remount()
+ self.adb.wait_for_device(timeout=WAIT_FOR_DEVICE_TIMEOUT)
+
+ def reboot(self, timeout_minutes=15.0):
+ """Reboots the device.
+
+ Reboot the device, wait for device to complete booting.
+ """
+ logging.debug("Rebooting %s", self.serial_number)
+ self.adb.reboot()
+
+ timeout_start = time.time()
+ timeout = timeout_minutes * 60
+ # Android sometimes return early after `adb reboot` is called. This
+ # means subsequent calls may make it to the device before the reboot
+ # goes through, return false positives for getprops such as
+ # sys.boot_completed.
+ while time.time() < timeout_start + timeout:
+ try:
+ self.adb.get_state()
+ time.sleep(.1)
+ except AdbError:
+ # get_state will raise an error if the device is not found. We
+ # want the device to be missing to prove the device has kicked
+ # off the reboot.
+ break
+ minutes_left = timeout_minutes - (time.time() - timeout_start) / 60.0
+ self.wait_for_boot_completion(timeout_minutes=minutes_left)
+
+ def wait_for_boot_completion(self, timeout_minutes=15.0):
+ """Waits for Android framework to broadcast ACTION_BOOT_COMPLETED.
+ """
+ timeout_start = time.time()
+ timeout = timeout_minutes * 60
+
+ self.adb.wait_for_device(timeout=WAIT_FOR_DEVICE_TIMEOUT)
+ while time.time() < timeout_start + timeout:
+ try:
+ completed = self.adb.getprop("sys.boot_completed")
+ if completed == '1':
+ return
+ except AdbError:
+ # adb shell calls may fail during certain period of booting
+ # process, which is normal. Ignoring these errors.
+ pass
+ time.sleep(5)
+ asserts.fail(msg='Device %s booting process timed out.' %
+ self.serial_number)
