autotest: Add retries to jetstream host verification
We're seeing some provisioning failures in the Destiny lab
due to the AP controller not running immediately after
provisioning. This adds up to 1 minute of retries.
BUG=b:62576902
TEST=patched to moblab, test successful and unsuccessful verification.
Change-Id: I1a59ac9e7c892ee0942db005f89f2389871fd4ac
Reviewed-on: https://chromium-review.googlesource.com/527880
Commit-Ready: Laurence Goodby <lgoodby@chromium.org>
Tested-by: Laurence Goodby <lgoodby@chromium.org>
Reviewed-by: Prathmesh Prabhu <pprabhu@chromium.org>
diff --git a/server/hosts/cros_repair.py b/server/hosts/cros_repair.py
index 32f62ba..1847455 100644
--- a/server/hosts/cros_repair.py
+++ b/server/hosts/cros_repair.py
@@ -11,6 +11,7 @@
from autotest_lib.client.common_lib import error
from autotest_lib.client.common_lib import global_config
from autotest_lib.client.common_lib import hosts
+from autotest_lib.client.common_lib.cros import retry
from autotest_lib.server import afe_utils
from autotest_lib.server import crashcollect
from autotest_lib.server.hosts import repair
@@ -298,10 +299,17 @@
class JetstreamServicesVerifier(hosts.Verifier):
"""Verify that Jetstream services are running."""
+ # Retry for b/62576902
+ @retry.retry(error.AutoservError, timeout_min=1, delay_sec=10)
def verify(self, host):
- if not host.upstart_status('ap-controller'):
+ try:
+ if not host.upstart_status('ap-controller'):
+ raise hosts.AutoservVerifyError(
+ 'ap-controller service is not running')
+ except error.AutoservRunError:
raise hosts.AutoservVerifyError(
- 'ap-controller service is not running')
+ 'ap-controller service not found')
+
try:
host.run('pgrep ap-controller')
except error.AutoservRunError:
