[moblab] Call new upstart command that correctly restarts apache.
restart moblab-apache-init does not cleanly or correctly restart
apache, call a new upstart config that calls the correct command
to restart apache.
Replace the use of the boto library with a gsutil command.
Use the same path to the gsutil command that the devserver uses.
BUG=chromium:714294
TEST=locally on moblab, unittests
Change-Id: I0befe734d2c6130dadcfdaa5984aec23bba07807
Reviewed-on: https://chromium-review.googlesource.com/497169
Commit-Ready: Keith Haddow <haddowk@chromium.org>
Tested-by: Keith Haddow <haddowk@chromium.org>
Reviewed-by: Keith Haddow <haddowk@chromium.org>
Reviewed-by: Michael Tang <ntang@chromium.org>
diff --git a/frontend/afe/moblab_rpc_interface.py b/frontend/afe/moblab_rpc_interface.py
index d73aea8..ee7a60b 100644
--- a/frontend/afe/moblab_rpc_interface.py
+++ b/frontend/afe/moblab_rpc_interface.py
@@ -7,14 +7,6 @@
on moblab.
"""
-# The boto module is only available/used in Moblab for validation of cloud
-# storage access. The module is not available in the test lab environment,
-# and the import error is handled.
-try:
- import boto
-except ImportError:
- boto = None
-
import ConfigParser
import common
import logging
@@ -31,7 +23,7 @@
from autotest_lib.frontend.afe import rpc_utils
from autotest_lib.server import frontend
from autotest_lib.server.hosts import moblab_host
-
+from chromite.lib import gs
_CONFIG = global_config.global_config
MOBLAB_BOTO_LOCATION = '/home/moblab/.boto'
@@ -45,7 +37,7 @@
# Contants used in Json RPC field names.
_IMAGE_STORAGE_SERVER = 'image_storage_server'
_GS_ACCESS_KEY_ID = 'gs_access_key_id'
-_GS_SECRETE_ACCESS_KEY = 'gs_secret_access_key'
+_GS_SECRET_ACCESS_KEY = 'gs_secret_access_key'
_RESULT_STORAGE_SERVER = 'results_storage_server'
_USE_EXISTING_BOTO_FILE = 'use_existing_boto_file'
@@ -55,6 +47,12 @@
# File where information about the current device is stored.
_ETC_LSB_RELEASE = '/etc/lsb-release'
+# Full path to the correct gsutil command to run.
+_GSUTIL_CMD = gs.GSContext.GetDefaultGSUtilBin()
+
+class BucketPerformanceTestException(Exception):
+ pass
+
@rpc_utils.moblab_only
def get_config_values():
"""Returns all config values parsed from global and shadow configs.
@@ -254,8 +252,8 @@
def _get_network_info():
"""Gets the network information.
- TCP socket is used to test the connectivity. If there is no connectivity, try to
- get the public IP with UDP socket.
+ TCP socket is used to test the connectivity. If there is no connectivity,
+ try to get the public IP with UDP socket.
@return: a tuple as (public_ip_address, connected_to_internet).
"""
@@ -320,9 +318,9 @@
if _GS_ACCESS_KEY_ID in options:
value = boto_config.get('Credentials', _GS_ACCESS_KEY_ID)
cloud_storage_info[_GS_ACCESS_KEY_ID] = value
- if _GS_SECRETE_ACCESS_KEY in options:
- value = boto_config.get('Credentials', _GS_SECRETE_ACCESS_KEY)
- cloud_storage_info[_GS_SECRETE_ACCESS_KEY] = value
+ if _GS_SECRET_ACCESS_KEY in options:
+ value = boto_config.get('Credentials', _GS_SECRET_ACCESS_KEY)
+ cloud_storage_info[_GS_SECRET_ACCESS_KEY] = value
return rpc_utils.prepare_for_serialization(cloud_storage_info)
@@ -338,61 +336,12 @@
return match.group('bucket')
return None
-
-def _is_valid_boto_key(key_id, key_secret):
- """Checks if the boto key is valid.
-
- @param: key_id: The boto key id string.
- @param: key_secret: The boto key string.
-
- @return: A tuple as (valid_boolean, details_string).
- """
- if not key_id or not key_secret:
- return (False, "Empty key id or secret.")
- conn = boto.connect_gs(key_id, key_secret)
- try:
- buckets = conn.get_all_buckets()
- return (True, None)
- except boto.exception.GSResponseError:
- details = "The boto access key is not valid"
- return (False, details)
- finally:
- conn.close()
-
-
-def _is_valid_bucket(key_id, key_secret, bucket_name):
- """Checks if a bucket is valid and accessible.
-
- @param: key_id: The boto key id string.
- @param: key_secret: The boto key string.
- @param: bucket name string.
-
- @return: A tuple as (valid_boolean, details_string).
- """
- if not key_id or not key_secret or not bucket_name:
- return (False, "Server error: invalid argument")
- conn = boto.connect_gs(key_id, key_secret)
- bucket = conn.lookup(bucket_name)
- conn.close()
- if bucket:
- return (True, None)
- return (False, "Bucket %s does not exist." % bucket_name)
-
-
-def _is_valid_bucket_url(key_id, key_secret, bucket_url):
- """Validates the bucket url is accessible.
-
- @param: key_id: The boto key id string.
- @param: key_secret: The boto key string.
- @param: bucket url string.
-
- @return: A tuple as (valid_boolean, details_string).
- """
- bucket_name = _get_bucket_name_from_url(bucket_url)
- if bucket_name:
- return _is_valid_bucket(key_id, key_secret, bucket_name)
- return (False, "Bucket url %s is not valid" % bucket_url)
-
+def _is_valid_boto_key(key_id, key_secret, directory):
+ try:
+ _run_bucket_performance_test(key_id, key_secret, directory)
+ except BucketPerformanceTestException as e:
+ return(False, str(e))
+ return(True, None)
def _validate_cloud_storage_info(cloud_storage_info):
"""Checks if the cloud storage information is valid.
@@ -405,17 +354,9 @@
details = None
if not cloud_storage_info[_USE_EXISTING_BOTO_FILE]:
key_id = cloud_storage_info[_GS_ACCESS_KEY_ID]
- key_secret = cloud_storage_info[_GS_SECRETE_ACCESS_KEY]
- valid, details = _is_valid_boto_key(key_id, key_secret)
-
- if valid:
- valid, details = _is_valid_bucket_url(
- key_id, key_secret, cloud_storage_info[_IMAGE_STORAGE_SERVER])
-
- # allows result bucket to be empty.
- if valid and cloud_storage_info[_RESULT_STORAGE_SERVER]:
- valid, details = _is_valid_bucket_url(
- key_id, key_secret, cloud_storage_info[_RESULT_STORAGE_SERVER])
+ key_secret = cloud_storage_info[_GS_SECRET_ACCESS_KEY]
+ valid, details = _is_valid_boto_key(
+ key_id, key_secret, cloud_storage_info[_IMAGE_STORAGE_SERVER])
return (valid, details)
@@ -464,16 +405,18 @@
boto_config.add_section('Credentials')
boto_config.set('Credentials', _GS_ACCESS_KEY_ID,
cloud_storage_info[_GS_ACCESS_KEY_ID])
- boto_config.set('Credentials', _GS_SECRETE_ACCESS_KEY,
- cloud_storage_info[_GS_SECRETE_ACCESS_KEY])
+ boto_config.set('Credentials', _GS_SECRET_ACCESS_KEY,
+ cloud_storage_info[_GS_SECRET_ACCESS_KEY])
_write_config_file(MOBLAB_BOTO_LOCATION, boto_config, True)
_CONFIG.parse_config_file()
- services = ['moblab-devserver-init', 'moblab-apache-init',
+ services = ['moblab-devserver-init',
'moblab-devserver-cleanup-init', ' moblab-gsoffloader_s-init',
- 'moblab-base-container-init', 'moblab-scheduler-init', 'moblab-gsoffloader-init']
- cmd = ';/sbin/restart '.join(services)
- os.system(cmd)
+ 'moblab-base-container-init', 'moblab-scheduler-init',
+ 'moblab-gsoffloader-init']
+ cmd = ';sudo /sbin/restart '.join(services)
+ cmd += ';sudo /usr/sbin/apache2 -k graceful'
+ os.system("export ATEST_RESULTS_DIR=/usr/local/autotest/results;" + cmd)
return _create_operation_status_response(True, None)
@@ -581,7 +524,8 @@
if label:
label.host_set.add(host_obj)
return (True, 'Added label %s to DUT %s' % (label_name, ipaddress))
- return (False, 'Failed to add label %s to DUT %s' % (label_name, ipaddress))
+ return (False,
+ 'Failed to add label %s to DUT %s' % (label_name, ipaddress))
@rpc_utils.moblab_only
@@ -599,11 +543,13 @@
def _get_connected_dut_labels(requested_label, only_first_label=True):
- """ Query the DUT's attached to the moblab and return a filtered list of labels.
+ """ Query the DUT's attached to the moblab and return a filtered list
+ of labels.
@param requested_label: the label name you are requesting.
- @param only_first_label: if the device has the same label name multiple times only
- return the first label value in the list.
+ @param only_first_label: if the device has the same label name multiple
+ times only return the first label value in the
+ list.
@return: A de-duped list of requested dut labels attached to the moblab.
"""
@@ -668,7 +614,8 @@
return _get_builds_for_in_directory(board_name + '-firmware')
-def _get_builds_for_in_directory(directory_name, milestone_limit=3, build_limit=20):
+def _get_builds_for_in_directory(directory_name, milestone_limit=3,
+ build_limit=20):
""" Fetch the most recent builds for the last three milestones from gcs.
@@ -676,16 +623,18 @@
storage bucket to search.
- @return: A string list no longer than <milestone_limit> x <build_limit> items,
- containing the most recent <build_limit> builds from the last
- milestone_limit milestones.
+ @return: A string list no longer than <milestone_limit> x <build_limit>
+ items, containing the most recent <build_limit> builds from the
+ last milestone_limit milestones.
"""
output = StringIO.StringIO()
gs_image_location =_CONFIG.get_config_value('CROS', _IMAGE_STORAGE_SERVER)
- utils.run('gsutil', args=('ls', gs_image_location + directory_name), stdout_tee=output)
+ utils.run(_GSUTIL_CMD, args=('ls', gs_image_location + directory_name),
+ stdout_tee=output)
lines = output.getvalue().split('\n')
output.close()
- builds = [line.replace(gs_image_location,'').strip('/ ') for line in lines if line != '']
+ builds = [line.replace(gs_image_location,'').strip('/ ')
+ for line in lines if line != '']
build_matcher = re.compile(r'^.*\/R([0-9]*)-.*')
build_map = {}
for build in builds:
@@ -706,8 +655,44 @@
return build_list
+def _run_bucket_performance_test(key_id, key_secret, bucket_name,
+ test_size='1M', iterations='1',
+ result_file='/tmp/gsutil_perf.json'):
+ """Run a gsutil perfdiag on a supplied bucket and output the results"
+
+ @param key_id: boto key of the bucket to be accessed
+ @param key_secret: boto secret of the bucket to be accessed
+ @param bucket_name: bucket to be tested.
+ @param test_size: size of file to use in test, see gsutil perfdiag help.
+ @param iterations: number of times each test is run.
+ @param result_file: name of file to write results out to.
+
+ @return None
+ @raises BucketPerformanceTestException if the command fails.
+ """
+ try:
+ utils.run(_GSUTIL_CMD, args=(
+ '-o', 'Credentials:gs_access_key_id=%s' % key_id,
+ '-o', 'Credentials:gs_secret_access_key=%s' % key_secret,
+ 'perfdiag', '-s', test_size, '-o', result_file,
+ '-n', iterations,
+ bucket_name))
+ except error.CmdError as e:
+ logging.error(e)
+ # Extract useful error from the stacktrace
+ errormsg = str(e)
+ start_error_pos = errormsg.find("<Error>")
+ end_error_pos = errormsg.find("</Error>", start_error_pos)
+ extracted_error_msg = errormsg[start_error_pos:end_error_pos]
+ raise BucketPerformanceTestException(
+ extracted_error_msg if extracted_error_msg else errormsg)
+ # TODO(haddowk) send the results to the cloud console when that feature is
+ # enabled.
+
+
@rpc_utils.moblab_only
-def run_suite(board, build, suite, ro_firmware=None, rw_firmware=None, pool=None):
+def run_suite(board, build, suite, ro_firmware=None, rw_firmware=None,
+ pool=None):
""" RPC handler to run a test suite.
@param board: a board name connected to the moblab.
@@ -728,3 +713,4 @@
afe.run('create_suite_job', board=board, builds=builds, name=suite,
pool=pool, run_prod_code=False, test_source_build=build,
wait_for_results=False)
+
diff --git a/frontend/afe/moblab_rpc_interface_unittest.py b/frontend/afe/moblab_rpc_interface_unittest.py
index 1633405..d7eb30a 100644
--- a/frontend/afe/moblab_rpc_interface_unittest.py
+++ b/frontend/afe/moblab_rpc_interface_unittest.py
@@ -10,12 +10,7 @@
# The boto module is only available/used in Moblab for validation of cloud
# storage access. The module is not available in the test lab environment,
# and the import error is handled.
-try:
- import boto
-except ImportError:
- boto = None
import ConfigParser
-import logging
import mox
import StringIO
import unittest
@@ -31,6 +26,7 @@
from autotest_lib.frontend.afe import rpc_utils
from autotest_lib.server import utils
from autotest_lib.server.hosts import moblab_host
+from autotest_lib.client.common_lib import utils as common_lib_utils
class MoblabRpcInterfaceTest(mox.MoxTestBase,
@@ -160,20 +156,6 @@
moblab_rpc_interface.reset_config_settings()
- def testSetBotoKey(self):
- """Ensure that the botokey path supplied is copied correctly."""
- self.setIsMoblab(True)
- boto_key = '/tmp/boto'
- moblab_rpc_interface.os.path = self.mox.CreateMockAnything()
- moblab_rpc_interface.os.path.exists(boto_key).AndReturn(
- True)
- moblab_rpc_interface.shutil = self.mox.CreateMockAnything()
- moblab_rpc_interface.shutil.copyfile(
- boto_key, moblab_rpc_interface.MOBLAB_BOTO_LOCATION)
- self.mox.ReplayAll()
- moblab_rpc_interface.set_boto_key(boto_key)
-
-
def testSetLaunchControlKey(self):
"""Ensure that the Launch Control key path supplied is copied correctly.
"""
@@ -278,16 +260,11 @@
'gs_secret_access_key': 'secret',
'image_storage_server': 'gs://bucket1',
'results_storage_server': 'gs://bucket2'}
- self.mox.StubOutWithMock(moblab_rpc_interface, '_is_valid_boto_key')
- self.mox.StubOutWithMock(moblab_rpc_interface, '_is_valid_bucket')
- moblab_rpc_interface._is_valid_boto_key(
- 'key', 'secret').AndReturn((True, None))
- moblab_rpc_interface._is_valid_bucket(
- 'key', 'secret', 'bucket1').AndReturn((True, None))
- moblab_rpc_interface._is_valid_bucket(
- 'key', 'secret', 'bucket2').AndReturn((True, None))
- rpc_utils.prepare_for_serialization(
- {'status_ok': True })
+ self.mox.StubOutWithMock(moblab_rpc_interface,
+ '_run_bucket_performance_test')
+ moblab_rpc_interface._run_bucket_performance_test(
+ 'key', 'secret', 'gs://bucket1').AndReturn((True, None))
+ rpc_utils.prepare_for_serialization({'status_ok': True })
self.mox.ReplayAll()
moblab_rpc_interface.validate_cloud_storage_info(cloud_storage_info)
self.mox.VerifyAll()
@@ -311,75 +288,6 @@
'bucket_name-123/a/b/c'))
- def testIsValidBotoKeyValid(self):
- """Tests the boto key validation flow."""
- if boto is None:
- logging.info('skip test since boto module not installed')
- return
- conn = self.mox.CreateMockAnything()
- self.mox.StubOutWithMock(boto, 'connect_gs')
- boto.connect_gs('key', 'secret').AndReturn(conn)
- conn.get_all_buckets().AndReturn(['a', 'b'])
- conn.close()
- self.mox.ReplayAll()
- valid, details = moblab_rpc_interface._is_valid_boto_key('key', 'secret')
- self.assertTrue(valid)
- self.mox.VerifyAll()
-
-
- def testIsValidBotoKeyInvalid(self):
- """Tests the boto key validation with invalid key."""
- if boto is None:
- logging.info('skip test since boto module not installed')
- return
- conn = self.mox.CreateMockAnything()
- self.mox.StubOutWithMock(boto, 'connect_gs')
- boto.connect_gs('key', 'secret').AndReturn(conn)
- conn.get_all_buckets().AndRaise(
- boto.exception.GSResponseError('bad', 'reason'))
- conn.close()
- self.mox.ReplayAll()
- valid, details = moblab_rpc_interface._is_valid_boto_key('key', 'secret')
- self.assertFalse(valid)
- self.assertEquals('The boto access key is not valid', details)
- self.mox.VerifyAll()
-
-
- def testIsValidBucketValid(self):
- """Tests the bucket vaildation flow."""
- if boto is None:
- logging.info('skip test since boto module not installed')
- return
- conn = self.mox.CreateMockAnything()
- self.mox.StubOutWithMock(boto, 'connect_gs')
- boto.connect_gs('key', 'secret').AndReturn(conn)
- conn.lookup('bucket').AndReturn('bucket')
- conn.close()
- self.mox.ReplayAll()
- valid, details = moblab_rpc_interface._is_valid_bucket(
- 'key', 'secret', 'bucket')
- self.assertTrue(valid)
- self.mox.VerifyAll()
-
-
- def testIsValidBucketInvalid(self):
- """Tests the bucket validation flow with invalid key."""
- if boto is None:
- logging.info('skip test since boto module not installed')
- return
- conn = self.mox.CreateMockAnything()
- self.mox.StubOutWithMock(boto, 'connect_gs')
- boto.connect_gs('key', 'secret').AndReturn(conn)
- conn.lookup('bucket').AndReturn(None)
- conn.close()
- self.mox.ReplayAll()
- valid, details = moblab_rpc_interface._is_valid_bucket(
- 'key', 'secret', 'bucket')
- self.assertFalse(valid)
- self.assertEquals("Bucket bucket does not exist.", details)
- self.mox.VerifyAll()
-
-
def testGetShadowConfigFromPartialUpdate(self):
"""Tests getting shadow configuration based on partial upate."""
partial_config = {
@@ -462,5 +370,27 @@
self.mox.VerifyAll()
+ def testRunBucketPerformanceTestFail(self):
+ self.mox.StubOutWithMock(common_lib_utils, 'run')
+ common_lib_utils.run(moblab_rpc_interface._GSUTIL_CMD,
+ args=(
+ '-o', 'Credentials:gs_access_key_id=key',
+ '-o', 'Credentials:gs_secret_access_key=secret',
+ 'perfdiag', '-s', '1K',
+ '-o', 'testoutput',
+ '-n', '10',
+ 'gs://bucket1')).AndRaise(
+ error.CmdError("fakecommand", common_lib_utils.CmdResult(),
+ "xxxxxx<Error>yyyyyyyyyy</Error>"))
+
+ self.mox.ReplayAll()
+ self.assertRaisesRegexp(
+ moblab_rpc_interface.BucketPerformanceTestException,
+ '<Error>yyyyyyyyyy',
+ moblab_rpc_interface._run_bucket_performance_test,
+ 'key', 'secret', 'gs://bucket1', '1K', '10', 'testoutput')
+ self.mox.VerifyAll()
+
+
if __name__ == '__main__':
unittest.main()
