[autotest] Create a new bvt-arc suite.
This creates a new, public bvt-arc suite. The suite provides BVT
test coverage for ARC features for all release builders, plus the
Chrome and Android PFQs, and selected paladins.
This suite replaces the arc-bvt-cq suite, which is private, and not
available to non-cheets boards. Its content is otherwise identical.
BUG=chromium:647494
TEST=use suite_enumerator to see that arc-bvt-cq and bvt-arc are equivalent
Change-Id: I3026c0bf328d07d828865cc5237911e7dfa38ad7
Reviewed-on: https://chromium-review.googlesource.com/514432
Commit-Ready: Richard Barnette <jrbarnette@chromium.org>
Tested-by: Richard Barnette <jrbarnette@chromium.org>
Reviewed-by: Ilja H. Friedel <ihf@chromium.org>
diff --git a/client/site_tests/cheets_StartAndroid/control.stress.0 b/client/site_tests/cheets_StartAndroid/control.stress.0
index 32e0158..9b13e6c 100644
--- a/client/site_tests/cheets_StartAndroid/control.stress.0
+++ b/client/site_tests/cheets_StartAndroid/control.stress.0
@@ -6,7 +6,7 @@
AUTHOR = "ARC Team"
# Do not remove any of these suites without wider discussion as it will cause
# instability in the testing lab.
-ATTRIBUTES = "suite:arc-bvt-cq, suite:bvt-cq, suite:bvt-inline"
+ATTRIBUTES = "suite:bvt-arc, suite:arc-bvt-cq, suite:bvt-cq, suite:bvt-inline"
NAME = "cheets_StartAndroid.stress.0"
TIME = "LONG"
TEST_CATEGORY = "Stress"
diff --git a/client/site_tests/desktopui_ExitOnSupervisedUserCrash/control.arc b/client/site_tests/desktopui_ExitOnSupervisedUserCrash/control.arc
index 6a7d989..3b19407 100644
--- a/client/site_tests/desktopui_ExitOnSupervisedUserCrash/control.arc
+++ b/client/site_tests/desktopui_ExitOnSupervisedUserCrash/control.arc
@@ -4,7 +4,7 @@
AUTHOR = "cmasone, antrim"
NAME = "desktopui_ExitOnSupervisedUserCrash.arc"
-ATTRIBUTES = "suite:arc-bvt-cq, suite:bvt-cq"
+ATTRIBUTES = "suite:bvt-arc, suite:arc-bvt-cq, suite:bvt-cq"
TIME = "SHORT"
TEST_CATEGORY = "General"
TEST_CLASS = "desktopui"
diff --git a/client/site_tests/graphics_Idle/control.arc b/client/site_tests/graphics_Idle/control.arc
index b002de2..10f9814 100644
--- a/client/site_tests/graphics_Idle/control.arc
+++ b/client/site_tests/graphics_Idle/control.arc
@@ -4,7 +4,7 @@
AUTHOR = 'chromeos-gfx'
NAME = "graphics_Idle.arc"
-ATTRIBUTES = "suite:arc-bvt-cq, suite:bvt-cq"
+ATTRIBUTES = "suite:bvt-arc, suite:arc-bvt-cq, suite:bvt-cq"
PURPOSE = "Verify that graphics behaves as expected on idle."
CRITERIA = """
This test will fail if we don't see the appropriate GPU idle states.
diff --git a/client/site_tests/security_NetworkListeners/control b/client/site_tests/security_NetworkListeners/control
index d9ac0ea..03cf4c1 100644
--- a/client/site_tests/security_NetworkListeners/control
+++ b/client/site_tests/security_NetworkListeners/control
@@ -13,7 +13,7 @@
Fail if the list of processes listening on the network doesn't match the
baseline
"""
-ATTRIBUTES = "suite:bvt-cq, suite:smoke, suite:arc-bvt-cq"
+ATTRIBUTES = "suite:bvt-cq, suite:smoke, suite:bvt-arc, suite:arc-bvt-cq"
TEST_CLASS = "security"
TEST_CATEGORY = "Functional"
TEST_TYPE = "client"
diff --git a/client/site_tests/telemetry_LoginTest/control.arc b/client/site_tests/telemetry_LoginTest/control.arc
index 2e5a527..4d81921 100644
--- a/client/site_tests/telemetry_LoginTest/control.arc
+++ b/client/site_tests/telemetry_LoginTest/control.arc
@@ -10,7 +10,7 @@
the autotest extension doesn't load/is inaccessible, or if
chrome.autotestPrivate.loginStatus doesn't work.
"""
-ATTRIBUTES = "suite:arc-bvt-cq, suite:bvt-cq"
+ATTRIBUTES = "suite:bvt-arc, suite:arc-bvt-cq, suite:bvt-cq"
TIME = "SHORT"
TEST_CATEGORY = "Functional"
TEST_CLASS = "login"
diff --git a/server/site_tests/cheets_CTS/generate_controlfiles.py b/server/site_tests/cheets_CTS/generate_controlfiles.py
index 443f020..4d4b94a 100755
--- a/server/site_tests/cheets_CTS/generate_controlfiles.py
+++ b/server/site_tests/cheets_CTS/generate_controlfiles.py
@@ -65,11 +65,11 @@
]
# Any test in SMOKE (VMTest) should also be in CQ (HWTest).
-_SUITE_BVT_CQ = _SUITE_SMOKE + [
+_SUITE_BVT_ARC = _SUITE_SMOKE + [
'com.android.cts.dram'
]
-_SUITE_ARC_BVT_CQ = _SUITE_BVT_CQ
+_SUITE_BVT_CQ = _SUITE_BVT_ARC
_SUITE_BVT_PERBUILD = [
'android.signature',
@@ -175,8 +175,8 @@
attributes += ', suite:smoke'
if package in _SUITE_BVT_CQ:
attributes += ', suite:bvt-cq'
- if package in _SUITE_ARC_BVT_CQ:
- attributes += ', suite:arc-bvt-cq'
+ if package in _SUITE_BVT_ARC:
+ attributes += ', suite:bvt-arc, suite:arc-bvt-cq'
if package in _SUITE_BVT_PERBUILD and abi == 'arm':
attributes += ', suite:bvt-perbuild'
# Adding arc-cts-stable runs all packages twice on stable.
@@ -226,7 +226,7 @@
name)
# cheets_CTS internal retries limited due to time constraints on cq.
retry = ''
- if (package in (_SUITE_SMOKE + _SUITE_BVT_CQ + _SUITE_ARC_BVT_CQ) or
+ if (package in (_SUITE_SMOKE + _SUITE_BVT_CQ + _SUITE_BVT_ARC) or
package in _SUITE_BVT_PERBUILD and abi == 'arm'):
retry = '\n max_retry=3,'
return _CONTROLFILE_TEMPLATE.format(
diff --git a/site_utils/attribute_whitelist.txt b/site_utils/attribute_whitelist.txt
index 0e2ca34..300de21 100644
--- a/site_utils/attribute_whitelist.txt
+++ b/site_utils/attribute_whitelist.txt
@@ -1,3 +1,6 @@
+subsystem:default
+subsystem:network
+subsystem:wifi
suite:AFDO_page_replay
suite:AFDO_record
suite:android_bluetooth
@@ -31,14 +34,6 @@
suite:arc-bvt-cq
suite:arc-bvt-perbuild
suite:arc-cts
-suite:arc-cts-dev
-suite:arc-cts-dev-fri
-suite:arc-cts-dev-mon
-suite:arc-cts-dev-sat
-suite:arc-cts-dev-sun
-suite:arc-cts-dev-thu
-suite:arc-cts-dev-tue
-suite:arc-cts-dev-wed
suite:arc-cts-beta
suite:arc-cts-beta-fri
suite:arc-cts-beta-mon
@@ -47,11 +42,18 @@
suite:arc-cts-beta-thu
suite:arc-cts-beta-tue
suite:arc-cts-beta-wed
+suite:arc-cts-dev
+suite:arc-cts-dev-fri
+suite:arc-cts-dev-mon
+suite:arc-cts-dev-sat
+suite:arc-cts-dev-sun
+suite:arc-cts-dev-thu
+suite:arc-cts-dev-tue
+suite:arc-cts-dev-wed
suite:arc-cts-perbuild
suite:arc-cts-qual
suite:arc-cts-stable
suite:arc-cts-terra
-suite:gts
suite:arc-gts
suite:arc-gts-perbuild
suite:arc-gts-tot
@@ -67,6 +69,7 @@
suite:brillo-presubmit
suite:brillo-pts
suite:brillo-smoke
+suite:bvt-arc
suite:bvt-cq
suite:bvt-inline
suite:bvt-perbuild
@@ -88,8 +91,8 @@
suite:chameleon_hdmi_stress
suite:chameleon_vga
suite:cr50_stress_experimental
-suite:crosbolt_arc_perf_perbuild
suite:crosbolt_arc_perf_nightly
+suite:crosbolt_arc_perf_perbuild
suite:crosbolt_perf_nightly
suite:crosbolt_perf_perbuild
suite:dummy
@@ -108,13 +111,13 @@
suite:faft_dev
suite:faft_dev_ryu
suite:faft_ec
+suite:faft_ec3po
suite:faft_ec_au_1
suite:faft_ec_au_2
suite:faft_ec_au_3
suite:faft_ec_ryu
suite:faft_ec_tot
suite:faft_ec_wp
-suite:faft_ec3po
suite:faft_lab
suite:faft_lv1
suite:faft_lv2
@@ -131,6 +134,7 @@
suite:graphics_browser
suite:graphics_per-day
suite:graphics_system
+suite:gts
suite:hotrod
suite:hwqual
suite:jailed_build
@@ -164,10 +168,10 @@
suite:paygen_au_stable
suite:perf_v2
suite:perfalerts
-suite:powerplay
suite:power_build
suite:power_daily
suite:power_requirements
+suite:powerplay
suite:push_to_prod
suite:regression
suite:sanity
@@ -217,6 +221,3 @@
suite:wimax
suite:youtube_mse_eme
suite:youtube_page
-subsystem:default
-subsystem:network
-subsystem:wifi
diff --git a/site_utils/presubmit_hooks/check_control_files.py b/site_utils/presubmit_hooks/check_control_files.py
index f32a679..fd14d11 100755
--- a/site_utils/presubmit_hooks/check_control_files.py
+++ b/site_utils/presubmit_hooks/check_control_files.py
@@ -23,7 +23,7 @@
DEPENDENCY_ARC = 'arc'
-SUITES_NEED_RETRY = set(['bvt-cq', 'bvt-inline', 'arc-bvt-cq'])
+SUITES_NEED_RETRY = set(['bvt-arc', 'bvt-cq', 'bvt-inline', 'arc-bvt-cq'])
TESTS_NEED_ARC = 'cheets_'
diff --git a/test_suites/control.bvt-arc b/test_suites/control.bvt-arc
new file mode 100644
index 0000000..9a4fc78
--- /dev/null
+++ b/test_suites/control.bvt-arc
@@ -0,0 +1,65 @@
+# Copyright 2017 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+AUTHOR = "Chrome OS Team"
+NAME = "bvt-arc"
+PURPOSE = "Test basic functionality of ARC."
+
+TIME = "SHORT"
+TEST_CATEGORY = "General"
+TEST_CLASS = "suite"
+TEST_TYPE = "Server"
+
+DOC = """
+This is the portion of the Build Verification Test suite required
+for devices supporting ARC. Tests in this suite must pass in the
+CQ, and in the Chrome and Android PFQs. Test failures prevent code
+from being included in the tree or in canary builds:
+ * Failures in the canary turn the tree red, block lower priority
+ tests for the build, and generally mean that QA cannot further
+ evaluate the build's fitness for release.
+ * Chrome OS CLs must pass these tests in the Commit Queue prior to
+ being accepted into the tree.
+ * A new Chrome or Android build must pass these tests prior to the
+ build being included in a Chrome OS canary build.
+
+Requirements for a test to be in this suite:
+ 1. The test should be SHORT or MEDIUM, and should not require any
+ specialized lab resources.
+ 2. A test failure should indicate one or more of the following
+ impacts is possible:
+ * The failure may prevent discovery of other ARC bugs.
+ * The failure may block ordinary development tasks for ARC.
+ 3. A test failure must reliably indicate a bug in the product, and
+ not a bug in the test.
+
+@param build: The name of the image to test.
+ Ex: veyron_minnie-release/R60-9575.0.0
+@param board: The board to test on. Ex: veyron_minnie
+@param pool: The pool of machines to utilize for scheduling.
+"""
+
+import common
+from autotest_lib.server.cros import provision
+from autotest_lib.server.cros.dynamic_suite import dynamic_suite
+
+
+# Values specified in this bug template will override default values when
+# filing bugs on tests that are a part of this suite. If left unspecified
+# the bug filer will fallback to its defaults.
+_BUG_TEMPLATE = {
+ 'labels': ['bvt'],
+ 'owner': '',
+ 'status': None,
+ 'summary': None,
+ 'title': None,
+}
+
+args_dict['max_runtime_mins'] = 60
+args_dict['name'] = 'bvt-arc'
+args_dict['job'] = job
+args_dict['version_prefix'] = provision.CROS_VERSION_PREFIX
+args_dict['bug_template'] = _BUG_TEMPLATE
+
+dynamic_suite.reimage_and_run(**args_dict)
