graphics_Drm.py: Change of test framework
1) Combining persistent passing test into one control file
2) Keep other tests in individual control file as before
3) Change graphics_Drm.py to handle running multiple tests at once
BUG=chromium:717664
TEST=test_that on glimmer
Change-Id: I4410fed061f45bdca9c45c400c8b15041a66eb03
Reviewed-on: https://chromium-review.googlesource.com/503436
Commit-Ready: Ilja H. Friedel <ihf@chromium.org>
Tested-by: Ilja H. Friedel <ihf@chromium.org>
Reviewed-by: Ilja H. Friedel <ihf@chromium.org>
diff --git a/client/site_tests/graphics_Drm/control.atomictest b/client/site_tests/graphics_Drm/control
similarity index 82%
rename from client/site_tests/graphics_Drm/control.atomictest
rename to client/site_tests/graphics_Drm/control
index b870a84..4607388 100644
--- a/client/site_tests/graphics_Drm/control.atomictest
+++ b/client/site_tests/graphics_Drm/control
@@ -2,7 +2,7 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
-NAME = 'graphics_Drm.atomictest'
+NAME = 'graphics_Drm'
AUTHOR = 'chromeos-gfx'
PURPOSE = 'Uses drm-tests to sanity check DRM graphics output'
CRITERIA = """
@@ -18,7 +18,7 @@
}
DOC = """
-Runs atomictest.
+Runs various graphics_Drm related tests.
"""
-job.run_test('graphics_Drm', tag='atomictest', cmd='atomictest')
+job.run_test('graphics_Drm')
diff --git a/client/site_tests/graphics_Drm/control.swrast_test b/client/site_tests/graphics_Drm/control.bvt
similarity index 62%
rename from client/site_tests/graphics_Drm/control.swrast_test
rename to client/site_tests/graphics_Drm/control.bvt
index 2d99aef..3d595d7 100644
--- a/client/site_tests/graphics_Drm/control.swrast_test
+++ b/client/site_tests/graphics_Drm/control.bvt
@@ -2,13 +2,13 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
-NAME = 'graphics_Drm.swrast_test'
+NAME = 'graphics_Drm.bvt'
AUTHOR = 'chromeos-gfx'
PURPOSE = 'Uses drm-tests to sanity check DRM graphics output'
CRITERIA = """
Runs drm-tests. All tests must run without error.
"""
-ATTRIBUTES = 'suite:graphics_per-day, suite:drm, suite:graphics, suite:graphics_system, suite:bvt-cq'
+ATTRIBUTES = 'suite:graphics_per-day, suite:drm, suite:graphics, suite:graphics_system, suite:bvt-inline'
TIME='FAST'
TEST_CATEGORY = 'Functional'
TEST_CLASS = "gl"
@@ -18,8 +18,10 @@
}
DOC = """
-Runs swrast_test.
+Runs various graphics_Drm related tests.
"""
-job.run_test('graphics_Drm', tag='swrast_test', cmd='swrast_test',
- stop_ui=True, display_required=False)
+job.run_test('graphics_Drm', tests=['drm_cursor_test',
+ 'linear_bo_test',
+ 'null_platform_test',
+ 'swrast_test'])
diff --git a/client/site_tests/graphics_Drm/control.drm_cursor_test b/client/site_tests/graphics_Drm/control.drm_cursor_test
deleted file mode 100644
index e720c7d..0000000
--- a/client/site_tests/graphics_Drm/control.drm_cursor_test
+++ /dev/null
@@ -1,24 +0,0 @@
-# Copyright 2016 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-NAME = 'graphics_Drm.drm_cursor_test'
-AUTHOR = 'chromeos-gfx'
-PURPOSE = 'Uses drm-tests to sanity check DRM graphics output'
-CRITERIA = """
-Runs drm-tests. All tests must run without error.
-"""
-ATTRIBUTES = 'suite:graphics_per-day, suite:drm, suite:graphics, suite:graphics_system, suite:bvt-cq'
-TIME='FAST'
-TEST_CATEGORY = 'Functional'
-TEST_CLASS = "gl"
-TEST_TYPE = 'client'
-BUG_TEMPLATE = {
- 'components': ['OS>Kernel>Graphics'],
-}
-
-DOC = """
-Runs drm_cursor_test.
-"""
-
-job.run_test('graphics_Drm', tag='drm_cursor_test', cmd='drm_cursor_test')
diff --git a/client/site_tests/graphics_Drm/control.gamma_test b/client/site_tests/graphics_Drm/control.gamma_test
index ded5cf9..b406628 100644
--- a/client/site_tests/graphics_Drm/control.gamma_test
+++ b/client/site_tests/graphics_Drm/control.gamma_test
@@ -8,7 +8,7 @@
CRITERIA = """
Runs drm-tests. All tests must run without error.
"""
-ATTRIBUTES = 'suite:graphics_per-day, suite:drm, suite:graphics, suite:graphics_system'
+ATTRIBUTES = ''
TIME='FAST'
TEST_CATEGORY = 'Functional'
TEST_CLASS = "gl"
@@ -21,4 +21,5 @@
Runs gamma_test.
"""
-job.run_test('graphics_Drm', tag='gamma_test', cmd='gamma_test')
+job.run_test('graphics_Drm', tag='gamma_test',
+ tests=['gamma_test'])
diff --git a/client/site_tests/graphics_Drm/control.linear_bo_test b/client/site_tests/graphics_Drm/control.linear_bo_test
deleted file mode 100644
index cadaa33..0000000
--- a/client/site_tests/graphics_Drm/control.linear_bo_test
+++ /dev/null
@@ -1,24 +0,0 @@
-# Copyright 2016 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-NAME = 'graphics_Drm.linear_bo_test'
-AUTHOR = 'chromeos-gfx'
-PURPOSE = 'Uses drm-tests to sanity check DRM graphics output'
-CRITERIA = """
-Runs drm-tests. All tests must run without error.
-"""
-ATTRIBUTES = 'suite:graphics_per-day, suite:drm, suite:graphics, suite:graphics_system, suite:bvt-cq'
-TIME='FAST'
-TEST_CATEGORY = 'Functional'
-TEST_CLASS = "gl"
-TEST_TYPE = 'client'
-BUG_TEMPLATE = {
- 'components': ['OS>Kernel>Graphics'],
-}
-
-DOC = """
-Runs linear_bo_test.
-"""
-
-job.run_test('graphics_Drm', tag='linear_bo_test', cmd='linear_bo_test')
diff --git a/client/site_tests/graphics_Drm/control.mmap_test b/client/site_tests/graphics_Drm/control.mmap_test
index 5dcc4a0..75d35cb 100644
--- a/client/site_tests/graphics_Drm/control.mmap_test
+++ b/client/site_tests/graphics_Drm/control.mmap_test
@@ -8,7 +8,7 @@
CRITERIA = """
Runs drm-tests. All tests must run without error.
"""
-ATTRIBUTES = 'suite:graphics_per-day, suite:drm, suite:graphics, suite:graphics_system'
+ATTRIBUTES = ''
TIME='FAST'
TEST_CATEGORY = 'Functional'
TEST_CLASS = "gl"
@@ -21,5 +21,5 @@
Runs vgem_fb_test.
"""
-job.run_test('graphics_Drm', tag='mmap_test', cmd='mmap_test', opts = args +
- ['--use_vgem'])
+job.run_test('graphics_Drm', tag='mmap_test',
+ tests=['mmap_test'])
diff --git a/client/site_tests/graphics_Drm/control.null_platform_test b/client/site_tests/graphics_Drm/control.null_platform_test
deleted file mode 100644
index 3b1b311..0000000
--- a/client/site_tests/graphics_Drm/control.null_platform_test
+++ /dev/null
@@ -1,24 +0,0 @@
-# Copyright 2016 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-NAME = 'graphics_Drm.null_platform_test'
-AUTHOR = 'chromeos-gfx'
-PURPOSE = 'Uses drm-tests to sanity check DRM graphics output'
-CRITERIA = """
-Runs drm-tests. All tests must run without error.
-"""
-ATTRIBUTES = 'suite:graphics_per-day, suite:drm, suite:graphics, suite:graphics_system, suite:bvt-cq'
-TIME='FAST'
-TEST_CATEGORY = 'Functional'
-TEST_CLASS = "gl"
-TEST_TYPE = 'client'
-BUG_TEMPLATE = {
- 'components': ['OS>Kernel>Graphics'],
-}
-
-DOC = """
-Runs null_platform_test.
-"""
-
-job.run_test('graphics_Drm', tag='null_platform_test', cmd='null_platform_test')
diff --git a/client/site_tests/graphics_Drm/control.vgem_test b/client/site_tests/graphics_Drm/control.vgem_test
index 7cadba5..8ae3750 100644
--- a/client/site_tests/graphics_Drm/control.vgem_test
+++ b/client/site_tests/graphics_Drm/control.vgem_test
@@ -8,7 +8,7 @@
CRITERIA = """
Runs drm-tests. All tests must run without error.
"""
-ATTRIBUTES = 'suite:graphics_per-day, suite:drm, suite:graphics, suite:graphics_system'
+ATTRIBUTES = ''
TIME='FAST'
TEST_CATEGORY = 'Functional'
TEST_CLASS = "gl"
@@ -21,5 +21,5 @@
Runs vgem_test.
"""
-job.run_test('graphics_Drm', tag='vgem_test', cmd='vgem_test',
- stop_ui=True, display_required=False)
+job.run_test('graphics_Drm', tag='vgem_test',
+ tests=['vgem_test'])
diff --git a/client/site_tests/graphics_Drm/control.atomictest b/client/site_tests/graphics_Drm/control.vk_glow
similarity index 70%
copy from client/site_tests/graphics_Drm/control.atomictest
copy to client/site_tests/graphics_Drm/control.vk_glow
index b870a84..63bcc74 100644
--- a/client/site_tests/graphics_Drm/control.atomictest
+++ b/client/site_tests/graphics_Drm/control.vk_glow
@@ -2,13 +2,13 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
-NAME = 'graphics_Drm.atomictest'
+NAME = 'graphics_Drm.vk_glow'
AUTHOR = 'chromeos-gfx'
PURPOSE = 'Uses drm-tests to sanity check DRM graphics output'
CRITERIA = """
Runs drm-tests. All tests must run without error.
"""
-ATTRIBUTES = 'suite:graphics_per-day, suite:drm, suite:graphics, suite:graphics_system'
+ATTRIBUTES = ''
TIME='FAST'
TEST_CATEGORY = 'Functional'
TEST_CLASS = "gl"
@@ -18,7 +18,8 @@
}
DOC = """
-Runs atomictest.
+Runs vk_glow.
"""
-job.run_test('graphics_Drm', tag='atomictest', cmd='atomictest')
+job.run_test('graphics_Drm', tag='vk_glow',
+ tests=['vk_glow'])
diff --git a/client/site_tests/graphics_Drm/control.vk_glow_test b/client/site_tests/graphics_Drm/control.vk_glow_test
deleted file mode 100644
index 1fc6cf1..0000000
--- a/client/site_tests/graphics_Drm/control.vk_glow_test
+++ /dev/null
@@ -1,25 +0,0 @@
-# Copyright 2016 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-NAME = 'graphics_Drm.vk_glow_test'
-AUTHOR = 'chromeos-gfx'
-PURPOSE = 'Uses drm-tests to sanity check DRM graphics output'
-CRITERIA = """
-Runs drm-tests. All tests must run without error.
-"""
-ATTRIBUTES = 'suite:graphics_per-day, suite:drm, suite:graphics, suite:graphics_system'
-TIME='FAST'
-TEST_CATEGORY = 'Functional'
-TEST_CLASS = "gl"
-TEST_TYPE = 'client'
-BUG_TEMPLATE = {
- 'components': ['OS>Kernel>Graphics'],
-}
-
-DOC = """
-Runs vk_glow.
-"""
-
-job.run_test('graphics_Drm', tag='vk_glow', cmd='vk_glow',
- stop_ui=True, display_required=True, vulkan_required=True)
diff --git a/client/site_tests/graphics_Drm/graphics_Drm.py b/client/site_tests/graphics_Drm/graphics_Drm.py
index a4b331f..7099d44 100644
--- a/client/site_tests/graphics_Drm/graphics_Drm.py
+++ b/client/site_tests/graphics_Drm/graphics_Drm.py
@@ -10,65 +10,127 @@
from autotest_lib.client.cros.graphics import graphics_utils
-class graphics_Drm(test.test):
- """Runs one of the drm-tests.
- """
- version = 1
- GSC = None
- _services = None
- _timeout = 120
+class DrmTest(object):
+ # Default argument dictionary
+ _command = None
- def initialize(self):
- self.GSC = graphics_utils.GraphicsStateChecker()
- self._supported_apis = graphics_utils.GraphicsApiHelper().get_supported_apis()
- self._services = service_stopper.ServiceStopper(['ui'])
-
- def cleanup(self):
- if self.GSC:
- self.GSC.finalize()
- if self._services:
- self._services.restore_services()
-
- def run_once(self, cmd, **kargs):
- opts = {
- 'stop_ui': True,
+ def __init__(self, command, **kargs):
+ self._opts = {
+ 'timeout': 20,
'display_required': True,
'vulkan_required': False,
'min_kernel_version': None
}
- opts.update(kargs)
+ self._opts.update(kargs)
+ self._command = command
+ def can_run(self):
+ """Indicate if the test can be run on the configuration."""
num_displays = graphics_utils.get_num_outputs_on()
- # Sanity check to guard against incorrect silent passes.
if num_displays == 0 and utils.get_device_type() == 'CHROMEBOOK':
- raise error.TestFail('Error: found Chromebook without display.')
- if opts['display_required'] and num_displays == 0:
+ # Sanity check to guard against incorrect silent passes.
+ logging.error('Error: found Chromebook without display.')
+ return False
+ return True
+
+ def should_run(self):
+ """Indicate if the test should be run on current configuration."""
+ supported_apis = graphics_utils.GraphicsApiHelper().get_supported_apis()
+ num_displays = graphics_utils.get_num_outputs_on()
+ if num_displays == 0 and self._opts['display_required']:
# If a test needs a display and we don't have a display,
# consider it a pass.
logging.warning('No display connected, skipping test.')
- return
- if opts['vulkan_required'] and 'vk' not in self._supported_apis:
+ return False
+ if self._opts['vulkan_required'] and 'vk' not in supported_apis:
# If a test needs vulkan to run and we don't have it,
# consider it a pass
- logging.warning('Vulkan is required by test but is not available '
- 'on system. Skipping test.')
- return
+ logging.warning('Vulkan is required by test but is not '
+ 'available on system. Skipping test.')
+ return False
+ return True
- if opts['stop_ui']:
- self._services.stop_services()
+ def run(self):
try:
- result = utils.run(cmd,
- timeout=self._timeout,
- ignore_status=True,
- stderr_is_expected=True,
- verbose=True,
- stdout_tee=utils.TEE_TO_LOGS,
- stderr_tee=utils.TEE_TO_LOGS)
- except Exception:
- # Fail on exceptions.
- raise error.TestFail('Failed: Exception running %s' % cmd)
+ # TODO(pwang): consider TEE to another file if drmtests keep
+ # spewing so much output.
+ utils.run(
+ self._command,
+ timeout=self._opts['timeout'],
+ stderr_is_expected=True,
+ verbose=True,
+ stdout_tee=utils.TEE_TO_LOGS,
+ stderr_tee=utils.TEE_TO_LOGS
+ )
+ except error.CmdTimeoutError as e:
+ logging.error('Failed: Timeout while running %s' % self._command)
+ logging.debug(e)
+ return False
+ except error.CmdError as e:
+ logging.error('Failed: %s (exit=%d)' % (self._command,
+ e.result_obj.exit_status))
+ logging.debug(e)
+ return False
+ except Exception as e:
+ logging.error('Failed: Unexpected exception while running %s'
+ % self._command)
+ logging.debug(e)
+ return False
+ logging.info('Passed: %s', self._command)
+ return True
- # Last but not least check return code and use it for triage.
- if result.exit_status != 0:
- raise error.TestFail('Failed: %s (exit=%d)' %
- (cmd, result.exit_status))
+
+drm_tests = {
+ 'atomictest': DrmTest('atomictest'),
+ 'drm_cursor_test': DrmTest('drm_cursor_test'),
+ 'gamma_test': DrmTest('gamma_test'),
+ 'linear_bo_test': DrmTest('linear_bo_test'),
+ 'mmap_test': DrmTest('mmap_test'),
+ 'null_platform_test': DrmTest('null_platform_test'),
+ 'swrast_test': DrmTest('swrast_test', display_required=False),
+ 'vgem_test': DrmTest('vgem_test', display_required=False),
+ 'vk_glow': DrmTest('vk_glow', vulkan_required=True),
+}
+
+
+class graphics_Drm(test.test):
+ """Runs one, several or all of the drm-tests."""
+ version = 1
+ _services = None
+ _GSC = None
+
+ def initialize(self):
+ self._failures = []
+ self._GSC = graphics_utils.GraphicsStateChecker()
+ self._services = service_stopper.ServiceStopper(['ui'])
+ self._services.stop_services()
+
+ def cleanup(self):
+ if self._services:
+ self._services.restore_services()
+ if self._GSC:
+ self._GSC.finalize()
+
+ # graphics_Drm runs all available tests if tests = None.
+ def run_once(self, tests=None):
+ for test_name in drm_tests:
+ if tests and test_name not in tests:
+ continue
+
+ test = drm_tests.get(test_name)
+ logging.info('-----------------[%s]-----------------' % test_name)
+ if test.should_run():
+ if test.can_run():
+ logging.debug('Running test %s.', test_name)
+ passed = test.run()
+ if not passed:
+ self._failures.append(test_name)
+ else:
+ logging.info('Failed: test %s can not be run on current '
+ 'configurations.' % test_name)
+ self._failures.append(test_name)
+ else:
+ logging.info('Skipping test: %s.' % test_name)
+
+ if self._failures:
+ raise error.TestFail('Failed: %s' % self._failures)
