autotest: add board_lists for suite_scheduler
This CL adds board_lists for suite_scheduler, to let CrOS tasks re-use
board whitelist by specifying a pre-defined list in option 'boards:'.
BUG=chromium:717074
TEST=Run driver_unittest & task_unittest.
Run ./suite_scheduler.py --sanity.
Run ./suite_scheduler.py -d /usr/local/autotest/logs -f
xixuan_test_1.ini -r /tmp/_autotmp__suite_scheduler
Change-Id: I2c7811cf8041834e5d5779c373fea425577374fe
Reviewed-on: https://chromium-review.googlesource.com/494887
Commit-Ready: Xixuan Wu <xixuan@chromium.org>
Tested-by: Xixuan Wu <xixuan@chromium.org>
Reviewed-by: Xixuan Wu <xixuan@chromium.org>
diff --git a/site_utils/suite_scheduler/driver.py b/site_utils/suite_scheduler/driver.py
index e41e8e2..3dcc964 100644
--- a/site_utils/suite_scheduler/driver.py
+++ b/site_utils/suite_scheduler/driver.py
@@ -22,6 +22,9 @@
POOL_SIZE = 32
+BOARD_WHITELIST_SECTION = 'board_lists'
+PRE_SECTIONS = [BOARD_WHITELIST_SECTION]
+
class Driver(object):
"""Implements the main loop of the suite_scheduler.
@@ -76,6 +79,26 @@
self._events = self._CreateEventsWithTasks(config, mv)
+ def _ReadBoardWhitelist(self, config):
+ """Read board whitelist from config and save as dict.
+
+ @param config: an instance of ForgivingConfigParser.
+ """
+ board_lists = {}
+ if BOARD_WHITELIST_SECTION not in config.sections():
+ return board_lists
+
+ for option in config.options(BOARD_WHITELIST_SECTION):
+ if option in board_lists:
+ raise task.MalformedConfigEntry(
+ 'Board list name must be unique.')
+ else:
+ board_lists[option] = config.getstring(
+ BOARD_WHITELIST_SECTION, option)
+
+ return board_lists
+
+
def _CreateEventsWithTasks(self, config, mv):
"""Create task lists from config, and assign to newly-minted events.
@@ -109,12 +132,14 @@
@return dict of {event_keyword: [tasks]} mappings.
@raise MalformedConfigEntry on a task parsing error.
"""
+ board_lists = self._ReadBoardWhitelist(config)
tasks = {}
for section in config.sections():
- if not base_event.HonoredSection(section):
+ if (not base_event.HonoredSection(section) and
+ section not in PRE_SECTIONS):
try:
keyword, new_task = task.Task.CreateFromConfigSection(
- config, section)
+ config, section, board_lists=board_lists)
except task.MalformedConfigEntry as e:
logging.warning('%s is malformed: %s', section, str(e))
continue
diff --git a/site_utils/suite_scheduler/driver_unittest.py b/site_utils/suite_scheduler/driver_unittest.py
index 0173b74..9640597 100644
--- a/site_utils/suite_scheduler/driver_unittest.py
+++ b/site_utils/suite_scheduler/driver_unittest.py
@@ -65,15 +65,24 @@
return [mock_nightly, mock_weekly, mock_new_build]
+ def _ExpectBoardListConfig(self):
+ board_lists = 'sub_board1,sub_board2'
+ self.config.add_section(driver.BOARD_WHITELIST_SECTION)
+ self.config.set(driver.BOARD_WHITELIST_SECTION,
+ self._BOARDS[0], board_lists)
+
+
def _ExpectTaskConfig(self):
self.config.add_section(timed_event.Nightly.KEYWORD)
self.config.add_section(timed_event.Weekly.KEYWORD)
self.mox.StubOutWithMock(task.Task, 'CreateFromConfigSection')
task.Task.CreateFromConfigSection(
- self.config, timed_event.Nightly.KEYWORD).InAnyOrder().AndReturn(
+ self.config, timed_event.Nightly.KEYWORD,
+ board_lists=mox.IgnoreArg()).InAnyOrder().AndReturn(
(timed_event.Nightly.KEYWORD, self.nightly_bvt))
task.Task.CreateFromConfigSection(
- self.config, timed_event.Weekly.KEYWORD).InAnyOrder().AndReturn(
+ self.config, timed_event.Weekly.KEYWORD,
+ board_lists=mox.IgnoreArg()).InAnyOrder().AndReturn(
(timed_event.Weekly.KEYWORD, self.weekly_bvt))
@@ -125,6 +134,7 @@
def testTasksFromConfig(self):
"""Test that we can build a list of Tasks from a config."""
self._ExpectTaskConfig()
+ self._ExpectBoardListConfig()
self.mox.ReplayAll()
tasks = self.driver.TasksFromConfig(self.config)
self.assertTrue(self.nightly_bvt in tasks[timed_event.Nightly.KEYWORD])
@@ -135,6 +145,7 @@
"""Test that we can build a list of Tasks from a config twice."""
events = self._ExpectSetup()
self._ExpectTaskConfig()
+ self._ExpectBoardListConfig()
self.mox.ReplayAll()
self.driver.SetUpEventsAndTasks(self.config, self.mv)
@@ -153,7 +164,8 @@
self._ExpectSetup()
self.mox.StubOutWithMock(task.Task, 'CreateFromConfigSection')
task.Task.CreateFromConfigSection(
- self.config, timed_event.Nightly.KEYWORD).InAnyOrder().AndReturn(
+ self.config, timed_event.Nightly.KEYWORD,
+ board_lists=mox.IgnoreArg()).InAnyOrder().AndReturn(
(timed_event.Nightly.KEYWORD, self.nightly_bvt))
self.mox.ReplayAll()
self.driver.SetUpEventsAndTasks(self.config, self.mv)
@@ -167,6 +179,7 @@
def testHandleAllEventsOnce(self):
"""Test that all events being ready is handled correctly."""
events = self._ExpectSetup()
+ self._ExpectBoardListConfig()
self._ExpectEnumeration()
launch_control_build_called = False
for event in events:
@@ -182,6 +195,7 @@
def testHandleNightlyEventOnce(self):
"""Test that one ready event is handled correctly."""
events = self._ExpectSetup()
+ self._ExpectBoardListConfig()
self._ExpectEnumeration()
for event in events:
if event.keyword == timed_event.Nightly.KEYWORD:
@@ -198,6 +212,7 @@
def testForceOnceForBuild(self):
"""Test that one event being forced is handled correctly."""
events = self._ExpectSetup()
+ self._ExpectBoardListConfig()
board = 'board'
type = 'release'
diff --git a/site_utils/suite_scheduler/task.py b/site_utils/suite_scheduler/task.py
index 0036a1d..325809c 100644
--- a/site_utils/suite_scheduler/task.py
+++ b/site_utils/suite_scheduler/task.py
@@ -148,7 +148,7 @@
@staticmethod
- def CreateFromConfigSection(config, section):
+ def CreateFromConfigSection(config, section, board_lists={}):
"""Create a Task from a section of a config file.
The section to parse should look like this:
@@ -174,6 +174,7 @@
@param config: a ForgivingConfigParser.
@param section: the section to parse into a Task.
+ @param board_lists: a dict including all board whitelist for tasks.
@return keyword, Task object pair. One or both will be None on error.
@raise MalformedConfigEntry if there's a problem parsing |section|.
"""
@@ -299,6 +300,20 @@
board_name, board_name)
boards += '%s,' % board_name
boards = boards.strip(',')
+ elif os_type == OS_TYPE_CROS:
+ if board_lists:
+ if boards not in board_lists:
+ logging.debug(
+ 'The board_list name %s does not exist in '
+ 'section board_lists in config.', boards)
+ # TODO(xixuan): Raise MalformedConfigEntry when a CrOS task
+ # specify a 'boards' which is not defined in board_lists.
+ # Currently exception won't be raised to make sure suite
+ # scheduler keeps running when developers are in the middle
+ # of migrating boards.
+ else:
+ boards = board_lists[boards]
+
return keyword, Task(section, suite, specs, pool, num, boards,
priority, timeout,
diff --git a/site_utils/suite_scheduler/task_unittest.py b/site_utils/suite_scheduler/task_unittest.py
index 8dc802f..0ffa047 100644
--- a/site_utils/suite_scheduler/task_unittest.py
+++ b/site_utils/suite_scheduler/task_unittest.py
@@ -81,6 +81,28 @@
self.assertFalse(new_task._FitsSpec('12'))
+ def testCreateFromConfigCheckBoards(self):
+ """Ensure a CrOS Task can be built from a correct config boards."""
+ board_whitelist = 'board2,board3'
+ board_lists = {self._BOARD: board_whitelist}
+ keyword, new_task = task.Task.CreateFromConfigSection(
+ self.config, self._TASK_NAME, board_lists=board_lists)
+ self.assertEquals(keyword, self._EVENT_KEY)
+ self.assertEquals(new_task.boards,
+ set([x.strip() for x in board_whitelist.split(',')]))
+
+
+ def testCreateFromConfigCheckNonExistBoards(self):
+ """Ensure a CrOS Task can be built if board_list is not specified."""
+ board_whitelist = 'board2,board3'
+ board_lists = {'test-%s' % self._BOARD: board_whitelist}
+ keyword, new_task = task.Task.CreateFromConfigSection(
+ self.config, self._TASK_NAME, board_lists=board_lists)
+ self.assertEquals(keyword, self._EVENT_KEY)
+ self.assertEquals(new_task.boards,
+ set([x.strip() for x in self._BOARD.split(',')]))
+
+
def testCreateFromConfigEqualBranch(self):
"""Ensure a Task can be built from a correct config with support of
branch_specs: ==RXX."""
