commit a71d9e8c5deb9bc7b94fa10152097270047b54cb
author Prathmesh Prabhu <pprabhu@chromium.org> Mon Jun 05 19:14:28 2017 -0700
committer chrome-bot <chrome-bot@chromium.org> Fri Jun 30 16:02:48 2017 -0700
tree 16802f2315d6730fdcfd152848cab8084b94065a
parent aa353b340b8dec69aee2affc1646ace0dceed9a0

[autotest] Use process pools to read dark mark in drone_utility.

This CL uses a multiprocessing.Pool for another slow operation executed
by drone_utility in every scheduler tick.

BUG=chromium:718181
TEST=[1] unittests
     [2] Test run on local autotest instance.
     [3] moblab_RunSuite.

Change-Id: Ic3eb7856690c0f1c352fdf149bb4e79dcf77bc01
Reviewed-on: https://chromium-review.googlesource.com/525076
Commit-Ready: Prathmesh Prabhu <pprabhu@chromium.org>
Tested-by: Prathmesh Prabhu <pprabhu@chromium.org>
Reviewed-by: Xixuan Wu <xixuan@chromium.org>

scheduler/drone_utility.py

diff --git a/scheduler/drone_utility.py b/scheduler/drone_utility.py
index a71597a..9df8a02 100755
--- a/scheduler/drone_utility.py
+++ b/scheduler/drone_utility.py
@@ -525,8 +525,8 @@
         autoserv_processes, extra_warnings = self._filter_proc_infos(
                 proc_infos, 'autoserv')
         warnings += extra_warnings
-        parse_processes, extra_warnings = self._filter_proc_infos(
-                proc_infos, 'parse')
+        parse_processes, extra_warnings = self._filter_proc_infos(proc_infos,
+                                                                  'parse')
         warnings += extra_warnings
         site_parse_processes, extra_warnings = self._filter_proc_infos(
                 proc_infos, 'site_parse')
@@ -576,15 +576,24 @@
         if not self._check_mark:
             return proc_infos, []
 
+        if self._use_pool:
+            dark_marks = self._pool.map(
+                    _process_has_dark_mark,
+                    [info['pid'] for info in proc_infos]
+            )
+        else:
+            dark_marks = [_process_has_dark_mark(info['pid'])
+                          for info in proc_infos]
+
         marked_proc_infos = []
-        unmarked_proc_infos = []
-        for info in proc_infos:
-            if _process_has_dark_mark(info['pid']):
+        warnings = []
+        for marked, info in itertools.izip(dark_marks, proc_infos):
+            if marked:
                 marked_proc_infos.append(info)
             else:
-                unmarked_proc_infos.append(info)
-        warnings = ['%(comm)s process pid %(pid)s has no dark mark; ignoring.' %
-                    info for info in unmarked_proc_infos]
+                warnings.append(
+                        '%(comm)s process pid %(pid)s has no dark mark; '
+                        'ignoring.' % info)
         return marked_proc_infos, warnings
 
 
@@ -625,7 +634,6 @@
 def return_data(data):
     print pickle.dumps(data)
 
-
 def _process_has_dark_mark(pid):
     """Checks if a process was launched earlier by drone_utility.
 

scheduler/drone_utility_unittest.py

diff --git a/scheduler/drone_utility_unittest.py b/scheduler/drone_utility_unittest.py
index 83ebc3d..7a0c38c 100755
--- a/scheduler/drone_utility_unittest.py
+++ b/scheduler/drone_utility_unittest.py
@@ -112,7 +112,9 @@
     def test_respect_dark_mark(self):
         """When check_mark=True, dark mark check is performed and respected.
 
-        Only filtered processes with dark mark should be returned.
+        Only filtered processes with dark mark should be returned. We only test
+        this with use_pool=False because mocking out _process_has_dark_mark with
+        multiprocessing.Pool is hard.
         """
         self.maxDiff = None
         process_refresher = drone_utility.ProcessRefresher(check_mark=True)