Merging 18 commit(s) from Chromium's toolchain-utils am: 0ae38c8498 am: 2a19d36a82 am: e617e3393d
Change-Id: I3f25c7ee034b2e20e37ed941b8eae24eec7043eb
diff --git a/afdo_metadata/kernel_afdo.json b/afdo_metadata/kernel_afdo.json
index 967c348..751af04 100644
--- a/afdo_metadata/kernel_afdo.json
+++ b/afdo_metadata/kernel_afdo.json
@@ -3,9 +3,9 @@
"name": "R81-12607.58-1578524353"
},
"chromeos-kernel-4_4": {
- "name": "R81-12861.0-1580726347"
+ "name": "R82-12874.0-1581330812"
},
"chromeos-kernel-3_18": {
- "name": "R81-12861.0-1580725953"
+ "name": "R82-12861.0-1580725956"
}
}
\ No newline at end of file
diff --git a/afdo_redaction/redact_profile.py b/afdo_redaction/redact_profile.py
index 96375fe..02bae92 100755
--- a/afdo_redaction/redact_profile.py
+++ b/afdo_redaction/redact_profile.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2018 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -30,6 +30,7 @@
import re
import sys
+
def _count_samples(samples):
"""Count the total number of samples in a function."""
line_re = re.compile(r'^(\s*)\d+(?:\.\d+)?: (\d+)\s*$')
@@ -132,7 +133,7 @@
continue
if line[0].isspace():
- assert function_line is not None, "sample exists outside of a function?"
+ assert function_line is not None, 'sample exists outside of a function?'
samples.append(line)
continue
@@ -170,12 +171,13 @@
counts[_normalize_samples(record.samples)].append(record)
# Be sure that we didn't see any duplicate functions, since that's bad...
- total_functions_recorded = sum(len(records)
- for records in counts.itervalues())
+ total_functions_recorded = sum(len(records) for records in counts.values())
- unique_function_names = set(record.function_line.split(':')[0]
- for records in counts.itervalues()
- for record in records)
+ unique_function_names = {
+ record.function_line.split(':')[0]
+ for records in counts.values()
+ for record in records
+ }
assert len(unique_function_names) == total_functions_recorded, \
'duplicate function names?'
@@ -187,7 +189,7 @@
num_samples_total = 0
num_top_samples_total = 0
- for normalized_samples, records in counts.iteritems():
+ for normalized_samples, records in counts.items():
top_sample_count, all_sample_count = _count_samples(normalized_samples)
top_sample_count *= len(records)
all_sample_count *= len(records)
@@ -205,11 +207,13 @@
for record in records:
yield record
- print('Retained {:,}/{:,} functions'.format(num_kept, num_total),
- file=summary_file)
- print('Retained {:,}/{:,} samples, total'.format(num_samples_kept,
- num_samples_total),
- file=summary_file)
+ print(
+ 'Retained {:,}/{:,} functions'.format(num_kept, num_total),
+ file=summary_file)
+ print(
+ 'Retained {:,}/{:,} samples, total'.format(num_samples_kept,
+ num_samples_total),
+ file=summary_file)
print('Retained {:,}/{:,} top-level samples' \
.format(num_top_samples_kept, num_top_samples_total),
file=summary_file)
@@ -220,15 +224,17 @@
# Sort this so we get deterministic output. AFDO doesn't care what order it's
# in.
- deduped = sorted(dedup_records(profile_records, summary_output_file),
- key=lambda r: r.function_line)
+ deduped = sorted(
+ dedup_records(profile_records, summary_output_file),
+ key=lambda r: r.function_line)
for function_line, samples in deduped:
print(function_line, file=profile_output_file)
print('\n'.join(samples), file=profile_output_file)
def _main():
- run(profile_input_file=sys.stdin, summary_output_file=sys.stderr,
+ run(profile_input_file=sys.stdin,
+ summary_output_file=sys.stderr,
profile_output_file=sys.stdout)
diff --git a/afdo_redaction/redact_profile_test.py b/afdo_redaction/redact_profile_test.py
index 27fb534..e243897 100755
--- a/afdo_redaction/redact_profile_test.py
+++ b/afdo_redaction/redact_profile_test.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2018 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -8,12 +8,12 @@
from __future__ import division, print_function
-import StringIO
+import io
import unittest
-import redact_profile
+from afdo_redaction import redact_profile
-_redact_limit = redact_profile.dedup_records.func_defaults[0]
+_redact_limit = redact_profile.dedup_records.__defaults__[0]
def _redact(input_lines, summary_to=None):
@@ -21,17 +21,18 @@
input_lines = input_lines.splitlines()
if summary_to is None:
- summary_to = StringIO.StringIO()
+ summary_to = io.StringIO()
- output_to = StringIO.StringIO()
- redact_profile.run(profile_input_file=input_lines,
- summary_output_file=summary_to,
- profile_output_file=output_to)
+ output_to = io.StringIO()
+ redact_profile.run(
+ profile_input_file=input_lines,
+ summary_output_file=summary_to,
+ profile_output_file=output_to)
return output_to.getvalue()
def _redact_with_summary(input_lines):
- summary = StringIO.StringIO()
+ summary = io.StringIO()
result = _redact(input_lines, summary_to=summary)
return result, summary.getvalue()
@@ -64,6 +65,7 @@
class Tests(unittest.TestCase):
"""All of our tests for redact_profile."""
+
def test_no_input_works(self):
self.assertEqual(_redact(''), '')
@@ -93,13 +95,13 @@
result_file = '\n'.join(kept_lines) + '\n'
- lines = _generate_repeated_function_body(_redact_limit,
- fn_name='_discard_me')
+ lines = _generate_repeated_function_body(
+ _redact_limit, fn_name='_discard_me')
self.assertEqual(_redact(kept_lines + lines), result_file)
self.assertEqual(_redact(lines + kept_lines), result_file)
- more_lines = _generate_repeated_function_body(_redact_limit,
- fn_name='_and_discard_me')
+ more_lines = _generate_repeated_function_body(
+ _redact_limit, fn_name='_and_discard_me')
self.assertEqual(_redact(lines + kept_lines + more_lines), result_file)
self.assertEqual(_redact(lines + more_lines), '')
diff --git a/afdo_redaction/remove_indirect_calls.py b/afdo_redaction/remove_indirect_calls.py
index b879b2f..0dc1507 100755
--- a/afdo_redaction/remove_indirect_calls.py
+++ b/afdo_redaction/remove_indirect_calls.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -21,7 +21,6 @@
import argparse
import re
-import sys
def _remove_indirect_call_targets(lines):
diff --git a/afdo_redaction/remove_indirect_calls_test.py b/afdo_redaction/remove_indirect_calls_test.py
index 1499af2..164b284 100755
--- a/afdo_redaction/remove_indirect_calls_test.py
+++ b/afdo_redaction/remove_indirect_calls_test.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -11,12 +11,12 @@
import io
import unittest
-import remove_indirect_calls
+from afdo_redaction import remove_indirect_calls
def _run_test(input_lines):
- input_buf = io.BytesIO('\n'.join(input_lines))
- output_buf = io.BytesIO()
+ input_buf = io.StringIO('\n'.join(input_lines))
+ output_buf = io.StringIO()
remove_indirect_calls.run(input_buf, output_buf)
return output_buf.getvalue().splitlines()
diff --git a/afdo_tools/bisection/afdo_prof_analysis.py b/afdo_tools/bisection/afdo_prof_analysis.py
index 3653110..94e5366 100755
--- a/afdo_tools/bisection/afdo_prof_analysis.py
+++ b/afdo_tools/bisection/afdo_prof_analysis.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -109,7 +109,7 @@
self.state_file)
return
- with open(self.state_file) as f:
+ with open(self.state_file, encoding='utf-8') as f:
try:
data = json.load(f)
except:
@@ -127,7 +127,7 @@
def save_state(self):
state = {'seed': self.seed, 'accumulated_results': self.accumulated_results}
tmp_file = self.state_file + '.new'
- with open(tmp_file, 'w') as f:
+ with open(tmp_file, 'w', encoding='utf-8') as f:
json.dump(state, f, indent=2)
os.rename(tmp_file, self.state_file)
logging.info('Logged state to %s...', self.state_file)
@@ -270,7 +270,7 @@
def find_upper_border(good_copy, funcs, lo, hi, last_bad_val=None):
"""Finds the upper border of problematic range."""
mid = average(lo, hi)
- if mid == lo or mid == hi:
+ if mid in (lo, hi):
return last_bad_val or hi
for func in funcs[lo:mid]:
@@ -288,7 +288,7 @@
def find_lower_border(good_copy, funcs, lo, hi, last_bad_val=None):
"""Finds the lower border of problematic range."""
mid = average(lo, hi)
- if mid == lo or mid == hi:
+ if mid in (lo, hi):
return last_bad_val or lo
for func in funcs[lo:mid]:
@@ -428,7 +428,7 @@
'good_only_functions': gnb_result,
'bad_only_functions': bng_result
}
- with open(flags.analysis_output_file, 'wb') as f:
+ with open(flags.analysis_output_file, 'w', encoding='utf-8') as f:
json.dump(results, f, indent=2)
if flags.remove_state_on_completion:
os.remove(flags.state_file)
diff --git a/afdo_tools/bisection/afdo_prof_analysis_e2e_test.py b/afdo_tools/bisection/afdo_prof_analysis_e2e_test.py
index 85c1c17..b293b8a 100755
--- a/afdo_tools/bisection/afdo_prof_analysis_e2e_test.py
+++ b/afdo_tools/bisection/afdo_prof_analysis_e2e_test.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -15,13 +15,13 @@
import unittest
from datetime import date
-import afdo_prof_analysis as analysis
+from afdo_tools.bisection import afdo_prof_analysis as analysis
class ObjectWithFields(object):
"""Turns kwargs given to the constructor into fields on an object.
- Example usage:
+ Examples:
x = ObjectWithFields(a=1, b=2)
assert x.a == 1
assert x.b == 2
diff --git a/afdo_tools/bisection/afdo_prof_analysis_test.py b/afdo_tools/bisection/afdo_prof_analysis_test.py
index 7bd3050..245edc3 100755
--- a/afdo_tools/bisection/afdo_prof_analysis_test.py
+++ b/afdo_tools/bisection/afdo_prof_analysis_test.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -8,12 +8,12 @@
from __future__ import print_function
-import afdo_prof_analysis as analysis
-
import random
-import StringIO
+import io
import unittest
+from afdo_tools.bisection import afdo_prof_analysis as analysis
+
class AfdoProfAnalysisTest(unittest.TestCase):
"""Class for testing AFDO Profile Analysis"""
@@ -33,17 +33,17 @@
analysis.random.seed(5) # 5 is an arbitrary choice. For consistent testing
def test_text_to_json(self):
- test_data = StringIO.StringIO('deflate_slow:87460059:3\n'
- ' 3: 24\n'
- ' 14: 54767\n'
- ' 15: 664 fill_window:22\n'
- ' 16: 661\n'
- ' 19: 637\n'
- ' 41: 36692 longest_match:36863\n'
- ' 44: 36692\n'
- ' 44.2: 5861\n'
- ' 46: 13942\n'
- ' 46.1: 14003\n')
+ test_data = io.StringIO('deflate_slow:87460059:3\n'
+ ' 3: 24\n'
+ ' 14: 54767\n'
+ ' 15: 664 fill_window:22\n'
+ ' 16: 661\n'
+ ' 19: 637\n'
+ ' 41: 36692 longest_match:36863\n'
+ ' 44: 36692\n'
+ ' 44.2: 5861\n'
+ ' 46: 13942\n'
+ ' 46.1: 14003\n')
expected = {
'deflate_slow': ':87460059:3\n'
' 3: 24\n'
@@ -115,7 +115,7 @@
self.bad_items, common_funcs, 0,
len(common_funcs))
- self.assertEquals(['func_a', 'func_b'], problem_range)
+ self.assertEqual(['func_a', 'func_b'], problem_range)
def test_check_good_not_bad(self):
func_in_good = 'func_c'
diff --git a/afdo_tools/generate_afdo_from_tryjob.py b/afdo_tools/generate_afdo_from_tryjob.py
index b8a2d66..3ed578e 100755
--- a/afdo_tools/generate_afdo_from_tryjob.py
+++ b/afdo_tools/generate_afdo_from_tryjob.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -115,7 +115,7 @@
'--tryjob',
required=True,
type=_tryjob_arg,
- help='Path to our tryjob\'s artifacts. Accepts a gs:// path, pantheon '
+ help="Path to our tryjob's artifacts. Accepts a gs:// path, pantheon "
'link, or tryjob ID, e.g. R75-11965.0.0-b3648595. In the last case, '
'the assumption is that you ran a chell-chrome-pfq-tryjob.')
parser.add_argument(
@@ -127,7 +127,7 @@
'-k',
'--keep_artifacts_on_failure',
action='store_true',
- help='Don\'t remove the tempdir on failure')
+ help="Don't remove the tempdir on failure")
args = parser.parse_args()
if not distutils.spawn.find_executable(_CREATE_LLVM_PROF):
diff --git a/afdo_tools/run_afdo_tryjob.py b/afdo_tools/run_afdo_tryjob.py
index de45af0..e14cd91 100755
--- a/afdo_tools/run_afdo_tryjob.py
+++ b/afdo_tools/run_afdo_tryjob.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -94,7 +94,7 @@
user_patches = args.patch
if tag_profiles_with_current_time and use_afdo_generation_stage:
- raise ValueError('You can\'t tag profiles with the time + have '
+ raise ValueError("You can't tag profiles with the time + have "
'afdo-generate')
if not tag_profiles_with_current_time and not use_afdo_generation_stage:
diff --git a/auto_delete_nightly_test_data.py b/auto_delete_nightly_test_data.py
index 699e12a..429d0b4 100755
--- a/auto_delete_nightly_test_data.py
+++ b/auto_delete_nightly_test_data.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 The Chromium OS Authors. All rights reserved.
@@ -22,6 +22,8 @@
from cros_utils import misc
DIR_BY_WEEKDAY = ('Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun')
+NIGHTLY_TESTS_WORKSPACE = os.path.join(constants.CROSTC_WORKSPACE,
+ 'nightly-tests')
def CleanNumberedDir(s, dry_run=False):
@@ -48,7 +50,7 @@
## check 's' to make sure it is sane. A valid dir to be removed must be
## '/usr/local/google/crostc/(SUN|MON|TUE...|SAT)'.
valid_dir_pattern = (
- '^' + constants.CROSTC_WORKSPACE + '/(' + '|'.join(DIR_BY_WEEKDAY) + ')')
+ '^' + NIGHTLY_TESTS_WORKSPACE + '/(' + '|'.join(DIR_BY_WEEKDAY) + ')')
if not re.search(valid_dir_pattern, s):
print('Trying to delete an invalid dir "{0}" (must match "{1}"), '
'please check.'.format(s, valid_dir_pattern))
@@ -196,7 +198,7 @@
dated_dir = DIR_BY_WEEKDAY[i - 1]
rv += 0 if CleanDatedDir(
- os.path.join(constants.CROSTC_WORKSPACE, dated_dir),
+ os.path.join(NIGHTLY_TESTS_WORKSPACE, dated_dir),
options.dry_run) else 1
diff --git a/binary_search_tool/MAINTENANCE b/binary_search_tool/MAINTENANCE
index 8e5b3c6..8f96ff1 100644
--- a/binary_search_tool/MAINTENANCE
+++ b/binary_search_tool/MAINTENANCE
@@ -1,3 +1,7 @@
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
This document is for future maintainers of the binary search/bisection tools.
Authors:
@@ -21,7 +25,9 @@
TESTING:
All unit tests live under the ./test directory. However, these tests
-specifically test binary_search_state.py, binary_search_perforce.py, bisect.py.
+specifically test binary_search_state.py, binary_search_perforce.py,
+run_bisect.py.
+
These unit tests will not test the specific logic for ChromeOS/Android
bisection. To test the ChromeOS/Android bisectors, use the common/hash_test.sh
test. This is a simple test case that just checks the hashes of files on your
@@ -62,10 +68,11 @@
the code unfortunately. I will attempt to clear up the major offenders of this:
1. common.py's argument dictionary:
- binary_search_state.py and bisect.py both have to have near identical
- arguments in order to support argument overriding in bisect.py. However
- they do have to be slightly different. Mainly, bisect.py needs to have no
- default values for arguments (so it can determine what's being overriden).
+ binary_search_state.py and run_bisect.py both have to have near identical
+ arguments in order to support argument overriding in run_bisect.py. However
+ they do have to be slightly different. Mainly, run_bisect.py needs to have
+ no default values for arguments (so it can determine what's being
+ overriden).
In order to reduce huge amounts of code duplication for the argument
building, we put argument building in common.py. That way both modules
diff --git a/binary_search_tool/README.bisect.md b/binary_search_tool/README.bisect.md
index 74715ca..bd9e0f1 100644
--- a/binary_search_tool/README.bisect.md
+++ b/binary_search_tool/README.bisect.md
@@ -1,6 +1,6 @@
-# `bisect.py`
+# `run_bisect.py`
-`bisect.py` is a wrapper around the general purpose
+`run_bisect.py` is a wrapper around the general purpose
`binary_search_state.py`. It provides a user friendly interface for
bisecting various compilation errors. The 2 currently provided
methods of bisecting are ChromeOS package and object bisection. Each
@@ -33,8 +33,8 @@
/build/${board}.work - A full copy of /build/${board}.bad
```
-1. Cleanup: bisect.py does most cleanup for you, the only thing required by the
- user is to cleanup all built images and the three build trees made in
+1. Cleanup: run_bisect.py does most cleanup for you, the only thing required by
+ the user is to cleanup all built images and the three build trees made in
`/build/`
1. Default Arguments:
@@ -58,13 +58,13 @@
1. Basic interactive test package bisection, on daisy board:
```
- ./bisect.py package daisy 172.17.211.184
+ ./run_bisect.py package daisy 172.17.211.184
```
2. Basic boot test package bisection, on daisy board:
```
- ./bisect.py package daisy 172.17.211.184 -t cros_pkg/boot_test.sh
+ ./run_bisect.py package daisy 172.17.211.184 -t cros_pkg/boot_test.sh
```
### ChromeOS Object
@@ -125,10 +125,10 @@
1. Examples:
1. Basic interactive test object bisection, on daisy board for cryptohome
- package: `./bisect.py object daisy 172.17.211.184 cryptohome`
+ package: `./run_bisect.py object daisy 172.17.211.184 cryptohome`
2. Basic boot test package bisection, on daisy board for cryptohome
- package: `./bisect.py object daisy 172.17.211.184 cryptohome
+ package: `./run_bisect.py object daisy 172.17.211.184 cryptohome
--test_script=sysroot_wrapper/boot_test.sh`
### Android object
@@ -202,16 +202,17 @@
1. Examples:
1. Basic interactive test android bisection, where the android source is at
- ~/android_src: `./bisect.py android ~/android_src`
+ ~/android_src: `./run_bisect.py android ~/android_src`
2. Basic boot test android bisection, where the android source is at
- `~/android_src`, and 10 jobs will be used to build android: `./bisect.py
+ `~/android_src`, and 10 jobs will be used to build android:
+ `./run_bisect.py
android ~/android_src --num_jobs=10
--test_script=sysroot_wrapper/boot_test.sh`
### Resuming
-`bisect.py` and `binary_search_state.py` offer the
+`run_bisect.py` and `binary_search_state.py` offer the
ability to resume a bisection in case it was interrupted by a
SIGINT, power failure, etc. Every time the tool completes a
bisection iteration its state is saved to disk (usually to the file
@@ -221,14 +222,14 @@
### Overriding
-You can run `./bisect.py --help` or `./binary_search_state.py
+You can run `./run_bisect.py --help` or `./binary_search_state.py
--help` for a full list of arguments that can be overriden. Here are
a couple of examples:
Example 1 (do boot test instead of interactive test):
```
-./bisect.py package daisy 172.17.211.182 --test_script=cros_pkg/boot_test.sh
+./run_bisect.py package daisy 172.17.211.182 --test_script=cros_pkg/boot_test.sh
```
Example 2 (do package bisector system test instead of interactive test, this
@@ -236,13 +237,13 @@
hash_test.sh for more details):
```
-./bisect.py package daisy 172.17.211.182 \
+./run_bisect.py package daisy 172.17.211.182 \
--test_script=common/hash_test.sh --test_setup_script=""
```
Example 3 (enable verbose mode, disable pruning, and disable verification):
```
-./bisect.py package daisy 172.17.211.182
+./run_bisect.py package daisy 172.17.211.182
--verbose --prune=False --verify=False
```
diff --git a/binary_search_tool/README.pass_bisect.md b/binary_search_tool/README.pass_bisect.md
index d03a563..eb0f12a 100644
--- a/binary_search_tool/README.pass_bisect.md
+++ b/binary_search_tool/README.pass_bisect.md
@@ -41,7 +41,7 @@
bisection:
```
- ./bisect.py android PATH_TO_ANDROID_HOME_DIR
+ ./run_bisect.py android PATH_TO_ANDROID_HOME_DIR
--pass_bisect=’android/generate_cmd.sh’
--prune=False
--ir_diff
diff --git a/binary_search_tool/README.testing.md b/binary_search_tool/README.testing.md
index 139dc08..0f06679 100644
--- a/binary_search_tool/README.testing.md
+++ b/binary_search_tool/README.testing.md
@@ -28,7 +28,7 @@
```
$ cd toolchain-utils/binary_search_tool/test
-$ ./binary_search_tool_tester.py
+$ ./binary_search_tool_test.py
```
# Running the bisection tests, testing the compiler wrapper.
@@ -59,7 +59,7 @@
```
$ cd toolchain-utils/binary_search_tool
-$ ./run_bisect_test.py
+$ ./run_bisect_tests.py
```
If you are testing with the CHROMEOS COMPILER WRAPPER, you MUST run the
@@ -82,7 +82,7 @@
Run the test script:
```
-$ ./run_bisect_test.py
+$ ./run_bisect_tests.py
```
diff --git a/binary_search_tool/__init__.py b/binary_search_tool/__init__.py
index 8b13789..76500de 100644
--- a/binary_search_tool/__init__.py
+++ b/binary_search_tool/__init__.py
@@ -1 +1,4 @@
-
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
diff --git a/binary_search_tool/binary_search_perforce.py b/binary_search_tool/binary_search_perforce.py
index a4f8c1c..5065f70 100755
--- a/binary_search_tool/binary_search_perforce.py
+++ b/binary_search_tool/binary_search_perforce.py
@@ -1,9 +1,11 @@
-#!/usr/bin/env python2
-
-# Copyright 2018 The Chromium OS Authors. All rights reserved.
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+
"""Module of binary serch for perforce."""
+from __future__ import division
from __future__ import print_function
import math
@@ -25,7 +27,7 @@
if ' ' not in p4_path:
p4_string += ' -a %s' % p4_path
else:
- p4_string += " -a \"" + (' //' + client_name + '/').join(p4_path) + "\""
+ p4_string += ' -a "' + (' //' + client_name + '/').join(p4_path) + '"'
return p4_string
@@ -50,8 +52,8 @@
command += ' && cd ' + checkoutdir
command += ' && cp ${HOME}/.p4config .'
command += ' && chmod u+w .p4config'
- command += " && echo \"P4PORT=" + p4_port + "\" >> .p4config"
- command += " && echo \"P4CLIENT=" + client_name + "\" >> .p4config"
+ command += ' && echo "P4PORT=' + p4_port + '" >> .p4config'
+ command += ' && echo "P4CLIENT=' + client_name + '" >> .p4config'
command += (' && g4 client ' + _GetP4ClientSpec(client_name, p4_paths))
command += ' && g4 sync '
command += ' && cd -'
@@ -84,7 +86,7 @@
# For the first run, update self.hi with total pass/transformation count
if self.hi == 0:
self.hi = self.total
- self.current = (self.hi + self.lo) / 2
+ self.current = (self.hi + self.lo) // 2
message = ('Bisecting between: (%d, %d)' % (self.lo, self.hi))
self.logger.LogOutput(message, print_to_console=verbose)
message = ('Current limit number: %d' % self.current)
@@ -100,11 +102,11 @@
If status == 1, it means that runtime error still happens with current pass/
transformation, so we need to decrease upper bound for binary search.
- Return:
+ Returns:
True if we find the bad pass/transformation, or cannot find bad one after
decreasing to the first pass/transformation. Otherwise False.
"""
- assert status == 0 or status == 1 or status == 125
+ assert status in (0, 1, 125), status
if self.current == 0:
message = ('Runtime error occurs before first pass/transformation. '
@@ -145,7 +147,7 @@
self.logger = logger.GetLogger()
def SetSortedList(self, sorted_list):
- assert len(sorted_list) > 0
+ assert sorted_list
self.sorted_list = sorted_list
self.index_log = []
self.hi = len(sorted_list) - 1
@@ -159,7 +161,7 @@
message = ('Revision: %s index: %d returned: %d' %
(self.sorted_list[self.current], self.current, status))
self.logger.LogOutput(message, print_to_console=verbose)
- assert status == 0 or status == 1 or status == 125
+ assert status in (0, 1, 125), status
self.index_log.append(self.current)
self.status_log.append(status)
bsp = BinarySearchPoint(self.sorted_list[self.current], status, tag)
@@ -168,13 +170,13 @@
if status == 125:
self.skipped_indices.append(self.current)
- if status == 0 or status == 1:
+ if status in (0, 1):
if status == 0:
self.lo = self.current + 1
elif status == 1:
self.hi = self.current
self.logger.LogOutput('lo: %d hi: %d\n' % (self.lo, self.hi))
- self.current = (self.lo + self.hi) / 2
+ self.current = (self.lo + self.hi) // 2
if self.lo == self.hi:
message = ('Search complete. First bad version: %s'
@@ -194,19 +196,19 @@
def GetNextFlakyBinary(self):
t = (self.lo, self.current, self.hi)
q = [t]
- while len(q):
+ while q:
element = q.pop(0)
if element[1] in self.skipped_indices:
# Go top
- to_add = (element[0], (element[0] + element[1]) / 2, element[1])
+ to_add = (element[0], (element[0] + element[1]) // 2, element[1])
q.append(to_add)
# Go bottom
- to_add = (element[1], (element[1] + element[2]) / 2, element[2])
+ to_add = (element[1], (element[1] + element[2]) // 2, element[2])
q.append(to_add)
else:
self.current = element[1]
return
- assert len(q), 'Queue should never be 0-size!'
+ assert q, 'Queue should never be 0-size!'
def GetNextFlakyLinear(self):
current_hi = self.current
@@ -225,7 +227,7 @@
current_lo -= 1
def GetNext(self):
- self.current = (self.hi + self.lo) / 2
+ self.current = (self.hi + self.lo) // 2
# Try going forward if current is skipped.
if self.current in self.skipped_indices:
self.GetNextFlakyBinary()
@@ -295,7 +297,7 @@
def GetNextRevision(self):
pass
- def CheckoutRevision(self, revision):
+ def CheckoutRevision(self, current_revision):
pass
def SetStatus(self, status):
@@ -362,14 +364,14 @@
if not os.path.isfile(self.checkout_dir + '/.p4config'):
command = 'cd %s' % self.checkout_dir
command += ' && cp ${HOME}/.p4config .'
- command += " && echo \"P4PORT=" + self.p4_port + "\" >> .p4config"
- command += " && echo \"P4CLIENT=" + self.client_name + "\" >> .p4config"
+ command += ' && echo "P4PORT=' + self.p4_port + '" >> .p4config'
+ command += ' && echo "P4CLIENT=' + self.client_name + '" >> .p4config'
self.ce.RunCommand(command)
command = 'cd %s' % self.checkout_dir
command += '; g4 changes -c %s' % self.client_name
_, out, _ = self.ce.RunCommandWOUTPUOT(command)
changes = re.findall(r'Change (\d+)', out)
- if len(changes) != 0:
+ if changes:
command = 'cd %s' % self.checkout_dir
for change in changes:
command += '; g4 revert -c %s' % change
@@ -413,9 +415,8 @@
for pr in problematic_ranges:
if cr in range(pr[0], pr[1]):
patch_file = '/home/asharif/triage_tool/%d-%d.patch' % (pr[0], pr[1])
- f = open(patch_file)
- patch = f.read()
- f.close()
+ with open(patch_file, encoding='utf-8') as f:
+ patch = f.read()
files = re.findall('--- (//.*)', patch)
command += '; cd %s' % self.checkout_dir
for f in files:
@@ -505,7 +506,7 @@
logger.GetLogger().LogOutput('Cleaning up...')
finally:
logger.GetLogger().LogOutput(str(p4gccbs.bs), print_to_console=verbose)
- status = p4gccbs.Cleanup()
+ p4gccbs.Cleanup()
if __name__ == '__main__':
diff --git a/binary_search_tool/binary_search_state.py b/binary_search_tool/binary_search_state.py
index f6c8ac7..1ddd65c 100755
--- a/binary_search_tool/binary_search_state.py
+++ b/binary_search_tool/binary_search_state.py
@@ -1,10 +1,12 @@
-#!/usr/bin/env python2
-
-# Copyright 2018 The Chromium OS Authors. All rights reserved.
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+
"""The binary search wrapper."""
+from __future__ import division
from __future__ import print_function
import argparse
@@ -20,15 +22,14 @@
import time
# Adds cros_utils to PYTHONPATH
-import common
+from binary_search_tool import binary_search_perforce
+from binary_search_tool import common
+from binary_search_tool import pass_mapping
# Now we do import from cros_utils
from cros_utils import command_executer
from cros_utils import logger
-import binary_search_perforce
-import pass_mapping
-
GOOD_SET_VAR = 'BISECT_GOOD_SET'
BAD_SET_VAR = 'BISECT_BAD_SET'
@@ -37,11 +38,6 @@
os.path.dirname(STATE_FILE), '.%s' % os.path.basename(STATE_FILE))
-class Error(Exception):
- """The general binary search tool error class."""
- pass
-
-
@contextlib.contextmanager
def SetFile(env_var, items):
"""Generate set files that can be used by switch/test scripts.
@@ -59,7 +55,7 @@
env_var: What environment variable to store the file name in.
items: What items are in this set.
"""
- with tempfile.NamedTemporaryFile() as f:
+ with tempfile.NamedTemporaryFile('w', encoding='utf-8') as f:
os.environ[env_var] = f.name
f.write('\n'.join(items))
f.flush()
@@ -100,10 +96,10 @@
self.cmd_script = None
self.mode = None
self.PopulateItemsUsingCommand(self.get_initial_items)
- self.currently_good_items = set([])
- self.currently_bad_items = set([])
- self.found_items = set([])
- self.known_good = set([])
+ self.currently_good_items = set()
+ self.currently_bad_items = set()
+ self.found_items = set()
+ self.known_good = set()
self.start_time = time.time()
@@ -161,7 +157,7 @@
item_list: list of all items to be switched
"""
if self.file_args:
- with tempfile.NamedTemporaryFile() as f:
+ with tempfile.NamedTemporaryFile('w', encoding='utf-8') as f:
f.write('\n'.join(item_list))
f.flush()
command = '%s %s' % (switch_script, f.name)
@@ -174,9 +170,8 @@
command, print_to_console=self.verbose)
except OSError as e:
if e.errno == errno.E2BIG:
- raise Error('Too many arguments for switch script! Use --file_args')
- else:
- raise
+ raise RuntimeError('Too many arguments for switch script! Use '
+ '--file_args')
assert ret == 0, 'Switch script %s returned %d' % (switch_script, ret)
def TestScript(self):
@@ -349,12 +344,12 @@
return None
def BuildWithPassLimit(self, limit, generate_ir=False):
- """ Rebuild bad item with pass level bisect limit
+ """Rebuild bad item with pass level bisect limit
Run command line script generated by GenerateBadCommandScript(), with
pass level limit flags.
- Return:
+ Returns:
pass_num: current number of the pass, or total number of passes if
limit set to -1.
pass_name: The debugcounter name of current limit pass.
@@ -389,7 +384,7 @@
break
pass_num += 1
last_pass = l
- if limit != -1 and pass_num != limit:
+ if limit not in (-1, pass_num):
raise ValueError('[Error] While building, limit number does not match.')
return pass_num, self.CollectPassName(last_pass)
@@ -398,16 +393,19 @@
pass_name=None,
pass_limit=-1,
generate_ir=False):
- """ Rebuild bad item with transformation level bisect limit
+ """Rebuild bad item with transformation level bisect limit
Run command line script generated by GenerateBadCommandScript(), with
pass level limit flags and transformation level limit flags.
Args:
- limit: transformation level limit for bad item
- pass_name: name of bad pass debugcounter from pass level bisect result
- pass_limit: pass level limit from pass level bisect result
- Return: Total number of transformations if limit set to -1, else return 0.
+ limit: transformation level limit for bad item.
+ pass_name: name of bad pass debugcounter from pass level bisect result.
+ pass_limit: pass level limit from pass level bisect result.
+ generate_ir: Whether to generate IR comparison.
+
+ Returns:
+ Total number of transformations if limit set to -1, else return 0.
"""
counter_name = pass_name
@@ -513,7 +511,7 @@
'Total %s number: %d' % (self.mode, self.binary_search.total))
trans_index, _ = self.DoBinarySearchBadPass(pass_index, pass_name)
- if (trans_index == 0):
+ if trans_index == 0:
raise ValueError('Bisecting %s cannot reproduce good result.' % pass_name)
if self.ir_diff:
@@ -530,13 +528,15 @@
Args:
pass_index: Works for transformation level bisection, indicates the limit
- number of pass from pass level bisecting result.
+ number of pass from pass level bisecting result.
pass_name: Works for transformation level bisection, indicates
- DebugCounter name of the bad pass from pass level bisecting result.
- Return:
+ DebugCounter name of the bad pass from pass level bisecting
+ result.
+
+ Returns:
index: Index of problematic pass/transformation.
pass_name: Works for pass level bisection, returns DebugCounter name for
- bad pass.
+ bad pass.
"""
# If in resume mode don't reset search_cycles
if not self.resumed:
@@ -609,7 +609,7 @@
new data.
Raises:
- Error if STATE_FILE already exists but is not a symlink.
+ OSError if STATE_FILE already exists but is not a symlink.
"""
ce, l = self.ce, self.l
self.ce, self.l, self.binary_search.logger = None, None, None
@@ -623,8 +623,8 @@
if os.path.islink(STATE_FILE):
old_state = os.readlink(STATE_FILE)
else:
- raise Error(('%s already exists and is not a symlink!\n'
- 'State file saved to %s' % (STATE_FILE, path)))
+ raise OSError(('%s already exists and is not a symlink!\n'
+ 'State file saved to %s' % (STATE_FILE, path)))
# Create new link and atomically overwrite old link
temp_link = '%s.link' % HIDDEN_STATE_FILE
@@ -642,26 +642,27 @@
if not os.path.isfile(STATE_FILE):
return None
try:
- bss = pickle.load(file(STATE_FILE))
- bss.l = logger.GetLogger()
- bss.ce = command_executer.GetCommandExecuter()
- bss.binary_search.logger = bss.l
- bss.start_time = time.time()
+ with open(STATE_FILE, 'rb') as f:
+ bss = pickle.load(f)
+ bss.l = logger.GetLogger()
+ bss.ce = command_executer.GetCommandExecuter()
+ bss.binary_search.logger = bss.l
+ bss.start_time = time.time()
- # Set resumed to be True so we can enter DoBinarySearch without the method
- # resetting our current search_cycles to 0.
- bss.resumed = True
+ # Set resumed to be True so we can enter DoBinarySearch without the
+ # method resetting our current search_cycles to 0.
+ bss.resumed = True
- # Set currently_good_items and currently_bad_items to empty so that the
- # first iteration after resuming will always be non-incremental. This is
- # just in case the environment changes, the user makes manual changes, or
- # a previous switch_script corrupted the environment.
- bss.currently_good_items = set([])
- bss.currently_bad_items = set([])
+ # Set currently_good_items and currently_bad_items to empty so that the
+ # first iteration after resuming will always be non-incremental. This
+ # is just in case the environment changes, the user makes manual
+ # changes, or a previous switch_script corrupted the environment.
+ bss.currently_good_items = set()
+ bss.currently_bad_items = set()
- binary_search_perforce.verbose = bss.verbose
- return bss
- except StandardError:
+ binary_search_perforce.verbose = bss.verbose
+ return bss
+ except Exception:
return None
def RemoveState(self):
@@ -686,8 +687,8 @@
"""Return h m s format of elapsed time since execution has started."""
diff = int(time.time() - self.start_time)
seconds = diff % 60
- minutes = (diff / 60) % 60
- hours = diff / (60 * 60)
+ minutes = (diff // 60) % 60
+ hours = diff // (60 * 60)
seconds = str(seconds).rjust(2)
minutes = str(minutes).rjust(2)
@@ -871,16 +872,12 @@
verify, file_args, verbose)
bss.DoVerify()
- try:
- bss.DoSearchBadItems()
- if pass_bisect:
- bss.DoSearchBadPass()
- bss.RemoveState()
- logger.GetLogger().LogOutput(
- 'Total execution time: %s' % bss.ElapsedTimeString())
- except Error as e:
- logger.GetLogger().LogError(e)
- return 1
+ bss.DoSearchBadItems()
+ if pass_bisect:
+ bss.DoSearchBadPass()
+ bss.RemoveState()
+ logger.GetLogger().LogOutput(
+ 'Total execution time: %s' % bss.ElapsedTimeString())
return 0
diff --git a/binary_search_tool/bisect_driver.py b/binary_search_tool/bisect_driver.py
index 82df15d..ac37ad9 100644
--- a/binary_search_tool/bisect_driver.py
+++ b/binary_search_tool/bisect_driver.py
@@ -1,10 +1,10 @@
-# Copyright 2016 Googie Inc. All rights Reserved.
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
#
# This script is used to help the compiler wrapper in the ChromeOS and
# Android build systems bisect for bad object files.
-#
-# pylint: disable=not-callable
-# pylint: disable=indentation
"""Utilities for bisection of ChromeOS and Android object files.
@@ -267,10 +267,10 @@
if CONTINUE_ON_REDUNDANCY:
return True
raise Exception(
- 'Trying to cache file %s multiple times. To avoid the error, set \
- CONTINUE_ON_REDUNDANCY to 1. For reference, the list of such files \
- will be written to %s' % (abs_file_path,
- os.path.join(population_dir, '_DUPS')))
+ 'Trying to cache file %s multiple times. To avoid the error, set ' \
+ 'BISECT_CONTINUE_ON_REDUNDANCY to 1. For reference, the list of ' \
+ 'such files will be written to %s' % (abs_file_path, os.path.join(
+ population_dir, '_DUPS')))
shutil.copy2(abs_file_path, bisect_path)
# Set cache object to be read-only so later compilations can't
@@ -278,7 +278,8 @@
os.chmod(bisect_path, 0o444)
return True
else:
- # File not found (happens when compilation fails but error code is still 0)
+ # File not found (happens when compilation fails but error code is still
+ # 0)
return False
except Exception:
print('Could not cache file %s' % abs_file_path, file=sys.stderr)
diff --git a/binary_search_tool/common.py b/binary_search_tool/common.py
index 40660b5..85cd478 100644
--- a/binary_search_tool/common.py
+++ b/binary_search_tool/common.py
@@ -1,4 +1,5 @@
-# Copyright 2018 The Chromium OS Authors. All rights reserved.
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
@@ -7,14 +8,14 @@
This module serves two main purposes:
1. Programatically include the utils module in PYTHONPATH
2. Create the argument parsing shared between binary_search_state.py and
- bisect.py
+ run_bisect.py
The argument parsing is handled by populating _ArgsDict with all arguments.
-_ArgsDict is required so that binary_search_state.py and bisect.py can share
-the argument parsing, but treat them slightly differently. For example,
-bisect.py requires that all argument defaults are suppressed so that overriding
-can occur properly (i.e. only options that are explicitly entered by the user
-end up in the resultant options dictionary).
+_ArgsDict is required so that binary_search_state.py and run_bisect.py can
+share the argument parsing, but treat them slightly differently. For example,
+run_bisect.py requires that all argument defaults are suppressed so that
+overriding can occur properly (i.e. only options that are explicitly entered
+by the user end up in the resultant options dictionary).
ArgumentDict inherits OrderedDict in order to preserve the order the args are
created so the help text is made properly.
@@ -32,8 +33,8 @@
os.path.dirname(sys.argv[0])))
else:
wdir = os.getcwd()
- utils_pythonpath = os.path.abspath('{0}/{1}/..'.format(wdir, os.path.dirname(
- sys.argv[0])))
+ utils_pythonpath = os.path.abspath('{0}/{1}/..'.format(
+ wdir, os.path.dirname(sys.argv[0])))
sys.path.append(utils_pythonpath)
@@ -55,8 +56,10 @@
...
}
"""
- _POSSIBLE_OPTIONS = ['action', 'nargs', 'const', 'default', 'type', 'choices',
- 'required', 'help', 'metavar', 'dest']
+ _POSSIBLE_OPTIONS = [
+ 'action', 'nargs', 'const', 'default', 'type', 'choices', 'required',
+ 'help', 'metavar', 'dest'
+ ]
def AddArgument(self, *args, **kwargs):
"""Add argument to ArgsDict, has same signature as argparse.add_argument
@@ -77,7 +80,7 @@
TypeError: if args is empty or if option in kwargs is not a valid
option for argparse.add_argument.
"""
- if len(args) == 0:
+ if not args:
raise TypeError('Argument needs at least one name')
for key in kwargs:
@@ -106,8 +109,8 @@
Args:
parser: type argparse.ArgumentParser, will call add_argument for every item
in _ArgsDict
- override: True if being called from bisect.py. Used to say that default and
- required options are to be ignored
+ override: True if being called from run_bisect.py. Used to say that default
+ and required options are to be ignored
Returns:
None
@@ -115,7 +118,7 @@
ArgsDict = GetArgsDict()
# Have no defaults when overriding
- for arg_names, arg_options in ArgsDict.iteritems():
+ for arg_names, arg_options in ArgsDict.items():
if override:
arg_options = arg_options.copy()
arg_options.pop('default', None)
@@ -147,34 +150,34 @@
'--get_initial_items',
dest='get_initial_items',
help='Script to run to get the initial objects. '
- 'If your script requires user input '
- 'the --verbose option must be used')
+ 'If your script requires user input '
+ 'the --verbose option must be used')
args.AddArgument(
'-g',
'--switch_to_good',
dest='switch_to_good',
help='Script to run to switch to good. '
- 'If your switch script requires user input '
- 'the --verbose option must be used')
+ 'If your switch script requires user input '
+ 'the --verbose option must be used')
args.AddArgument(
'-b',
'--switch_to_bad',
dest='switch_to_bad',
help='Script to run to switch to bad. '
- 'If your switch script requires user input '
- 'the --verbose option must be used')
+ 'If your switch script requires user input '
+ 'the --verbose option must be used')
args.AddArgument(
'-I',
'--test_setup_script',
dest='test_setup_script',
help='Optional script to perform building, flashing, '
- 'and other setup before the test script runs.')
+ 'and other setup before the test script runs.')
args.AddArgument(
'-t',
'--test_script',
dest='test_script',
help='Script to run to test the '
- 'output after packages are built.')
+ 'output after packages are built.')
# No input (evals to False),
# --prune (evals to True),
# --prune=False,
@@ -189,19 +192,19 @@
type=StrToBool,
metavar='bool',
help='If True, continue until all bad items are found. '
- 'Defaults to False.')
+ 'Defaults to False.')
args.AddArgument(
'-P',
'--pass_bisect',
dest='pass_bisect',
default=None,
help='Script to generate another script for pass level bisect, '
- 'which contains command line options to build bad item. '
- 'This will also turn on pass/transformation level bisection. '
- 'Needs support of `-opt-bisect-limit`(pass) and '
- '`-print-debug-counter`(transformation) from LLVM. '
- 'For now it only supports one single bad item, so to use it, '
- 'prune must be set to False.')
+ 'which contains command line options to build bad item. '
+ 'This will also turn on pass/transformation level bisection. '
+ 'Needs support of `-opt-bisect-limit`(pass) and '
+ '`-print-debug-counter`(transformation) from LLVM. '
+ 'For now it only supports one single bad item, so to use it, '
+ 'prune must be set to False.')
# No input (evals to False),
# --ir_diff (evals to True),
# --ir_diff=False,
@@ -216,8 +219,8 @@
type=StrToBool,
metavar='bool',
help='Whether to print IR differences before and after bad '
- 'pass/transformation to verbose output. Defaults to False, '
- 'only works when pass_bisect is enabled.')
+ 'pass/transformation to verbose output. Defaults to False, '
+ 'only works when pass_bisect is enabled.')
# No input (evals to False),
# --noincremental (evals to True),
# --noincremental=False,
@@ -231,8 +234,8 @@
default=False,
type=StrToBool,
metavar='bool',
- help='If True, don\'t propagate good/bad changes '
- 'incrementally. Defaults to False.')
+ help="If True, don't propagate good/bad changes "
+ 'incrementally. Defaults to False.')
# No input (evals to False),
# --file_args (evals to True),
# --file_args=False,
@@ -247,7 +250,7 @@
type=StrToBool,
metavar='bool',
help='Whether to use a file to pass arguments to scripts. '
- 'Defaults to False.')
+ 'Defaults to False.')
# No input (evals to True),
# --verify (evals to True),
# --verify=False,
@@ -261,7 +264,7 @@
type=StrToBool,
metavar='bool',
help='Whether to run verify iterations before searching. '
- 'Defaults to True.')
+ 'Defaults to True.')
args.AddArgument(
'-N',
'--prune_iterations',
@@ -289,5 +292,5 @@
dest='resume',
action='store_true',
help='Resume bisection tool execution from state file.'
- 'Useful if the last bisection was terminated '
- 'before it could properly finish.')
+ 'Useful if the last bisection was terminated '
+ 'before it could properly finish.')
diff --git a/binary_search_tool/compiler_wrapper.py b/binary_search_tool/compiler_wrapper.py
index a6d189b..8755ed4 100755
--- a/binary_search_tool/compiler_wrapper.py
+++ b/binary_search_tool/compiler_wrapper.py
@@ -1,4 +1,9 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Prototype compiler wrapper.
Only tested with: gcc, g++, clang, clang++
@@ -21,7 +26,7 @@
import shlex
import sys
-import bisect_driver
+from binary_search_tool import bisect_driver
WRAPPED = '%s.real' % sys.argv[0]
BISECT_STAGE = os.environ.get('BISECT_STAGE')
diff --git a/binary_search_tool/cros_pkg/create_cleanup_script.py b/binary_search_tool/cros_pkg/create_cleanup_script.py
index ed4eab6..62ee38f 100755
--- a/binary_search_tool/cros_pkg/create_cleanup_script.py
+++ b/binary_search_tool/cros_pkg/create_cleanup_script.py
@@ -1,6 +1,9 @@
-#!/usr/bin/env python2
-#
-# Copyright 2015 Google Inc. All Rights Reserved
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""The script to generate a cleanup script after setup.sh.
This script takes a set of flags, telling it what setup.sh changed
@@ -69,20 +72,22 @@
if options.old_link or options.renamed_tree:
if not options.tree_existed:
- Usage(parser, 'If --tree_existed is False, cannot have '
- '--renamed_tree or --old_link')
+ Usage(
+ parser, 'If --tree_existed is False, cannot have '
+ '--renamed_tree or --old_link')
if options.old_link and options.renamed_tree:
Usage(parser, '--old_link and --renamed_tree are incompatible options.')
if options.tree_existed:
if not options.old_link and not options.renamed_tree:
- Usage(parser, 'If --tree_existed is True, then must have either '
- '--old_link or --renamed_tree')
+ Usage(
+ parser, 'If --tree_existed is True, then must have either '
+ '--old_link or --renamed_tree')
out_filename = 'cros_pkg/' + options.board + '_cleanup.sh'
- with open(out_filename, 'w') as out_file:
+ with open(out_filename, 'w', encoding='utf-8') as out_file:
out_file.write('#!/bin/bash\n\n')
# First, remove the 'new' soft link.
out_file.write('sudo rm /build/%s\n' % options.board)
@@ -100,8 +105,8 @@
original_link = original_link[1:]
if original_link[-1] == "'":
original_link = original_link[:-1]
- out_file.write('sudo ln -s %s /build/%s\n' % (original_link,
- options.board))
+ out_file.write(
+ 'sudo ln -s %s /build/%s\n' % (original_link, options.board))
out_file.write('\n')
# Remove common.sh file
out_file.write('rm common/common.sh\n')
diff --git a/binary_search_tool/pass_mapping.py b/binary_search_tool/pass_mapping.py
index cb80910..2678fd6 100644
--- a/binary_search_tool/pass_mapping.py
+++ b/binary_search_tool/pass_mapping.py
@@ -1,6 +1,8 @@
-# Copyright 2018 The Chromium OS Authors. All rights reserved.
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+
"""Config file for pass level bisection
Provides a mapping from pass info from -opt-bisect result to DebugCounter name.
diff --git a/binary_search_tool/bisect.py b/binary_search_tool/run_bisect.py
similarity index 93%
rename from binary_search_tool/bisect.py
rename to binary_search_tool/run_bisect.py
index c7dd523..ef1048b 100755
--- a/binary_search_tool/bisect.py
+++ b/binary_search_tool/run_bisect.py
@@ -1,28 +1,29 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""The unified package/object bisecting tool."""
from __future__ import print_function
import abc
import argparse
+from argparse import RawTextHelpFormatter
import os
import sys
-from argparse import RawTextHelpFormatter
-import common
+from binary_search_tool import binary_search_state
+from binary_search_tool import common
from cros_utils import command_executer
from cros_utils import logger
-import binary_search_state
-
-class Bisector(object):
+class Bisector(object, metaclass=abc.ABCMeta):
"""The abstract base class for Bisectors."""
- # Make Bisector an abstract class
- __metaclass__ = abc.ABCMeta
-
def __init__(self, options, overrides=None):
"""Constructor for Bisector abstract base class
@@ -44,7 +45,7 @@
which arguments have been overridden.
Example output:
- ./bisect.py package daisy 172.17.211.184 -I "" -t cros_pkg/my_test.sh
+ ./run_bisect.py package daisy 172.17.211.184 -I "" -t cros_pkg/my_test.sh
Performing ChromeOS Package bisection
Method Config:
board : daisy
@@ -83,10 +84,8 @@
out += '\nBisection Config: (* = overridden)\n'
max_key_len = max([len(str(x)) for x in args.keys()])
# Print args in common._ArgsDict order
- args_order = [x['dest'] for x in common.GetArgsDict().itervalues()]
- compare = lambda x, y: cmp(args_order.index(x), args_order.index(y))
-
- for key in sorted(args, cmp=compare):
+ args_order = [x['dest'] for x in common.GetArgsDict().values()]
+ for key in sorted(args, key=args_order.index):
val = args[key]
key_str = str(key).rjust(max_key_len)
val_str = str(val)
@@ -193,9 +192,9 @@
if options.dir:
os.environ['BISECT_DIR'] = options.dir
self.options.dir = os.environ.get('BISECT_DIR', '/tmp/sysroot_bisect')
- self.setup_cmd = ('%s %s %s %s' %
- (self.sysroot_wrapper_setup, self.options.board,
- self.options.remote, self.options.package))
+ self.setup_cmd = (
+ '%s %s %s %s' % (self.sysroot_wrapper_setup, self.options.board,
+ self.options.remote, self.options.package))
self.ArgOverride(self.default_kwargs, overrides)
@@ -306,7 +305,7 @@
_HELP_EPILOG = """
-Run ./bisect.py {method} --help for individual method help/args
+Run ./run_bisect.py {method} --help for individual method help/args
------------------
@@ -320,7 +319,7 @@
override_parser = argparse.ArgumentParser(
add_help=False,
argument_default=argparse.SUPPRESS,
- usage='bisect.py {mode} [options]')
+ usage='run_bisect.py {mode} [options]')
common.BuildArgParser(override_parser, override=True)
epilog = _HELP_EPILOG + override_parser.format_help()
diff --git a/binary_search_tool/run_bisect_test.py b/binary_search_tool/run_bisect_tests.py
similarity index 95%
rename from binary_search_tool/run_bisect_test.py
rename to binary_search_tool/run_bisect_tests.py
index d4ff4f7..9172d67 100755
--- a/binary_search_tool/run_bisect_test.py
+++ b/binary_search_tool/run_bisect_tests.py
@@ -1,4 +1,9 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Run full bisection test."""
from __future__ import print_function
@@ -81,7 +86,7 @@
# We don't need to do any special setup if running inside a ChromeOS
# chroot.
if os.path.exists('/usr/bin/x86_64-cros-linux-gnu-gcc'):
- return
+ return True
message = """
*** IMPORTANT --- READ THIS CAREFULLY!! ***
diff --git a/binary_search_tool/sysroot_wrapper/testing_test.py b/binary_search_tool/sysroot_wrapper/testing_test.py
index a0d6ca1..b5ceec1 100755
--- a/binary_search_tool/sysroot_wrapper/testing_test.py
+++ b/binary_search_tool/sysroot_wrapper/testing_test.py
@@ -1,4 +1,9 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Test for sysroot_wrapper bisector.
All files in bad_files will be determined to be bad. This test was made for
@@ -15,9 +20,9 @@
base_path = ('/var/cache/chromeos-chrome/chrome-src-internal/src/out_daisy/'
'Release/obj/')
bad_files = [
- os.path.join(base_path, 'base/base.cpu.o'), os.path.join(
- base_path, 'base/base.version.o'), os.path.join(base_path,
- 'apps/apps.launcher.o')
+ os.path.join(base_path, 'base/base.cpu.o'),
+ os.path.join(base_path, 'base/base.version.o'),
+ os.path.join(base_path, 'apps/apps.launcher.o')
]
bisect_dir = os.environ.get('BISECT_DIR', '/tmp/sysroot_bisect')
diff --git a/binary_search_tool/test/__init__.py b/binary_search_tool/test/__init__.py
index 8b13789..76500de 100644
--- a/binary_search_tool/test/__init__.py
+++ b/binary_search_tool/test/__init__.py
@@ -1 +1,4 @@
-
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
diff --git a/binary_search_tool/test/binary_search_tool_tester.py b/binary_search_tool/test/binary_search_tool_test.py
similarity index 90%
rename from binary_search_tool/test/binary_search_tool_tester.py
rename to binary_search_tool/test/binary_search_tool_test.py
index aff45a8..ca9313b 100755
--- a/binary_search_tool/test/binary_search_tool_tester.py
+++ b/binary_search_tool/test/binary_search_tool_test.py
@@ -1,10 +1,12 @@
-#!/usr/bin/env python2
-
-# Copyright 2018 The Chromium OS Authors. All rights reserved.
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+
"""Tests for bisecting tool."""
+from __future__ import division
from __future__ import print_function
__author__ = 'shenhan@google.com (Han Shen)'
@@ -16,15 +18,15 @@
from cros_utils import command_executer
from binary_search_tool import binary_search_state
-from binary_search_tool import bisect
+from binary_search_tool import run_bisect
-import common
-import gen_obj
+from binary_search_tool.test import common
+from binary_search_tool.test import gen_obj
def GenObj():
obj_num = random.randint(100, 1000)
- bad_obj_num = random.randint(obj_num / 100, obj_num / 20)
+ bad_obj_num = random.randint(obj_num // 100, obj_num // 20)
if bad_obj_num == 0:
bad_obj_num = 1
gen_obj.Main(['--obj_num', str(obj_num), '--bad_obj_num', str(bad_obj_num)])
@@ -38,10 +40,10 @@
class BisectTest(unittest.TestCase):
- """Tests for bisect.py"""
+ """Tests for run_bisect.py"""
def setUp(self):
- with open('./is_setup', 'w'):
+ with open('./is_setup', 'w', encoding='utf-8'):
pass
try:
@@ -57,8 +59,8 @@
except OSError:
pass
- class FullBisector(bisect.Bisector):
- """Test bisector to test bisect.py with"""
+ class FullBisector(run_bisect.Bisector):
+ """Test bisector to test run_bisect.py with"""
def __init__(self, options, overrides):
super(BisectTest.FullBisector, self).__init__(options, overrides)
@@ -81,14 +83,14 @@
return 0
def test_full_bisector(self):
- ret = bisect.Run(self.FullBisector({}, {}))
- self.assertEquals(ret, 0)
+ ret = run_bisect.Run(self.FullBisector({}, {}))
+ self.assertEqual(ret, 0)
self.assertFalse(os.path.exists(common.OBJECTS_FILE))
self.assertFalse(os.path.exists(common.WORKING_SET_FILE))
def check_output(self):
_, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
- ('grep "Bad items are: " logs/binary_search_tool_tester.py.out | '
+ ('grep "Bad items are: " logs/binary_search_tool_test.py.out | '
'tail -n1'))
ls = out.splitlines()
self.assertEqual(len(ls), 1)
@@ -113,7 +115,7 @@
"""Generate [100-1000] object files, and 1-5% of which are bad ones."""
GenObj()
- with open('./is_setup', 'w'):
+ with open('./is_setup', 'w', encoding='utf-8'):
pass
try:
@@ -146,7 +148,7 @@
test_script='./is_good.py',
prune=True,
file_args=True)
- self.assertEquals(ret, 0)
+ self.assertEqual(ret, 0)
self.check_output()
def test_arg_parse(self):
@@ -156,7 +158,7 @@
'--test_script', './is_good.py', '--prune', '--file_args'
]
ret = binary_search_state.Main(args)
- self.assertEquals(ret, 0)
+ self.assertEqual(ret, 0)
self.check_output()
def test_test_setup_script(self):
@@ -178,7 +180,7 @@
test_setup_script='./test_setup.py',
prune=True,
file_args=True)
- self.assertEquals(ret, 0)
+ self.assertEqual(ret, 0)
self.check_output()
def test_bad_test_setup_script(self):
@@ -196,15 +198,15 @@
state_file = binary_search_state.STATE_FILE
hidden_state_file = os.path.basename(binary_search_state.HIDDEN_STATE_FILE)
- with open(state_file, 'w') as f:
+ with open(state_file, 'w', encoding='utf-8') as f:
f.write('test123')
bss = binary_search_state.MockBinarySearchState()
with self.assertRaises(binary_search_state.Error):
bss.SaveState()
- with open(state_file, 'r') as f:
- self.assertEquals(f.read(), 'test123')
+ with open(state_file, 'r', encoding='utf-8') as f:
+ self.assertEqual(f.read(), 'test123')
os.remove(state_file)
@@ -243,9 +245,9 @@
bss = None
bss2 = binary_search_state.MockBinarySearchState.LoadState()
- self.assertEquals(bss2.all_items, test_items)
- self.assertEquals(bss2.currently_good_items, set([]))
- self.assertEquals(bss2.currently_bad_items, set([]))
+ self.assertEqual(bss2.all_items, test_items)
+ self.assertEqual(bss2.currently_good_items, set([]))
+ self.assertEqual(bss2.currently_bad_items, set([]))
def test_tmp_cleanup(self):
bss = binary_search_state.MockBinarySearchState(
@@ -255,14 +257,14 @@
bss.SwitchToGood(['0', '1', '2', '3'])
tmp_file = None
- with open('tmp_file', 'r') as f:
+ with open('tmp_file', 'r', encoding='utf-8') as f:
tmp_file = f.read()
os.remove('tmp_file')
self.assertFalse(os.path.exists(tmp_file))
ws = common.ReadWorkingSet()
for i in range(3):
- self.assertEquals(ws[i], 42)
+ self.assertEqual(ws[i], 42)
def test_verify_fail(self):
bss = binary_search_state.MockBinarySearchState(
@@ -298,11 +300,11 @@
prune=False,
file_args=True)
bss.DoSearchBadItems()
- self.assertEquals(len(bss.found_items), 1)
+ self.assertEqual(len(bss.found_items), 1)
bad_objs = common.ReadObjectsFile()
found_obj = int(bss.found_items.pop())
- self.assertEquals(bad_objs[found_obj], 1)
+ self.assertEqual(bad_objs[found_obj], 1)
def test_set_file(self):
binary_search_state.Run(
@@ -326,12 +328,12 @@
noincremental=True,
file_args=True,
verify=False)
- self.assertEquals(ret, 0)
+ self.assertEqual(ret, 0)
self.check_output()
def check_output(self):
_, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
- ('grep "Bad items are: " logs/binary_search_tool_tester.py.out | '
+ ('grep "Bad items are: " logs/binary_search_tool_test.py.out | '
'tail -n1'))
ls = out.splitlines()
self.assertEqual(len(ls), 1)
@@ -354,7 +356,7 @@
def check_pass_output(self, pass_name, pass_num, trans_num):
_, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
- ('grep "Bad pass: " logs/binary_search_tool_tester.py.out | '
+ ('grep "Bad pass: " logs/binary_search_tool_test.py.out | '
'tail -n1'))
ls = out.splitlines()
self.assertEqual(len(ls), 1)
@@ -365,7 +367,7 @@
_, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
('grep "Bad transformation number: '
- '" logs/binary_search_tool_tester.py.out | '
+ '" logs/binary_search_tool_test.py.out | '
'tail -n1'))
ls = out.splitlines()
self.assertEqual(len(ls), 1)
@@ -383,7 +385,7 @@
pass_bisect='./generate_cmd.py',
prune=True,
file_args=True)
- self.assertEquals(ret, 1)
+ self.assertEqual(ret, 1)
def test_gen_cmd_script(self):
bss = binary_search_state.MockBinarySearchState(
@@ -488,7 +490,7 @@
prune=True,
file_args=True,
verify=False)
- self.assertEquals(ret, 0)
+ self.assertEqual(ret, 0)
self.check_output()
def test_every_index_is_bad(self):
@@ -506,12 +508,12 @@
test_script='./is_good.py',
prune=True,
file_args=True)
- self.assertEquals(ret, 0)
+ self.assertEqual(ret, 0)
self.check_output()
def check_output(self):
_, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
- ('grep "Bad items are: " logs/binary_search_tool_tester.py.out | '
+ ('grep "Bad items are: " logs/binary_search_tool_test.py.out | '
'tail -n1'))
ls = out.splitlines()
self.assertEqual(len(ls), 1)
diff --git a/binary_search_tool/test/cmd_script.py b/binary_search_tool/test/cmd_script.py
index eb91fe9..bfd5605 100755
--- a/binary_search_tool/test/cmd_script.py
+++ b/binary_search_tool/test/cmd_script.py
@@ -1,4 +1,9 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Command script without compiler support for pass level bisection.
This script generates a pseudo log which a workable compiler should print out.
@@ -11,7 +16,7 @@
import os
import sys
-import common
+from binary_search_tool.test import common
def Main(argv):
diff --git a/binary_search_tool/test/cmd_script_no_support.py b/binary_search_tool/test/cmd_script_no_support.py
index a817f30..badbedc 100644
--- a/binary_search_tool/test/cmd_script_no_support.py
+++ b/binary_search_tool/test/cmd_script_no_support.py
@@ -1,3 +1,8 @@
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Command script without compiler support for pass level bisection.
This script generates a pseudo log when certain bisecting flags are not
@@ -13,8 +18,9 @@
def Main():
if not os.path.exists('./is_setup'):
return 1
- print('No support for -opt-bisect-limit or -print-debug-counter.',
- file=sys.stderr)
+ print(
+ 'No support for -opt-bisect-limit or -print-debug-counter.',
+ file=sys.stderr)
return 0
diff --git a/binary_search_tool/test/common.py b/binary_search_tool/test/common.py
index 5c3ff53..cf5300f 100755
--- a/binary_search_tool/test/common.py
+++ b/binary_search_tool/test/common.py
@@ -1,4 +1,9 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Common utility functions."""
DEFAULT_OBJECT_NUMBER = 1238
@@ -9,33 +14,29 @@
def ReadWorkingSet():
working_set = []
- f = open(WORKING_SET_FILE, 'r')
- for l in f:
- working_set.append(int(l))
- f.close()
+ with open(WORKING_SET_FILE, 'r', encoding='utf-8') as f:
+ for l in f:
+ working_set.append(int(l))
return working_set
def WriteWorkingSet(working_set):
- f = open(WORKING_SET_FILE, 'w')
- for o in working_set:
- f.write('{0}\n'.format(o))
- f.close()
+ with open(WORKING_SET_FILE, 'w', encoding='utf-8') as f:
+ for o in working_set:
+ f.write('{0}\n'.format(o))
def ReadObjectsFile():
objects_file = []
- f = open(OBJECTS_FILE, 'r')
- for l in f:
- objects_file.append(int(l))
- f.close()
+ with open(OBJECTS_FILE, 'r', encoding='utf-8') as f:
+ for l in f:
+ objects_file.append(int(l))
return objects_file
def ReadObjectIndex(filename):
object_index = []
- f = open(filename, 'r')
- for o in f:
- object_index.append(int(o))
- f.close()
+ with open(filename, 'r', encoding='utf-8') as f:
+ for o in f:
+ object_index.append(int(o))
return object_index
diff --git a/binary_search_tool/test/gen_init_list.py b/binary_search_tool/test/gen_init_list.py
index 002fc35..bc5dd8f 100755
--- a/binary_search_tool/test/gen_init_list.py
+++ b/binary_search_tool/test/gen_init_list.py
@@ -1,4 +1,9 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Prints out index for every object file, starting from 0."""
from __future__ import print_function
@@ -6,7 +11,7 @@
import sys
from cros_utils import command_executer
-import common
+from binary_search_tool.test import common
def Main():
diff --git a/binary_search_tool/test/gen_obj.py b/binary_search_tool/test/gen_obj.py
index a2bc7d9..4f65c71 100755
--- a/binary_search_tool/test/gen_obj.py
+++ b/binary_search_tool/test/gen_obj.py
@@ -1,4 +1,9 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Script to generate a list of object files.
0 represents a good object file.
@@ -12,7 +17,7 @@
import random
import sys
-import common
+from binary_search_tool.test import common
def Main(argv):
@@ -77,12 +82,11 @@
if os.path.isfile(common.WORKING_SET_FILE):
os.remove(common.WORKING_SET_FILE)
- f = open(common.OBJECTS_FILE, 'w')
- w = open(common.WORKING_SET_FILE, 'w')
- for i in obj_list:
- f.write('{0}\n'.format(i))
- w.write('{0}\n'.format(i))
- f.close()
+ with open(common.OBJECTS_FILE, 'w', encoding='utf-8') as f:
+ with open(common.WORKING_SET_FILE, 'w', encoding='utf-8') as w:
+ for i in obj_list:
+ f.write('{0}\n'.format(i))
+ w.write('{0}\n'.format(i))
obj_num = len(obj_list)
bad_obj_num = obj_list.count(1)
diff --git a/binary_search_tool/test/generate_cmd.py b/binary_search_tool/test/generate_cmd.py
index f6876ed..51b36b0 100755
--- a/binary_search_tool/test/generate_cmd.py
+++ b/binary_search_tool/test/generate_cmd.py
@@ -1,4 +1,9 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Generate a virtual cmd script for pass level bisection.
This is a required argument for pass level bisecting. For unit test, we use
@@ -15,7 +20,7 @@
if not os.path.exists('./is_setup'):
return 1
file_name = 'cmd_script.sh'
- with open(file_name, 'w') as f:
+ with open(file_name, 'w', encoding='utf-8') as f:
f.write('Generated by generate_cmd.py')
return 0
diff --git a/binary_search_tool/test/is_good.py b/binary_search_tool/test/is_good.py
index a0be4a0..662921e 100755
--- a/binary_search_tool/test/is_good.py
+++ b/binary_search_tool/test/is_good.py
@@ -1,4 +1,9 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Check to see if the working set produces a good executable."""
from __future__ import print_function
@@ -6,7 +11,7 @@
import os
import sys
-import common
+from binary_search_tool.test import common
def Main():
diff --git a/binary_search_tool/test/is_good_noinc_prune.py b/binary_search_tool/test/is_good_noinc_prune.py
index a900bd3..c0e42bb 100755
--- a/binary_search_tool/test/is_good_noinc_prune.py
+++ b/binary_search_tool/test/is_good_noinc_prune.py
@@ -1,4 +1,9 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Check to see if the working set produces a good executable.
This test script is made for the noincremental-prune test. This makes sure
@@ -12,16 +17,16 @@
import os
import sys
-import common
+from binary_search_tool.test import common
def Main():
working_set = common.ReadWorkingSet()
- with open('noinc_prune_good', 'r') as good_args:
+ with open('noinc_prune_good', 'r', encoding='utf-8') as good_args:
num_good_args = len(good_args.readlines())
- with open('noinc_prune_bad', 'r') as bad_args:
+ with open('noinc_prune_bad', 'r', encoding='utf-8') as bad_args:
num_bad_args = len(bad_args.readlines())
num_args = num_good_args + num_bad_args
diff --git a/binary_search_tool/test/switch_tmp.py b/binary_search_tool/test/switch_tmp.py
index 51b7110..0f3c423 100755
--- a/binary_search_tool/test/switch_tmp.py
+++ b/binary_search_tool/test/switch_tmp.py
@@ -1,4 +1,9 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Change portions of the object files to good.
This file is a test switch script. Used only for the test test_tmp_cleanup.
@@ -11,7 +16,7 @@
import sys
-import common
+from binary_search_tool.test import common
def Main(argv):
@@ -23,7 +28,7 @@
working_set[int(oi)] = 42
common.WriteWorkingSet(working_set)
- with open('tmp_file', 'w') as f:
+ with open('tmp_file', 'w', encoding='utf-8') as f:
f.write(argv[1])
return 0
diff --git a/binary_search_tool/test/switch_to_bad.py b/binary_search_tool/test/switch_to_bad.py
index a1b6bd5..e3553eb 100755
--- a/binary_search_tool/test/switch_to_bad.py
+++ b/binary_search_tool/test/switch_to_bad.py
@@ -1,11 +1,16 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Switch part of the objects file in working set to (possible) bad ones."""
from __future__ import print_function
import sys
-import common
+from binary_search_tool.test import common
def Main(argv):
diff --git a/binary_search_tool/test/switch_to_bad_noinc_prune.py b/binary_search_tool/test/switch_to_bad_noinc_prune.py
index db76aca..81b558e 100755
--- a/binary_search_tool/test/switch_to_bad_noinc_prune.py
+++ b/binary_search_tool/test/switch_to_bad_noinc_prune.py
@@ -1,4 +1,9 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Switch part of the objects file in working set to (possible) bad ones.
The "portion" is defined by the file (which is passed as the only argument to
@@ -18,7 +23,7 @@
import shutil
import sys
-import common
+from binary_search_tool.test import common
def Main(argv):
diff --git a/binary_search_tool/test/switch_to_bad_set_file.py b/binary_search_tool/test/switch_to_bad_set_file.py
index edf226d..5b941c6 100755
--- a/binary_search_tool/test/switch_to_bad_set_file.py
+++ b/binary_search_tool/test/switch_to_bad_set_file.py
@@ -1,4 +1,9 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Switch part of the objects file in working set to (possible) bad ones.
This script is meant to be specifically used with the set_file test. This uses
@@ -10,7 +15,7 @@
import os
import sys
-import common
+from binary_search_tool.test import common
def Main(_):
diff --git a/binary_search_tool/test/switch_to_good.py b/binary_search_tool/test/switch_to_good.py
index 59a118c..9747932 100755
--- a/binary_search_tool/test/switch_to_good.py
+++ b/binary_search_tool/test/switch_to_good.py
@@ -1,4 +1,9 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Change portions of the object files to good.
The "portion" is defined by the file (which is passed as the only argument to
@@ -10,7 +15,7 @@
import sys
-import common
+from binary_search_tool.test import common
def Main(argv):
diff --git a/binary_search_tool/test/switch_to_good_noinc_prune.py b/binary_search_tool/test/switch_to_good_noinc_prune.py
index 00488a7..0b91a0d 100755
--- a/binary_search_tool/test/switch_to_good_noinc_prune.py
+++ b/binary_search_tool/test/switch_to_good_noinc_prune.py
@@ -1,4 +1,9 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Change portions of the object files to good.
The "portion" is defined by the file (which is passed as the only argument to
@@ -18,7 +23,7 @@
import shutil
import sys
-import common
+from binary_search_tool.test import common
def Main(argv):
diff --git a/binary_search_tool/test/switch_to_good_set_file.py b/binary_search_tool/test/switch_to_good_set_file.py
index b5e521f..1cb05e0 100755
--- a/binary_search_tool/test/switch_to_good_set_file.py
+++ b/binary_search_tool/test/switch_to_good_set_file.py
@@ -1,4 +1,9 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Change portions of the object files to good.
The "portion" is defined by the file (which is passed as the only argument to
@@ -14,7 +19,7 @@
import os
import sys
-import common
+from binary_search_tool.test import common
def Main(_):
diff --git a/binary_search_tool/test/test_setup.py b/binary_search_tool/test/test_setup.py
index 0d6a410..ecc8eb9 100755
--- a/binary_search_tool/test/test_setup.py
+++ b/binary_search_tool/test/test_setup.py
@@ -1,4 +1,9 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Emulate running of test setup script, is_good.py should fail without this."""
from __future__ import print_function
@@ -8,7 +13,7 @@
def Main():
# create ./is_setup
- with open('./is_setup', 'w'):
+ with open('./is_setup', 'w', encoding='utf-8'):
pass
return 0
diff --git a/binary_search_tool/test/test_setup_bad.py b/binary_search_tool/test/test_setup_bad.py
index d715f57..cbca3c2 100755
--- a/binary_search_tool/test/test_setup_bad.py
+++ b/binary_search_tool/test/test_setup_bad.py
@@ -1,4 +1,9 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Emulate test setup that fails (i.e. failed flash to device)"""
from __future__ import print_function
diff --git a/build_chromeos.py b/build_chromeos.py
index 0b0676d..e275da1 100755
--- a/build_chromeos.py
+++ b/build_chromeos.py
@@ -1,6 +1,10 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
#
-# Copyright 2010 Google Inc. All Rights Reserved.
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Script to checkout the ChromeOS source.
This script sets up the ChromeOS source in the given directory, matching a
@@ -92,8 +96,8 @@
dest='debug',
default=False,
action='store_true',
- help=("Optional. Build chrome browser with \"-g -O0\". "
- "Notice, this also turns on \'--dev\'. "
+ help=('Optional. Build chrome browser with "-g -O0". '
+ "Notice, this also turns on '--dev'. "
'Defaults to False.'))
parser.add_argument(
'--env', dest='env', default='', help='Env to pass to build_packages.')
@@ -133,7 +137,7 @@
'This flags is used internally by this script. '
'Contact the author for more detail.'))
- if options.rebuild == True:
+ if options.rebuild:
build_packages_env += ' EXTRA_BOARD_FLAGS=-e'
# EXTRA_BOARD_FLAGS=-e should clean up the object files for the chrome
# browser but it doesn't. So do it here.
@@ -166,10 +170,9 @@
if not os.path.isdir(options.chromeos_root + '/chroot/build/' +
options.board) or options.clobber_board:
# Run build_tc.py from binary package
- ret = cmd_executer.ChrootRunCommand(options.chromeos_root,
- misc.GetSetupBoardCommand(
- options.board,
- force=options.clobber_board))
+ ret = cmd_executer.ChrootRunCommand(
+ options.chromeos_root,
+ misc.GetSetupBoardCommand(options.board, force=options.clobber_board))
logger.GetLogger().LogFatalIf(ret, 'setup_board failed')
else:
logger.GetLogger().LogOutput('Did not setup_board '
@@ -179,23 +182,23 @@
# Perform 2-step build_packages to build a debug chrome browser.
# Firstly, build everything that chromeos-chrome depends on normally.
- if options.rebuild == True:
+ if options.rebuild:
# Give warning about "--rebuild" and "--debug". Under this combination,
# only dependencies of "chromeos-chrome" get rebuilt.
logger.GetLogger().LogWarning(
- "\"--rebuild\" does not correctly re-build every package when "
- "\"--debug\" is enabled. ")
+ '--rebuild" does not correctly re-build every package when '
+ '"--debug" is enabled. ')
# Replace EXTRA_BOARD_FLAGS=-e with "-e --onlydeps"
build_packages_env = build_packages_env.replace(
- 'EXTRA_BOARD_FLAGS=-e', 'EXTRA_BOARD_FLAGS=\"-e --onlydeps\"')
+ 'EXTRA_BOARD_FLAGS=-e', 'EXTRA_BOARD_FLAGS="-e --onlydeps"')
else:
build_packages_env += ' EXTRA_BOARD_FLAGS=--onlydeps'
ret = cmd_executer.ChrootRunCommand(
- options.chromeos_root, "CFLAGS=\"$(portageq-%s envvar CFLAGS) %s\" "
- "CXXFLAGS=\"$(portageq-%s envvar CXXFLAGS) %s\" "
- "LDFLAGS=\"$(portageq-%s envvar LDFLAGS) %s\" "
+ options.chromeos_root, 'CFLAGS="$(portageq-%s envvar CFLAGS) %s" '
+ 'CXXFLAGS="$(portageq-%s envvar CXXFLAGS) %s" '
+ 'LDFLAGS="$(portageq-%s envvar LDFLAGS) %s" '
'CHROME_ORIGIN=SERVER_SOURCE '
'%s '
'%s --skip_chroot_upgrade'
@@ -208,16 +211,16 @@
# Secondly, build chromeos-chrome using debug mode.
# Replace '--onlydeps' with '--nodeps'.
- if options.rebuild == True:
+ if options.rebuild:
build_packages_env = build_packages_env.replace(
- 'EXTRA_BOARD_FLAGS=\"-e --onlydeps\"', 'EXTRA_BOARD_FLAGS=--nodeps')
+ 'EXTRA_BOARD_FLAGS="-e --onlydeps"', 'EXTRA_BOARD_FLAGS=--nodeps')
else:
build_packages_env = build_packages_env.replace(
'EXTRA_BOARD_FLAGS=--onlydeps', 'EXTRA_BOARD_FLAGS=--nodeps')
ret = cmd_executer.ChrootRunCommand(
- options.chromeos_root, "CFLAGS=\"$(portageq-%s envvar CFLAGS) %s\" "
- "CXXFLAGS=\"$(portageq-%s envvar CXXFLAGS) %s\" "
- "LDFLAGS=\"$(portageq-%s envvar LDFLAGS) %s\" "
+ options.chromeos_root, 'CFLAGS="$(portageq-%s envvar CFLAGS) %s" '
+ 'CXXFLAGS="$(portageq-%s envvar CXXFLAGS) %s" '
+ 'LDFLAGS="$(portageq-%s envvar LDFLAGS) %s" '
'CHROME_ORIGIN=SERVER_SOURCE BUILDTYPE=Debug '
'%s '
'%s --skip_chroot_upgrade'
@@ -237,11 +240,11 @@
# Up to now, we have a debug built chromos-chrome browser.
# Fall through to build the rest of the world.
- # Build packages
+ # Build packages
ret = cmd_executer.ChrootRunCommand(
- options.chromeos_root, "CFLAGS=\"$(portageq-%s envvar CFLAGS) %s\" "
- "CXXFLAGS=\"$(portageq-%s envvar CXXFLAGS) %s\" "
- "LDFLAGS=\"$(portageq-%s envvar LDFLAGS) %s\" "
+ options.chromeos_root, 'CFLAGS="$(portageq-%s envvar CFLAGS) %s" '
+ 'CXXFLAGS="$(portageq-%s envvar CXXFLAGS) %s" '
+ 'LDFLAGS="$(portageq-%s envvar LDFLAGS) %s" '
'CHROME_ORIGIN=SERVER_SOURCE '
'%s '
'%s --skip_chroot_upgrade' %
@@ -261,19 +264,18 @@
flags_file_name = 'flags.txt'
flags_file_path = ('%s/src/build/images/%s/latest/%s' %
(options.chromeos_root, options.board, flags_file_name))
- flags_file = open(flags_file_path, 'wb')
- flags_file.write('CFLAGS=%s\n' % options.cflags)
- flags_file.write('CXXFLAGS=%s\n' % options.cxxflags)
- flags_file.write('LDFLAGS=%s\n' % options.ldflags)
- flags_file.close()
+ with open(flags_file_path, 'w', encoding='utf-8') as flags_file:
+ flags_file.write('CFLAGS=%s\n' % options.cflags)
+ flags_file.write('CXXFLAGS=%s\n' % options.cxxflags)
+ flags_file.write('LDFLAGS=%s\n' % options.ldflags)
if options.label:
image_dir_path = ('%s/src/build/images/%s/latest' % (options.chromeos_root,
options.board))
real_image_dir_path = os.path.realpath(image_dir_path)
- command = ('ln -sf -T %s %s/%s' %
- (os.path.basename(real_image_dir_path),
- os.path.dirname(real_image_dir_path), options.label))
+ command = ('ln -sf -T %s %s/%s' % (os.path.basename(real_image_dir_path),
+ os.path.dirname(real_image_dir_path),
+ options.label))
ret = cmd_executer.RunCommand(command)
logger.GetLogger().LogFatalIf(
diff --git a/build_tc.py b/build_tc.py
index c14b690..9b90f55 100755
--- a/build_tc.py
+++ b/build_tc.py
@@ -1,8 +1,9 @@
-#!/usr/bin/env python2
-#
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
# Copyright 2010 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+
"""Script to build the ChromeOS toolchain.
This script sets up the toolchain if you give it the gcctools directory.
@@ -100,8 +101,9 @@
command = 'mkdir -p %s' % build_dir
self._ce.RunCommand(command)
- mounted_build_dir = os.path.join(self._chromeos_root, 'chroot', '%s-%s' %
- (self._chroot_source_path, build_suffix))
+ mounted_build_dir = os.path.join(
+ self._chromeos_root, 'chroot',
+ '%s-%s' % (self._chroot_source_path, build_suffix))
build_mp = tc_enter_chroot.MountPoint(build_dir, mounted_build_dir,
getpass.getuser())
mount_points.append(build_mp)
@@ -148,10 +150,10 @@
if self._name == 'gcc' and not self._gcc_enable_ccache:
env['USE'] += ' -wrapper_ccache'
- env['%s_SOURCE_PATH' % self._name.upper()] = (os.path.join(
- '/', self._chroot_source_path))
+ env['%s_SOURCE_PATH' % self._name.upper()] = (
+ os.path.join('/', self._chroot_source_path))
env['ACCEPT_KEYWORDS'] = '~*'
- env_string = ' '.join(["%s=\"%s\"" % var for var in env.items()])
+ env_string = ' '.join(['%s="%s"' % var for var in env.items()])
command = 'emerge =cross-%s/%s-9999' % (self._ctarget, self._name)
full_command = 'sudo %s %s' % (env_string, command)
rv = self._ce.ChrootRunCommand(self._chromeos_root, full_command)
diff --git a/buildbot_test_llvm.py b/buildbot_test_llvm.py
index 7e7c6e6..968c67b 100755
--- a/buildbot_test_llvm.py
+++ b/buildbot_test_llvm.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2017 The Chromium OS Authors. All rights reserved.
@@ -125,7 +125,7 @@
"""Write data for waterfall report."""
fname = '%d-%02d-%02d.builds' % (date.year, date.month, date.day)
filename = os.path.join(DATA_DIR, 'rotating-builders', fname)
- with open(filename, 'w') as out_file:
+ with open(filename, 'w', encoding='utf-8') as out_file:
for board in results_dict.keys():
buildbucket_id = results_dict[board]
out_file.write('%s,%s\n' % (buildbucket_id, board))
@@ -172,7 +172,7 @@
if options.board:
fv = ToolchainVerifier(options.board, options.chromeos_root,
options.weekday, options.patches, options.compiler)
- return fv.Doall()
+ return fv.DoAll()
today = datetime.date.today()
delta = today - START_DATE
@@ -190,7 +190,7 @@
results_dict[board] = buildbucket_id
except SystemExit:
logfile = os.path.join(VALIDATION_RESULT_DIR, options.compiler, board)
- with open(logfile, 'w') as f:
+ with open(logfile, 'w', encoding='utf-8') as f:
f.write('Verifier got an exception, please check the log.\n')
WriteRotatingReportsData(results_dict, today)
diff --git a/buildbot_test_toolchains.py b/buildbot_test_toolchains.py
index 175dae5..5f7c21d 100755
--- a/buildbot_test_toolchains.py
+++ b/buildbot_test_toolchains.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2016 The Chromium OS Authors. All rights reserved.
@@ -34,11 +34,12 @@
USE_LLVM_NEXT_PATCH = '513590'
CROSTC_ROOT = '/usr/local/google/crostc'
+NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly-tests')
ROLE_ACCOUNT = 'mobiletc-prebuild'
TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
MAIL_PROGRAM = '~/var/bin/mail-sheriff'
PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives')
-NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports')
+NIGHTLY_TESTS_RESULTS = os.path.join(CROSTC_ROOT, 'nightly_test_reports')
IMAGE_DIR = '{board}-{image_type}'
IMAGE_VERSION_STR = r'{chrome_version}-{tip}\.{branch}\.{branch_branch}'
@@ -87,7 +88,7 @@
timestamp = datetime.datetime.strftime(datetime.datetime.now(),
'%Y-%m-%d_%H:%M:%S')
self._reports_dir = os.path.join(
- NIGHTLY_TESTS_DIR,
+ NIGHTLY_TESTS_RESULTS,
'%s.%s' % (timestamp, board),
)
@@ -145,7 +146,7 @@
Given the names of the trybot, vanilla and non-AFDO images, create the
appropriate crosperf experiment file and launch crosperf on it.
"""
- experiment_file_dir = os.path.join(CROSTC_ROOT, self._weekday)
+ experiment_file_dir = os.path.join(NIGHTLY_TESTS_DIR, self._weekday)
experiment_file_name = '%s_toolchain_experiment.txt' % self._board
compiler_string = 'llvm'
@@ -194,7 +195,7 @@
}
"""
- with open(experiment_file, 'w') as f:
+ with open(experiment_file, 'w', encoding='utf-8') as f:
f.write(experiment_header)
f.write(experiment_tests)
@@ -250,11 +251,10 @@
ret = self._ce.RunCommand(command)
if ret != 0:
raise RuntimeError('Crosperf execution error!')
- else:
- # Copy json report to pending archives directory.
- command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR)
- ret = self._ce.RunCommand(command)
- return
+
+ # Copy json report to pending archives directory.
+ command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR)
+ ret = self._ce.RunCommand(command)
def _SendEmail(self):
"""Find email message generated by crosperf and send it."""
diff --git a/chromiumos_image_diff.py b/chromiumos_image_diff.py
index 74906d3..66a54cc 100755
--- a/chromiumos_image_diff.py
+++ b/chromiumos_image_diff.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 The Chromium OS Authors. All rights reserved.
@@ -86,7 +86,7 @@
'{r}/var {r}/mnt/stateful_partition {r}; sudo umount {s} ; '
'rmdir {r} ; rmdir {s}\n').format(
r=self.rootfs, s=self.stateful)
- f = open(self.unmount_script, 'w')
+ f = open(self.unmount_script, 'w', encoding='utf-8')
f.write(command)
f.close()
self._ce.RunCommand(
@@ -160,9 +160,9 @@
i1 = self.images[0]
i2 = self.images[1]
t1 = i1.rootfs + '/'
- elfset1 = set([e.replace(t1, '') for e in i1.elf_files])
+ elfset1 = {e.replace(t1, '') for e in i1.elf_files}
t2 = i2.rootfs + '/'
- elfset2 = set([e.replace(t2, '') for e in i2.elf_files])
+ elfset2 = {e.replace(t2, '') for e in i2.elf_files}
dif1 = elfset1.difference(elfset2)
msg = None
if dif1:
@@ -210,15 +210,15 @@
if full_path1 == full_path2:
self.logger.LogError(
- 'Error: We\'re comparing the SAME file - {0}'.format(f1))
+ "Error: We're comparing the SAME file - {0}".format(f1))
continue
command = (
'objdump -d "{f1}" > {tempf1} ; '
'objdump -d "{f2}" > {tempf2} ; '
# Remove path string inside the dissemble
- 'sed -i \'s!{rootfs1}!!g\' {tempf1} ; '
- 'sed -i \'s!{rootfs2}!!g\' {tempf2} ; '
+ "sed -i 's!{rootfs1}!!g' {tempf1} ; "
+ "sed -i 's!{rootfs2}!!g' {tempf2} ; "
'diff {tempf1} {tempf2} 1>/dev/null 2>&1').format(
f1=full_path1,
f2=full_path2,
diff --git a/compiler_wrapper/build.py b/compiler_wrapper/build.py
index 763b3e6..037b940 100755
--- a/compiler_wrapper/build.py
+++ b/compiler_wrapper/build.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -39,10 +39,16 @@
'-X',
'main.Version=' + version,
]
+
+ # If the wrapper is intended for Chrome OS, we need to use libc's exec.
+ extra_args = []
+ if 'cros' in args.config:
+ extra_args = ['-tags', 'libc_exec']
+
return [
'go', 'build', '-o',
os.path.abspath(args.output_file), '-ldflags', ' '.join(ldFlags)
- ]
+ ] + extra_args
def read_version(build_dir):
@@ -52,7 +58,7 @@
return r.read()
last_commit_msg = subprocess.check_output(
- ['git', '-C', build_dir, 'log', '-1', '--pretty=%B'])
+ ['git', '-C', build_dir, 'log', '-1', '--pretty=%B'], encoding='utf-8')
# Use last found change id to support reverts as well.
change_ids = re.findall(r'Change-Id: (\w+)', last_commit_msg)
if not change_ids:
diff --git a/compiler_wrapper/bundle.py b/compiler_wrapper/bundle.py
index c1fa53e..173625f 100755
--- a/compiler_wrapper/bundle.py
+++ b/compiler_wrapper/bundle.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -32,7 +32,7 @@
def read_change_id(input_dir):
last_commit_msg = subprocess.check_output(
- ['git', '-C', input_dir, 'log', '-1', '--pretty=%B'])
+ ['git', '-C', input_dir, 'log', '-1', '--pretty=%B'], encoding='utf-8')
# Use last found change id to support reverts as well.
change_ids = re.findall(r'Change-Id: (\w+)', last_commit_msg)
if not change_ids:
@@ -41,14 +41,15 @@
def write_readme(input_dir, output_dir, change_id):
- with open(os.path.join(input_dir, 'bundle.README'), 'r') as r, \
- open(os.path.join(output_dir, 'README'), 'w') as w:
- content = r.read()
- w.write(content.format(change_id=change_id))
+ with open(
+ os.path.join(input_dir, 'bundle.README'), 'r', encoding='utf-8') as r:
+ with open(os.path.join(output_dir, 'README'), 'w', encoding='utf-8') as w:
+ content = r.read()
+ w.write(content.format(change_id=change_id))
def write_version(output_dir, change_id):
- with open(os.path.join(output_dir, 'VERSION'), 'w') as w:
+ with open(os.path.join(output_dir, 'VERSION'), 'w', encoding='utf-8') as w:
w.write(change_id)
diff --git a/compiler_wrapper/env.go b/compiler_wrapper/env.go
index 4d83b17..56b3a91 100644
--- a/compiler_wrapper/env.go
+++ b/compiler_wrapper/env.go
@@ -71,7 +71,7 @@
}
func (env *processEnv) exec(cmd *command) error {
- return libcExec(env, cmd)
+ return execCmd(env, cmd)
}
func (env *processEnv) run(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error {
diff --git a/compiler_wrapper/go_exec.go b/compiler_wrapper/go_exec.go
new file mode 100644
index 0000000..2f2e5ad
--- /dev/null
+++ b/compiler_wrapper/go_exec.go
@@ -0,0 +1,23 @@
+// Copyright 2020 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// +build !libc_exec
+
+package main
+
+import (
+ "os/exec"
+ "syscall"
+)
+
+// Implement exec for users that don't need to dynamically link with glibc
+// See b/144783188 and libc_exec.go.
+
+func execCmd(env env, cmd *command) error {
+ execCmd := exec.Command(cmd.Path, cmd.Args...)
+ mergedEnv := mergeEnvValues(env.environ(), cmd.EnvUpdates)
+
+ ret := syscall.Exec(execCmd.Path, execCmd.Args, mergedEnv)
+ return newErrorwithSourceLocf("exec error: %v", ret)
+}
diff --git a/compiler_wrapper/goldenutil_test.go b/compiler_wrapper/goldenutil_test.go
index 23e003c..2b391d7 100644
--- a/compiler_wrapper/goldenutil_test.go
+++ b/compiler_wrapper/goldenutil_test.go
@@ -14,7 +14,6 @@
"os"
"path/filepath"
"regexp"
- "runtime"
"strings"
)
@@ -128,6 +127,11 @@
}
cmdResult := record.Cmds[len(newCmds)]
cmdResult.Cmd = cmd
+ if numEnvUpdates := len(cmdResult.Cmd.EnvUpdates); numEnvUpdates > 0 {
+ if strings.HasPrefix(cmdResult.Cmd.EnvUpdates[numEnvUpdates-1], "PYTHONPATH") {
+ cmdResult.Cmd.EnvUpdates[numEnvUpdates-1] = "PYTHONPATH=/somepath/test_binary"
+ }
+ }
newCmds = append(newCmds, cmdResult)
io.WriteString(stdout, cmdResult.Stdout)
io.WriteString(stderr, cmdResult.Stderr)
@@ -146,13 +150,7 @@
// Create an empty wrapper at the given path.
// Needed as we are resolving symlinks which stats the wrapper file.
ctx.writeFile(record.WrapperCmd.Cmd.Path, "")
- // Assign a fixed path to os.Args[0] to pass the test.
- tmp := os.Args[0]
- // callCompiler verifies os.Args[0] exists, so use a real file.
- _, file, _, _ := runtime.Caller(1)
- os.Args[0] = file
record.WrapperCmd.ExitCode = callCompiler(ctx, ctx.cfg, record.WrapperCmd.Cmd)
- os.Args[0] = tmp
if hasInternalError(ctx.stderrString()) {
ctx.t.Errorf("found an internal error for wrapperCmd %#v and env #%v. Got: %s",
record.WrapperCmd.Cmd, record.Env, ctx.stderrString())
diff --git a/compiler_wrapper/libc_exec.go b/compiler_wrapper/libc_exec.go
index 268221d..f8db9d8 100644
--- a/compiler_wrapper/libc_exec.go
+++ b/compiler_wrapper/libc_exec.go
@@ -2,6 +2,8 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
+// +build libc_exec
+
package main
// #include <errno.h>
@@ -26,7 +28,10 @@
// LD_PRELOAD to work properly (e.g. gentoo sandbox).
// Note that this changes the go binary to be a dynamically linked one.
// See crbug.com/1000863 for details.
-func libcExec(env env, cmd *command) error {
+// To use this version of exec, please add '-tags libc_exec' when building Go binary.
+// Without the tags, libc_exec.go will be used.
+
+func execCmd(env env, cmd *command) error {
freeList := []unsafe.Pointer{}
defer func() {
for _, ptr := range freeList {
diff --git a/compiler_wrapper/testdata/android_golden/bisect.json b/compiler_wrapper/testdata/android_golden/bisect.json
index be07da6..25df7f1 100644
--- a/compiler_wrapper/testdata/android_golden/bisect.json
+++ b/compiler_wrapper/testdata/android_golden/bisect.json
@@ -27,7 +27,7 @@
"main.cc"
],
"env_updates": [
- "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper"
+ "PYTHONPATH=/somepath/test_binary"
]
}
}
@@ -62,7 +62,7 @@
"main.cc"
],
"env_updates": [
- "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper"
+ "PYTHONPATH=/somepath/test_binary"
]
}
}
@@ -100,7 +100,7 @@
"main.cc"
],
"env_updates": [
- "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper"
+ "PYTHONPATH=/somepath/test_binary"
]
},
"stdout": "somemessage",
diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json b/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json
index ad4feac..28adfee 100644
--- a/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json
+++ b/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json
@@ -41,7 +41,7 @@
"-Wno-implicit-int-float-conversion"
],
"env_updates": [
- "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper"
+ "PYTHONPATH=/somepath/test_binary"
]
}
}
@@ -90,7 +90,7 @@
"-Wno-implicit-int-float-conversion"
],
"env_updates": [
- "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper"
+ "PYTHONPATH=/somepath/test_binary"
]
}
}
@@ -142,7 +142,7 @@
"-Wno-implicit-int-float-conversion"
],
"env_updates": [
- "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper"
+ "PYTHONPATH=/somepath/test_binary"
]
},
"stdout": "somemessage",
diff --git a/compiler_wrapper/testdata/cros_hardened_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_golden/bisect.json
index 6e1d498..b00f974 100644
--- a/compiler_wrapper/testdata/cros_hardened_golden/bisect.json
+++ b/compiler_wrapper/testdata/cros_hardened_golden/bisect.json
@@ -55,7 +55,7 @@
"CCACHE_DIR=/var/cache/distfiles/ccache",
"CCACHE_UMASK=002",
"CCACHE_CPP2=yes",
- "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper"
+ "PYTHONPATH=/somepath/test_binary"
]
}
}
@@ -118,7 +118,7 @@
"CCACHE_DIR=/var/cache/distfiles/ccache",
"CCACHE_UMASK=002",
"CCACHE_CPP2=yes",
- "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper"
+ "PYTHONPATH=/somepath/test_binary"
]
}
}
@@ -184,7 +184,7 @@
"CCACHE_DIR=/var/cache/distfiles/ccache",
"CCACHE_UMASK=002",
"CCACHE_CPP2=yes",
- "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper"
+ "PYTHONPATH=/somepath/test_binary"
]
},
"stdout": "somemessage",
diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json
index 6e1d498..b00f974 100644
--- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json
+++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json
@@ -55,7 +55,7 @@
"CCACHE_DIR=/var/cache/distfiles/ccache",
"CCACHE_UMASK=002",
"CCACHE_CPP2=yes",
- "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper"
+ "PYTHONPATH=/somepath/test_binary"
]
}
}
@@ -118,7 +118,7 @@
"CCACHE_DIR=/var/cache/distfiles/ccache",
"CCACHE_UMASK=002",
"CCACHE_CPP2=yes",
- "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper"
+ "PYTHONPATH=/somepath/test_binary"
]
}
}
@@ -184,7 +184,7 @@
"CCACHE_DIR=/var/cache/distfiles/ccache",
"CCACHE_UMASK=002",
"CCACHE_CPP2=yes",
- "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper"
+ "PYTHONPATH=/somepath/test_binary"
]
},
"stdout": "somemessage",
diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json
index 3fafcf0..5075f84 100644
--- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json
+++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json
@@ -50,7 +50,7 @@
"x86_64-cros-linux-gnu"
],
"env_updates": [
- "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper"
+ "PYTHONPATH=/somepath/test_binary"
]
}
}
@@ -108,7 +108,7 @@
"x86_64-cros-linux-gnu"
],
"env_updates": [
- "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper"
+ "PYTHONPATH=/somepath/test_binary"
]
}
}
@@ -169,7 +169,7 @@
"x86_64-cros-linux-gnu"
],
"env_updates": [
- "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper"
+ "PYTHONPATH=/somepath/test_binary"
]
},
"stdout": "somemessage",
diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json b/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json
index 7d2acda..cb5dea0 100644
--- a/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json
+++ b/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json
@@ -47,7 +47,7 @@
"CCACHE_DIR=/var/cache/distfiles/ccache",
"CCACHE_UMASK=002",
"CCACHE_CPP2=yes",
- "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper"
+ "PYTHONPATH=/somepath/test_binary"
]
}
}
@@ -102,7 +102,7 @@
"CCACHE_DIR=/var/cache/distfiles/ccache",
"CCACHE_UMASK=002",
"CCACHE_CPP2=yes",
- "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper"
+ "PYTHONPATH=/somepath/test_binary"
]
}
}
@@ -160,7 +160,7 @@
"CCACHE_DIR=/var/cache/distfiles/ccache",
"CCACHE_UMASK=002",
"CCACHE_CPP2=yes",
- "PYTHONPATH=/media/storage/jiancai/chromeos/src/third_party/toolchain-utils/compiler_wrapper"
+ "PYTHONPATH=/somepath/test_binary"
]
},
"stdout": "somemessage",
diff --git a/compiler_wrapper/update_compiler_wrapper.sh b/compiler_wrapper/update_compiler_wrapper.sh
new file mode 100755
index 0000000..93de2be
--- /dev/null
+++ b/compiler_wrapper/update_compiler_wrapper.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# This script rebuilds and installs compiler wrappers
+
+if [[ ! -e /etc/cros_chroot_version ]]; then
+ echo "Please run this script inside chroot"
+ exit 1
+fi
+set -e
+cd "$(dirname "$(readlink -m "$0")")"
+echo "Updated files:"
+# Update the host wrapper
+./build.py --config=cros.host --use_ccache=false --use_llvm_next=false --output_file=./clang_host_wrapper
+sudo mv ./clang_host_wrapper /usr/bin/clang_host_wrapper
+echo "/usr/bin/clang_host_wrapper"
+sudo cp ../binary_search_tool/bisect_driver.py /usr/bin
+echo "/usr/bin/clang_host_wrapper/bisect_driver.py"
+# Update the target wrappers
+for GCC in cross-x86_64-cros-linux-gnu/gcc cross-armv7a-cros-linux-gnueabihf/gcc cross-aarch64-cros-linux-gnu/gcc; do
+ FILES="$(equery f $GCC)"
+ ./build.py --config=cros.hardened --use_ccache=false --use_llvm_next=false --output_file=./sysroot_wrapper.hardened.noccache
+ sudo mv ./sysroot_wrapper.hardened.noccache "$(grep sysroot_wrapper.hardened.noccache <<< "${FILES}")"
+ echo "$(grep sysroot_wrapper.hardened.noccache <<< "${FILES}")"
+ ./build.py --config=cros.hardened --use_ccache=true --use_llvm_next=false --output_file=./sysroot_wrapper.hardened.ccache
+ sudo mv ./sysroot_wrapper.hardened.ccache "$(grep sysroot_wrapper.hardened.ccache <<< "${FILES}")"
+ echo "$(grep sysroot_wrapper.hardened.ccache <<< "${FILES}")"
+ sudo cp ../binary_search_tool/bisect_driver.py "$(grep bisect_driver.py <<< "${FILES}")"
+ echo "$(grep bisect_driver.py <<< "${FILES}")"
+done
diff --git a/command_executer_timeout_test.py b/cros_utils/command_executer_timeout_test.py
similarity index 72%
rename from command_executer_timeout_test.py
rename to cros_utils/command_executer_timeout_test.py
index 26f3933..1c9c74c 100755
--- a/command_executer_timeout_test.py
+++ b/cros_utils/command_executer_timeout_test.py
@@ -1,6 +1,10 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
#
-# Copyright 2010 Google Inc. All Rights Reserved.
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Timeout test for command_executer."""
from __future__ import print_function
diff --git a/cros_utils/command_executer_unittest.py b/cros_utils/command_executer_unittest.py
index 959000f..22331ae 100755
--- a/cros_utils/command_executer_unittest.py
+++ b/cros_utils/command_executer_unittest.py
@@ -11,7 +11,7 @@
import time
import unittest
-import command_executer
+from cros_utils import command_executer
class CommandExecuterTest(unittest.TestCase):
diff --git a/cros_utils/email_sender.py b/cros_utils/email_sender.py
index 0019982..6b8893e 100755
--- a/cros_utils/email_sender.py
+++ b/cros_utils/email_sender.py
@@ -9,17 +9,35 @@
from __future__ import print_function
+import base64
+import contextlib
+import datetime
+import getpass
+import json
+import os
+import smtplib
+import tempfile
from email import encoders as Encoders
from email.mime.base import MIMEBase
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
-import getpass
-import os
-import smtplib
-import tempfile
from cros_utils import command_executer
+X20_PATH = '/google/data/rw/teams/c-compiler-chrome/prod_emails'
+
+
+@contextlib.contextmanager
+def AtomicallyWriteFile(file_path):
+ temp_path = file_path + '.in_progress'
+ try:
+ with open(temp_path, 'w') as f:
+ yield f
+ os.rename(temp_path, file_path)
+ except:
+ os.remove(temp_path)
+ raise
+
class EmailSender(object):
"""Utility class to send email through SMTP or SendGMR."""
@@ -31,6 +49,93 @@
self.name = name
self.content = content
+ def SendX20Email(self,
+ subject,
+ identifier,
+ well_known_recipients=(),
+ direct_recipients=(),
+ text_body=None,
+ html_body=None):
+ """Enqueues an email in our x20 outbox.
+
+ These emails ultimately get sent by the machinery in
+ //depot/google3/googleclient/chrome/chromeos_toolchain/mailer/mail.go. This
+ kind of sending is intended for accounts that don't have smtp or gmr access
+ (e.g., role accounts), but can be used by anyone with x20 access.
+
+ All emails are sent from `mdb.c-compiler-chrome+${identifier}@google.com`.
+
+ Args:
+ subject: email subject. Must be nonempty.
+ identifier: email identifier, or the text that lands after the `+` in the
+ "From" email address. Must be nonempty.
+ well_known_recipients: a list of well-known recipients for the email.
+ These are translated into addresses by our mailer.
+ Current potential values for this are ('sheriff',
+ 'cwp-team', 'cros-team', 'mage'). Either this or
+ direct_recipients must be a nonempty list.
+ direct_recipients: @google.com emails to send addresses to. Either this
+ or well_known_recipients must be a nonempty list.
+ text_body: a 'text/plain' email body to send. Either this or html_body
+ must be a nonempty string. Both may be specified
+ html_body: a 'text/html' email body to send. Either this or text_body
+ must be a nonempty string. Both may be specified
+ """
+ # `str`s act a lot like tuples/lists. Ensure that we're not accidentally
+ # iterating over one of those (or anything else that's sketchy, for that
+ # matter).
+ if not isinstance(well_known_recipients, (tuple, list)):
+ raise ValueError('`well_known_recipients` is unexpectedly a %s' %
+ type(well_known_recipients))
+
+ if not isinstance(direct_recipients, (tuple, list)):
+ raise ValueError(
+ '`direct_recipients` is unexpectedly a %s' % type(direct_recipients))
+
+ if not subject or not identifier:
+ raise ValueError('both `subject` and `identifier` must be nonempty')
+
+ if not (well_known_recipients or direct_recipients):
+ raise ValueError('either `well_known_recipients` or `direct_recipients` '
+ 'must be specified')
+
+ for recipient in direct_recipients:
+ if not recipient.endswith('@google.com'):
+ raise ValueError('All recipients must end with @google.com')
+
+ if not (text_body or html_body):
+ raise ValueError('either `text_body` or `html_body` must be specified')
+
+ email_json = {
+ 'email_identifier': identifier,
+ 'subject': subject,
+ }
+
+ if well_known_recipients:
+ email_json['well_known_recipients'] = well_known_recipients
+
+ if direct_recipients:
+ email_json['direct_recipients'] = direct_recipients
+
+ if text_body:
+ email_json['body'] = text_body
+
+ if html_body:
+ email_json['html_body'] = html_body
+
+ # The name of this has two parts:
+ # - An easily sortable time, to provide uniqueness and let our emailer
+ # send things in the order they were put into the outbox.
+ # - 64 bits of entropy, so two racing email sends don't clobber the same
+ # file.
+ now = datetime.datetime.utcnow().isoformat('T', 'seconds') + 'Z'
+ entropy = base64.urlsafe_b64encode(os.getrandom(8))
+ entropy_str = entropy.rstrip(b'=').decode('utf-8')
+ result_path = os.path.join(X20_PATH, now + '_' + entropy_str + '.json')
+
+ with AtomicallyWriteFile(result_path) as f:
+ json.dump(email_json, f)
+
def SendEmail(self,
email_to,
subject,
diff --git a/cros_utils/email_sender_unittest.py b/cros_utils/email_sender_unittest.py
new file mode 100755
index 0000000..7349219
--- /dev/null
+++ b/cros_utils/email_sender_unittest.py
@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Tests for email_sender."""
+
+from __future__ import print_function
+
+import contextlib
+import io
+import json
+import unittest
+import unittest.mock as mock
+
+import cros_utils.email_sender as email_sender
+
+
+class Test(unittest.TestCase):
+ """Tests for email_sender."""
+
+ @mock.patch('cros_utils.email_sender.AtomicallyWriteFile')
+ def test_x20_email_sending_rejects_invalid_inputs(self, write_file):
+ test_cases = [
+ {
+ # no subject
+ 'subject': '',
+ 'identifier': 'foo',
+ 'direct_recipients': ['gbiv@google.com'],
+ 'text_body': 'hi',
+ },
+ {
+ 'subject': 'foo',
+ # no identifier
+ 'identifier': '',
+ 'direct_recipients': ['gbiv@google.com'],
+ 'text_body': 'hi',
+ },
+ {
+ 'subject': 'foo',
+ 'identifier': 'foo',
+ # no recipients
+ 'direct_recipients': [],
+ 'text_body': 'hi',
+ },
+ {
+ 'subject': 'foo',
+ 'identifier': 'foo',
+ 'direct_recipients': ['gbiv@google.com'],
+ # no body
+ },
+ {
+ 'subject': 'foo',
+ 'identifier': 'foo',
+ # direct recipients lack @google.
+ 'direct_recipients': ['gbiv'],
+ 'text_body': 'hi',
+ },
+ {
+ 'subject': 'foo',
+ 'identifier': 'foo',
+ # non-list recipients
+ 'direct_recipients': 'gbiv@google.com',
+ 'text_body': 'hi',
+ },
+ {
+ 'subject': 'foo',
+ 'identifier': 'foo',
+ # non-list recipients
+ 'well_known_recipients': 'sheriff',
+ 'text_body': 'hi',
+ },
+ ]
+
+ sender = email_sender.EmailSender()
+ for case in test_cases:
+ with self.assertRaises(ValueError):
+ sender.SendX20Email(**case)
+
+ write_file.assert_not_called()
+
+ @mock.patch('cros_utils.email_sender.AtomicallyWriteFile')
+ def test_x20_email_sending_translates_to_reasonable_json(self, write_file):
+ written_obj = None
+
+ @contextlib.contextmanager
+ def actual_write_file(file_path):
+ nonlocal written_obj
+
+ self.assertTrue(
+ file_path.startswith(email_sender.X20_PATH + '/'), file_path)
+ f = io.StringIO()
+ yield f
+ written_obj = json.loads(f.getvalue())
+
+ write_file.side_effect = actual_write_file
+ email_sender.EmailSender().SendX20Email(
+ subject='hello',
+ identifier='world',
+ well_known_recipients=['sheriff'],
+ direct_recipients=['gbiv@google.com'],
+ text_body='text',
+ html_body='html',
+ )
+
+ self.assertEqual(
+ written_obj, {
+ 'subject': 'hello',
+ 'email_identifier': 'world',
+ 'well_known_recipients': ['sheriff'],
+ 'direct_recipients': ['gbiv@google.com'],
+ 'body': 'text',
+ 'html_body': 'html',
+ })
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/cros_utils/manifest_versions.py b/cros_utils/manifest_versions.py
index 83e8890..115c604 100644
--- a/cros_utils/manifest_versions.py
+++ b/cros_utils/manifest_versions.py
@@ -16,8 +16,8 @@
import tempfile
import time
-import command_executer
-import logger
+from cros_utils import command_executer
+from cros_utils import logger
def IsCrosVersion(version):
diff --git a/cros_utils/misc_test.py b/cros_utils/misc_test.py
index cb7b8c8..21a545e 100755
--- a/cros_utils/misc_test.py
+++ b/cros_utils/misc_test.py
@@ -14,7 +14,7 @@
import unittest
# Local modules
-import misc
+from cros_utils import misc
class UtilsTest(unittest.TestCase):
diff --git a/cros_utils/perf_diff.py b/cros_utils/perf_diff.py
index fed6a81..b8ddb0c 100755
--- a/cros_utils/perf_diff.py
+++ b/cros_utils/perf_diff.py
@@ -18,8 +18,8 @@
import re
import sys
-import misc
-import tabulator
+from cros_utils import misc
+from cros_utils import tabulator
ROWS_TO_SHOW = 'Rows_to_show_in_the_perf_table'
TOTAL_EVENTS = 'Total_events_of_this_profile'
diff --git a/cros_utils/tabulator_test.py b/cros_utils/tabulator_test.py
index 4c0f867..227e2d7 100755
--- a/cros_utils/tabulator_test.py
+++ b/cros_utils/tabulator_test.py
@@ -14,7 +14,7 @@
import unittest
# Local modules
-import tabulator
+from cros_utils import tabulator
class TabulatorTest(unittest.TestCase):
diff --git a/cros_utils/timeline_test.py b/cros_utils/timeline_test.py
index 314cf3c..8a10e54 100755
--- a/cros_utils/timeline_test.py
+++ b/cros_utils/timeline_test.py
@@ -13,7 +13,7 @@
import time
import unittest
-import timeline
+from cros_utils import timeline
class TimeLineTest(unittest.TestCase):
diff --git a/crosperf/benchmark_run.py b/crosperf/benchmark_run.py
index 74d461a..b5912c1 100644
--- a/crosperf/benchmark_run.py
+++ b/crosperf/benchmark_run.py
@@ -178,12 +178,10 @@
return machine
def GetExtraAutotestArgs(self):
- if self.benchmark.perf_args and self.benchmark.suite == 'telemetry':
- self._logger.LogError('Telemetry does not support profiler.')
- self.benchmark.perf_args = ''
-
- if self.benchmark.perf_args and self.benchmark.suite == 'test_that':
- self._logger.LogError('test_that does not support profiler.')
+ if (self.benchmark.perf_args and
+ self.benchmark.suite != 'telemetry_Crosperf'):
+ self._logger.LogError(
+ 'Non-telemetry benchmark does not support profiler.')
self.benchmark.perf_args = ''
if self.benchmark.perf_args:
diff --git a/crosperf/benchmark_run_unittest.py b/crosperf/benchmark_run_unittest.py
index 4cadc35..ab86300 100755
--- a/crosperf/benchmark_run_unittest.py
+++ b/crosperf/benchmark_run_unittest.py
@@ -365,16 +365,12 @@
'--profiler=custom_perf --profiler_args=\'perf_options="record -a -e '
'cycles"\'')
- self.test_benchmark.suite = 'telemetry'
- result = br.GetExtraAutotestArgs()
- self.assertEqual(result, '')
- self.assertEqual(self.err_msg, 'Telemetry does not support profiler.')
-
self.test_benchmark.perf_args = 'record -e cycles'
self.test_benchmark.suite = 'test_that'
result = br.GetExtraAutotestArgs()
self.assertEqual(result, '')
- self.assertEqual(self.err_msg, 'test_that does not support profiler.')
+ self.assertEqual(self.err_msg,
+ 'Non-telemetry benchmark does not support profiler.')
self.test_benchmark.perf_args = 'junk args'
self.test_benchmark.suite = 'telemetry_Crosperf'
diff --git a/crosperf/experiment_factory.py b/crosperf/experiment_factory.py
index 7d1689c..4527db5 100644
--- a/crosperf/experiment_factory.py
+++ b/crosperf/experiment_factory.py
@@ -296,7 +296,7 @@
iterations,
rm_chroot_tmp,
perf_args,
- '',
+ 'crosperf_Wrapper', # Use client wrapper in Autotest
show_all_results,
retries,
run_local=False,
diff --git a/crosperf/mock_instance.py b/crosperf/mock_instance.py
index 4271d8f..842d634 100644
--- a/crosperf/mock_instance.py
+++ b/crosperf/mock_instance.py
@@ -46,10 +46,10 @@
chrome_src=None)
benchmark1 = Benchmark('benchmark1', 'autotest_name_1', 'autotest_args', 2, '',
- perf_args, '', '')
+ perf_args, 'telemetry_Crosperf', '')
benchmark2 = Benchmark('benchmark2', 'autotest_name_2', 'autotest_args', 2, '',
- perf_args, '', '')
+ perf_args, 'telemetry_Crosperf', '')
keyval = {}
keyval[0] = {
diff --git a/crosperf/settings_factory.py b/crosperf/settings_factory.py
index 1993a6c..20ab2ad 100644
--- a/crosperf/settings_factory.py
+++ b/crosperf/settings_factory.py
@@ -40,7 +40,9 @@
'times to get a stable result.'))
self.AddField(
TextField(
- 'suite', default='', description='The type of the benchmark.'))
+ 'suite',
+ default='test_that',
+ description='The type of the benchmark.'))
self.AddField(
IntegerField(
'retries',
diff --git a/crosperf/settings_factory_unittest.py b/crosperf/settings_factory_unittest.py
index 12a87ae..a633903 100755
--- a/crosperf/settings_factory_unittest.py
+++ b/crosperf/settings_factory_unittest.py
@@ -24,7 +24,7 @@
self.assertEqual(res.GetField('test_name'), '')
self.assertEqual(res.GetField('test_args'), '')
self.assertEqual(res.GetField('iterations'), 0)
- self.assertEqual(res.GetField('suite'), '')
+ self.assertEqual(res.GetField('suite'), 'test_that')
class LabelSettingsTest(unittest.TestCase):
diff --git a/crosperf/suite_runner.py b/crosperf/suite_runner.py
index 71ca7e7..79ace20 100644
--- a/crosperf/suite_runner.py
+++ b/crosperf/suite_runner.py
@@ -15,7 +15,6 @@
import time
from cros_utils import command_executer
-from cros_utils.device_setup_utils import DutWrapper
TEST_THAT_PATH = '/usr/bin/test_that'
# TODO: Need to check whether Skylab is installed and set up correctly.
@@ -75,34 +74,12 @@
def Run(self, cros_machine, label, benchmark, test_args, profiler_args):
machine_name = cros_machine.name
- if benchmark.suite != 'telemetry_Crosperf':
- # Initialize command executer on DUT for test_that runs.
- run_on_dut = DutWrapper(
- label.chromeos_root,
- machine_name,
- logger=self.logger,
- log_level=self.log_level,
- ce=self._ce,
- dut_config=self.dut_config)
for i in range(0, benchmark.retries + 1):
- # TODO: For telemetry_Crosperf run, device setup has been moved into
- # server test script; for client runs, need to figure out wrapper to do
- # it before running, now it is still setup here.
- if benchmark.suite != 'telemetry_Crosperf':
- wait_time = run_on_dut.SetupDevice()
- # This is for accumulating wait time for test_that runs only,
- # for telemetry_Cropserf runs, please refer to result_cache.
- cros_machine.AddCooldownWaitTime(wait_time)
-
if label.skylab:
ret_tup = self.Skylab_Run(label, benchmark, test_args, profiler_args)
- elif benchmark.suite == 'telemetry_Crosperf':
- ret_tup = self.Telemetry_Crosperf_Run(machine_name, label, benchmark,
- test_args, profiler_args)
else:
ret_tup = self.Test_That_Run(machine_name, label, benchmark, test_args,
profiler_args)
-
if ret_tup[0] != 0:
self.logger.LogOutput('benchmark %s failed. Retries left: %s' %
(benchmark.name, benchmark.retries - i))
@@ -116,43 +93,94 @@
break
return ret_tup
+ def RemoveTelemetryTempFile(self, machine, chromeos_root):
+ filename = 'telemetry@%s' % machine
+ fullname = os.path.join(chromeos_root, 'chroot', 'tmp', filename)
+ if os.path.exists(fullname):
+ os.remove(fullname)
+
+ def GenTestArgs(self, benchmark, test_args, profiler_args):
+ args_list = []
+
+ if benchmark.suite != 'telemetry_Crosperf' and profiler_args:
+ self.logger.LogFatal('Tests other than telemetry_Crosperf do not '
+ 'support profiler.')
+
+ if test_args:
+ # Strip double quotes off args (so we can wrap them in single
+ # quotes, to pass through to Telemetry).
+ if test_args[0] == '"' and test_args[-1] == '"':
+ test_args = test_args[1:-1]
+ args_list.append("test_args='%s'" % test_args)
+
+ args_list.append(GetDutConfigArgs(self.dut_config))
+
+ if not (benchmark.suite == 'telemetry_Crosperf' or
+ benchmark.suite == 'crosperf_Wrapper'):
+ self.logger.LogWarning('Please make sure the server test has stage for '
+ 'device setup.\n')
+ else:
+ args_list.append('test=%s' % benchmark.test_name)
+ if benchmark.suite == 'telemetry_Crosperf':
+ args_list.append('run_local=%s' % benchmark.run_local)
+ args_list.append(GetProfilerArgs(profiler_args))
+
+ return args_list
+
def Test_That_Run(self, machine, label, benchmark, test_args, profiler_args):
"""Run the test_that test.."""
- options = []
- if label.board:
- options.append('--board=%s' % label.board)
- if test_args:
- options.append(test_args)
- if profiler_args:
- self.logger.LogFatal('test_that does not support profiler.')
+
+ # Remove existing test_that results
command = 'rm -rf /usr/local/autotest/results/*'
self._ce.CrosRunCommand(
command, machine=machine, chromeos_root=label.chromeos_root)
- autotest_dir = AUTOTEST_DIR
- if label.autotest_path != '':
- autotest_dir = label.autotest_path
+ if benchmark.suite == 'telemetry_Crosperf':
+ if not os.path.isdir(label.chrome_src):
+ self.logger.LogFatal('Cannot find chrome src dir to '
+ 'run telemetry: %s' % label.chrome_src)
+ # Check for and remove temporary file that may have been left by
+ # previous telemetry runs (and which might prevent this run from
+ # working).
+ self.RemoveTelemetryTempFile(machine, label.chromeos_root)
- autotest_dir_arg = '--autotest_dir %s' % autotest_dir
- # For non-telemetry tests, specify an autotest directory only if the
- # specified directory is different from default (crosbug.com/679001).
- if autotest_dir == AUTOTEST_DIR:
- autotest_dir_arg = ''
+ # --autotest_dir specifies which autotest directory to use.
+ autotest_dir_arg = '--autotest_dir=%s' % (
+ label.autotest_path if label.autotest_path else AUTOTEST_DIR)
- command = (('%s %s --fast '
- '%s %s %s') % (TEST_THAT_PATH, autotest_dir_arg,
- ' '.join(options), machine, benchmark.test_name))
+ # --fast avoids unnecessary copies of syslogs.
+ fast_arg = '--fast'
+ board_arg = '--board=%s' % label.board
+
+ args_list = self.GenTestArgs(benchmark, test_args, profiler_args)
+ args_arg = '--args=%s' % pipes.quote(' '.join(args_list))
+
+ command = ' '.join([
+ TEST_THAT_PATH, autotest_dir_arg, fast_arg, board_arg, args_arg,
+ machine, benchmark.suite if
+ (benchmark.suite == 'telemetry_Crosperf' or
+ benchmark.suite == 'crosperf_Wrapper') else benchmark.test_name
+ ])
+
+ # Use --no-ns-pid so that cros_sdk does not create a different
+ # process namespace and we can kill process created easily by their
+ # process group.
+ chrome_root_options = ('--no-ns-pid '
+ '--chrome_root={} --chrome_root_mount={} '
+ 'FEATURES="-usersandbox" '
+ 'CHROME_ROOT={}'.format(label.chrome_src,
+ CHROME_MOUNT_DIR,
+ CHROME_MOUNT_DIR))
+
if self.log_level != 'verbose':
self.logger.LogOutput('Running test.')
self.logger.LogOutput('CMD: %s' % command)
- # Use --no-ns-pid so that cros_sdk does not create a different
- # process namespace and we can kill process created easily by
- # their process group.
+
return self._ce.ChrootRunCommandWOutput(
label.chromeos_root,
command,
command_terminator=self._ct,
- cros_sdk_options='--no-ns-pid')
+ cros_sdk_options=chrome_root_options)
def DownloadResult(self, label, task_id):
gsutil_cmd = os.path.join(label.chromeos_root, GS_UTIL)
@@ -200,26 +228,9 @@
# TODO: now only put toolchain pool here, user need to be able to specify
# which pool to use. Need to request feature to not use this option at all.
options.append('-pool=toolchain')
- test_args_list = []
- if benchmark.suite == 'telemetry_Crosperf':
- if test_args:
- # Strip double quotes off args (so we can wrap them in single
- # quotes, to pass through to Telemetry).
- if test_args[0] == '"' and test_args[-1] == '"':
- test_args_list.append(test_args[1:-1])
- if profiler_args:
- test_args_list.append(GetProfilerArgs(profiler_args))
- if self.dut_config:
- test_args_list.append(GetDutConfigArgs(self.dut_config))
- test_args_list.append('run_local=%s' % benchmark.run_local)
- test_args_list.append('test=%s' % benchmark.test_name)
- else:
- if profiler_args:
- self.logger.LogFatal('Client tests do not support profiler.')
- if test_args:
- test_args_list.append(test_args)
- if test_args_list:
- options.append('-test-args=%s' % pipes.quote(' '.join(test_args_list)))
+
+ args_list = self.GenTestArgs(benchmark, test_args, profiler_args)
+ options.append('-test-args=%s' % pipes.quote(' '.join(args_list)))
dimensions = []
for dut in label.remote:
@@ -227,7 +238,9 @@
command = (('%s create-test %s %s %s') % \
(SKYLAB_PATH, ' '.join(dimensions), ' '.join(options),
- 'telemetry_Crosperf' if benchmark.suite == 'telemetry_Crosperf'
+ benchmark.suite if
+ (benchmark.suite == 'telemetry_Crosperf' or
+ benchmark.suite == 'crosperf_Wrapper')
else benchmark.test_name))
if self.log_level != 'verbose':
@@ -279,67 +292,6 @@
return (ret_tup[0], result_dir, ret_tup[2])
return ret_tup
- def RemoveTelemetryTempFile(self, machine, chromeos_root):
- filename = 'telemetry@%s' % machine
- fullname = os.path.join(chromeos_root, 'chroot', 'tmp', filename)
- if os.path.exists(fullname):
- os.remove(fullname)
-
- def Telemetry_Crosperf_Run(self, machine, label, benchmark, test_args,
- profiler_args):
- if not os.path.isdir(label.chrome_src):
- self.logger.LogFatal('Cannot find chrome src dir to'
- ' run telemetry: %s' % label.chrome_src)
-
- # Check for and remove temporary file that may have been left by
- # previous telemetry runs (and which might prevent this run from
- # working).
- self.RemoveTelemetryTempFile(machine, label.chromeos_root)
-
- # For telemetry runs, we can use the autotest copy from the source
- # location. No need to have one under /build/<board>.
- autotest_dir_arg = '--autotest_dir %s' % AUTOTEST_DIR
- if label.autotest_path != '':
- autotest_dir_arg = '--autotest_dir %s' % label.autotest_path
-
- profiler_args = GetProfilerArgs(profiler_args)
- dut_config_args = GetDutConfigArgs(self.dut_config)
- # --fast avoids unnecessary copies of syslogs.
- fast_arg = '--fast'
- args_string = ''
- if test_args:
- # Strip double quotes off args (so we can wrap them in single
- # quotes, to pass through to Telemetry).
- if test_args[0] == '"' and test_args[-1] == '"':
- test_args = test_args[1:-1]
- args_string = "test_args='%s'" % test_args
-
- args = '{} run_local={} test={} {} {}'.format(
- args_string, benchmark.run_local, benchmark.test_name, dut_config_args,
- profiler_args)
-
- cmd = ('{} {} {} --board={} --args={} {} telemetry_Crosperf'.format(
- TEST_THAT_PATH, autotest_dir_arg, fast_arg, label.board,
- pipes.quote(args), machine))
-
- # Use --no-ns-pid so that cros_sdk does not create a different
- # process namespace and we can kill process created easily by their
- # process group.
- chrome_root_options = ('--no-ns-pid '
- '--chrome_root={} --chrome_root_mount={} '
- 'FEATURES="-usersandbox" '
- 'CHROME_ROOT={}'.format(label.chrome_src,
- CHROME_MOUNT_DIR,
- CHROME_MOUNT_DIR))
- if self.log_level != 'verbose':
- self.logger.LogOutput('Running test.')
- self.logger.LogOutput('CMD: %s' % cmd)
- return self._ce.ChrootRunCommandWOutput(
- label.chromeos_root,
- cmd,
- command_terminator=self._ct,
- cros_sdk_options=chrome_root_options)
-
def CommandTerminator(self):
return self._ct
diff --git a/crosperf/suite_runner_unittest.py b/crosperf/suite_runner_unittest.py
index 7f3a7bc..73fcb45 100755
--- a/crosperf/suite_runner_unittest.py
+++ b/crosperf/suite_runner_unittest.py
@@ -10,8 +10,6 @@
from __future__ import print_function
import json
-import os.path
-import time
import unittest
import unittest.mock as mock
@@ -23,7 +21,6 @@
from cros_utils import command_executer
from cros_utils import logger
-from cros_utils.device_setup_utils import DutWrapper
from machine_manager import MockCrosMachine
@@ -49,23 +46,21 @@
'telemetry_Crosperf', # suite
True) # show_all_results
- test_that_bench = Benchmark(
+ crosperf_wrapper_bench = Benchmark(
'b2_test', # name
- 'octane', # test_name
+ 'webgl', # test_name
'', # test_args
3, # iterations
False, # rm_chroot_tmp
- 'record -e cycles') # perf_args
+ '', # perf_args
+ 'crosperf_Wrapper') # suite
def __init__(self, *args, **kwargs):
super(SuiteRunnerTest, self).__init__(*args, **kwargs)
- self.call_test_that_run = False
self.skylab_run_args = []
self.test_that_args = []
- self.telemetry_run_args = []
- self.telemetry_crosperf_args = []
self.call_skylab_run = False
- self.call_telemetry_crosperf_run = False
+ self.call_test_that_run = False
def setUp(self):
self.runner = suite_runner.SuiteRunner(
@@ -79,30 +74,30 @@
res = suite_runner.GetProfilerArgs(input_str)
self.assertEqual(res, output_str)
+ def test_get_dut_config_args(self):
+ dut_config = {'enable_aslr': False, 'top_interval': 1.0}
+ output_str = ('dut_config='
+ "'"
+ '{"enable_aslr": '
+ 'false, "top_interval": 1.0}'
+ "'"
+ '')
+ res = suite_runner.GetDutConfigArgs(dut_config)
+ self.assertEqual(res, output_str)
+
def test_run(self):
def reset():
+ self.test_that_args = []
+ self.skylab_run_args = []
self.call_test_that_run = False
self.call_skylab_run = False
- self.call_telemetry_crosperf_run = False
- self.skylab_run_args = []
- self.test_that_args = []
- self.telemetry_run_args = []
- self.telemetry_crosperf_args = []
def FakeSkylabRun(test_label, benchmark, test_args, profiler_args):
self.skylab_run_args = [test_label, benchmark, test_args, profiler_args]
self.call_skylab_run = True
return 'Ran FakeSkylabRun'
- def FakeTelemetryCrosperfRun(machine, test_label, benchmark, test_args,
- profiler_args):
- self.telemetry_crosperf_args = [
- machine, test_label, benchmark, test_args, profiler_args
- ]
- self.call_telemetry_crosperf_run = True
- return 'Ran FakeTelemetryCrosperfRun'
-
def FakeTestThatRun(machine, test_label, benchmark, test_args,
profiler_args):
self.test_that_args = [
@@ -111,16 +106,8 @@
self.call_test_that_run = True
return 'Ran FakeTestThatRun'
- def FakeRunner(command, ignore_status=False):
- # pylint fix for unused variable.
- del command, ignore_status
- return 0, '', ''
-
self.runner.Skylab_Run = FakeSkylabRun
- self.runner.Telemetry_Crosperf_Run = FakeTelemetryCrosperfRun
self.runner.Test_That_Run = FakeTestThatRun
- DutWrapper.SetupDevice = mock.Mock(return_value=0)
- DutWrapper.RunCommandOnDut = mock.Mock(return_value=FakeRunner)
self.runner.dut_config['enable_aslr'] = False
self.runner.dut_config['cooldown_time'] = 0
@@ -133,36 +120,63 @@
test_args = ''
profiler_args = ''
- reset()
+ # Test skylab run for telemetry_Crosperf and crosperf_Wrapper benchmarks.
self.mock_label.skylab = True
- self.runner.Run(cros_machine, self.mock_label, self.test_that_bench,
+ reset()
+ self.runner.Run(cros_machine, self.mock_label, self.crosperf_wrapper_bench,
test_args, profiler_args)
self.assertTrue(self.call_skylab_run)
self.assertFalse(self.call_test_that_run)
- self.assertFalse(self.call_telemetry_crosperf_run)
self.assertEqual(self.skylab_run_args,
- [self.mock_label, self.test_that_bench, '', ''])
- DutWrapper.SetupDevice.assert_called_once()
- self.mock_label.skylab = False
-
- reset()
- self.runner.Run(cros_machine, self.mock_label, self.test_that_bench,
- test_args, profiler_args)
- self.assertTrue(self.call_test_that_run)
- self.assertFalse(self.call_telemetry_crosperf_run)
- self.assertEqual(
- self.test_that_args,
- ['fake_machine', self.mock_label, self.test_that_bench, '', ''])
+ [self.mock_label, self.crosperf_wrapper_bench, '', ''])
reset()
self.runner.Run(cros_machine, self.mock_label,
self.telemetry_crosperf_bench, test_args, profiler_args)
+ self.assertTrue(self.call_skylab_run)
self.assertFalse(self.call_test_that_run)
- self.assertTrue(self.call_telemetry_crosperf_run)
- self.assertEqual(self.telemetry_crosperf_args, [
+ self.assertEqual(self.skylab_run_args,
+ [self.mock_label, self.telemetry_crosperf_bench, '', ''])
+
+ # Test test_that run for telemetry_Crosperf and crosperf_Wrapper benchmarks.
+ self.mock_label.skylab = False
+ reset()
+ self.runner.Run(cros_machine, self.mock_label, self.crosperf_wrapper_bench,
+ test_args, profiler_args)
+ self.assertTrue(self.call_test_that_run)
+ self.assertFalse(self.call_skylab_run)
+ self.assertEqual(
+ self.test_that_args,
+ ['fake_machine', self.mock_label, self.crosperf_wrapper_bench, '', ''])
+
+ reset()
+ self.runner.Run(cros_machine, self.mock_label,
+ self.telemetry_crosperf_bench, test_args, profiler_args)
+ self.assertTrue(self.call_test_that_run)
+ self.assertFalse(self.call_skylab_run)
+ self.assertEqual(self.test_that_args, [
'fake_machine', self.mock_label, self.telemetry_crosperf_bench, '', ''
])
+ def test_gen_test_args(self):
+ test_args = '--iterations=2'
+ perf_args = 'record -a -e cycles'
+
+ # Test crosperf_Wrapper benchmarks arg list generation
+ args_list = ["test_args='--iterations=2'", "dut_config='{}'", 'test=webgl']
+ res = self.runner.GenTestArgs(self.crosperf_wrapper_bench, test_args, '')
+ self.assertCountEqual(res, args_list)
+
+ # Test telemetry_Crosperf benchmarks arg list generation
+ args_list = [
+ "test_args='--iterations=2'", "dut_config='{}'", 'test=octane',
+ 'run_local=False'
+ ]
+ args_list.append(suite_runner.GetProfilerArgs(perf_args))
+ res = self.runner.GenTestArgs(self.telemetry_crosperf_bench, test_args,
+ perf_args)
+ self.assertCountEqual(res, args_list)
+
@mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommand')
@mock.patch.object(command_executer.CommandExecuter,
'ChrootRunCommandWOutput')
@@ -176,10 +190,12 @@
self.real_logger.LogMsg = FakeLogMsg
self.runner.logger = self.real_logger
+ # Test crosperf_Wrapper benchmarks cannot take perf_args
raised_exception = False
try:
self.runner.Test_That_Run('lumpy1.cros', self.mock_label,
- self.test_that_bench, '', 'record -a -e cycles')
+ self.crosperf_wrapper_bench, '',
+ 'record -a -e cycles')
except SystemExit:
raised_exception = True
self.assertTrue(raised_exception)
@@ -188,7 +204,8 @@
self.mock_cmd_exec.ChrootRunCommandWOutput = mock_chroot_runcmd
self.mock_cmd_exec.CrosRunCommand = mock_cros_runcmd
res = self.runner.Test_That_Run('lumpy1.cros', self.mock_label,
- self.test_that_bench, '--iterations=2', '')
+ self.crosperf_wrapper_bench,
+ '--iterations=2', '')
self.assertEqual(mock_cros_runcmd.call_count, 1)
self.assertEqual(mock_chroot_runcmd.call_count, 1)
self.assertEqual(res, 0)
@@ -197,48 +214,9 @@
args_list = mock_chroot_runcmd.call_args_list[0][0]
args_dict = mock_chroot_runcmd.call_args_list[0][1]
self.assertEqual(len(args_list), 2)
- self.assertEqual(args_list[0], '/tmp/chromeos')
- self.assertEqual(args_list[1], ('/usr/bin/test_that '
- '--fast --board=lumpy '
- '--iterations=2 lumpy1.cros octane'))
self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term)
self.real_logger.LogMsg = save_log_msg
- @mock.patch.object(os.path, 'isdir')
- @mock.patch.object(command_executer.CommandExecuter,
- 'ChrootRunCommandWOutput')
- def test_telemetry_crosperf_run(self, mock_chroot_runcmd, mock_isdir):
-
- mock_isdir.return_value = True
- mock_chroot_runcmd.return_value = 0
- self.mock_cmd_exec.ChrootRunCommandWOutput = mock_chroot_runcmd
- profiler_args = ("--profiler=custom_perf --profiler_args='perf_options"
- '="record -a -e cycles,instructions"\'')
- self.runner.dut_config['turbostat'] = True
- self.runner.dut_config['top_interval'] = 3
- res = self.runner.Telemetry_Crosperf_Run('lumpy1.cros', self.mock_label,
- self.telemetry_crosperf_bench, '',
- profiler_args)
- self.assertEqual(res, 0)
- self.assertEqual(mock_chroot_runcmd.call_count, 1)
- args_list = mock_chroot_runcmd.call_args_list[0][0]
- args_dict = mock_chroot_runcmd.call_args_list[0][1]
- self.assertEqual(args_list[0], '/tmp/chromeos')
- self.assertEqual(
- args_list[1],
- ('/usr/bin/test_that --autotest_dir '
- '/mnt/host/source/src/third_party/autotest/files --fast '
- "--board=lumpy --args=' run_local=False test=octane "
- 'dut_config=\'"\'"\'{"turbostat": true, "top_interval": 3}\'"\'"\' '
- 'profiler=custom_perf profiler_args=\'"\'"\'record -a -e '
- 'cycles,instructions\'"\'"\'\' lumpy1.cros telemetry_Crosperf'))
- self.assertEqual(args_dict['cros_sdk_options'],
- ('--no-ns-pid --chrome_root= '
- '--chrome_root_mount=/tmp/chrome_root '
- 'FEATURES="-usersandbox" CHROME_ROOT=/tmp/chrome_root'))
- self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term)
- self.assertEqual(len(args_dict), 2)
-
@mock.patch.object(command_executer.CommandExecuter, 'RunCommandWOutput')
@mock.patch.object(json, 'loads')
def test_skylab_run_client(self, mock_json_loads, mock_runcmd):
@@ -265,82 +243,21 @@
self.mock_label.skylab = True
self.runner.DownloadResult = FakeDownloadResult
- res = self.runner.Skylab_Run(self.mock_label, self.test_that_bench, '', '')
+ res = self.runner.Skylab_Run(self.mock_label, self.crosperf_wrapper_bench,
+ '', '')
ret_tup = (0, '\nResults placed in tmp/swarming-12345\n', '')
self.assertEqual(res, ret_tup)
self.assertEqual(mock_runcmd.call_count, 2)
args_list = mock_runcmd.call_args_list[0][0]
args_dict = mock_runcmd.call_args_list[0][1]
- self.assertEqual(args_list[0],
- ('/usr/local/bin/skylab create-test '
- '-dim dut_name:lumpy1 -dim dut_name:lumpy.cros2 '
- '-board=lumpy -image=build '
- '-pool=toolchain octane'))
+ self.assertEqual(len(args_list), 1)
self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term)
args_list = mock_runcmd.call_args_list[1][0]
self.assertEqual(args_list[0], ('skylab wait-task 12345'))
self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term)
- @mock.patch.object(command_executer.CommandExecuter, 'RunCommandWOutput')
- @mock.patch.object(json, 'loads')
- def test_skylab_run_telemetry_Crosperf(self, mock_json_loads, mock_runcmd):
-
- def FakeDownloadResult(l, task_id):
- if l and task_id:
- self.assertEqual(task_id, '12345')
- return None
-
- mock_runcmd.return_value = (
- 0,
- 'Created Swarming task https://swarming/task/b12345',
- '',
- )
- self.mock_cmd_exec.RunCommandWOutput = mock_runcmd
-
- mock_json_loads.return_value = {
- 'child-results': [{
- 'success': True,
- 'task-run-url': 'https://swarming/task?id=12345'
- }]
- }
- self.mock_json.loads = mock_json_loads
-
- self.mock_label.skylab = True
- self.runner.DownloadResult = FakeDownloadResult
- self.runner.Skylab_Run(self.mock_label, self.telemetry_crosperf_bench, '',
- '')
- args_list = mock_runcmd.call_args_list[0][0]
- self.assertEqual(
- args_list[0],
- ('/usr/local/bin/skylab create-test '
- '-dim dut_name:lumpy1 -dim dut_name:lumpy.cros2 '
- '-board=lumpy -image=build '
- "-pool=toolchain -test-args='run_local=False test=octane' "
- 'telemetry_Crosperf'))
-
- @mock.patch.object(time, 'sleep')
- @mock.patch.object(command_executer.CommandExecuter, 'RunCommand')
- def test_download_result(self, mock_runcmd, mock_sleep):
- mock_runcmd.return_value = 0
- mock_sleep.return_value = 0
- self.mock_cmd_exec.RunCommand = mock_runcmd
-
- self.runner.DownloadResult(self.mock_label, '12345')
-
- self.assertEqual(mock_runcmd.call_count, 2)
- cmd = mock_runcmd.call_args_list[0][0][0]
- self.assertEqual(cmd,
- ('/tmp/chromeos/src/chromium/depot_tools/gsutil.py ls '
- 'gs://chromeos-autotest-results/swarming-12345/'
- 'autoserv_test'))
- cmd = mock_runcmd.call_args_list[1][0][0]
- self.assertEqual(cmd,
- ('/tmp/chromeos/src/chromium/depot_tools/gsutil.py -mq '
- 'cp -r gs://chromeos-autotest-results/swarming-12345 '
- '/tmp/chromeos/chroot/tmp'))
-
if __name__ == '__main__':
unittest.main()
diff --git a/debug_info_test/check_cus.py b/debug_info_test/check_cus.py
index f68fe9c..d3cd636 100644
--- a/debug_info_test/check_cus.py
+++ b/debug_info_test/check_cus.py
@@ -1,7 +1,12 @@
+# -*- coding: utf-8 -*-
# Copyright 2018 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+"""check compile units."""
+
+from __future__ import print_function
+
import os
import subprocess
@@ -9,59 +14,62 @@
cu_checks = [check_ngcc.not_by_gcc]
+
def check_compile_unit(dso_path, producer, comp_path):
- """check all compiler flags used to build the compile unit.
+ """check all compiler flags used to build the compile unit.
- Args:
- dso_path: path to the elf/dso
- producer: DW_AT_producer contains the compiler command line.
- comp_path: DW_AT_comp_dir + DW_AT_name
+ Args:
+ dso_path: path to the elf/dso.
+ producer: DW_AT_producer contains the compiler command line.
+ comp_path: DW_AT_comp_dir + DW_AT_name.
- Returns:
- A set of failed tests.
- """
- failed = set()
- for c in cu_checks:
- if not c(dso_path, producer, comp_path):
- failed.add(c.__module__)
+ Returns:
+ A set of failed tests.
+ """
+ failed = set()
+ for c in cu_checks:
+ if not c(dso_path, producer, comp_path):
+ failed.add(c.__module__)
- return failed
+ return failed
+
def check_compile_units(dso_path):
- """check all compile units in the given dso.
+ """check all compile units in the given dso.
- Args:
- dso_path: path to the dso
- Return:
- True if everything looks fine otherwise False.
- """
+ Args:
+ dso_path: path to the dso.
- failed = set()
- producer = ''
- comp_path = ''
+ Returns:
+ True if everything looks fine otherwise False.
+ """
- readelf = subprocess.Popen(['readelf', '--debug-dump=info',
- '--dwarf-depth=1', dso_path],
- stdout=subprocess.PIPE,
- stderr=open(os.devnull, 'w'))
- for l in readelf.stdout:
- if 'DW_TAG_compile_unit' in l:
- if producer:
- failed = failed.union(check_compile_unit(dso_path, producer,
- comp_path))
- producer = ''
- comp_path = ''
- elif 'DW_AT_producer' in l:
- producer = l
- elif 'DW_AT_name' in l:
- comp_path = os.path.join(comp_path, l.split(':')[-1].strip())
- elif 'DW_AT_comp_dir' in l:
- comp_path = os.path.join(l.split(':')[-1].strip(), comp_path)
- if producer:
+ failed = set()
+ producer = ''
+ comp_path = ''
+
+ readelf = subprocess.Popen(
+ ['readelf', '--debug-dump=info', '--dwarf-depth=1', dso_path],
+ stdout=subprocess.PIPE,
+ stderr=open(os.devnull, 'w'),
+ encoding='utf-8')
+ for l in readelf.stdout:
+ if 'DW_TAG_compile_unit' in l:
+ if producer:
failed = failed.union(check_compile_unit(dso_path, producer, comp_path))
+ producer = ''
+ comp_path = ''
+ elif 'DW_AT_producer' in l:
+ producer = l
+ elif 'DW_AT_name' in l:
+ comp_path = os.path.join(comp_path, l.split(':')[-1].strip())
+ elif 'DW_AT_comp_dir' in l:
+ comp_path = os.path.join(l.split(':')[-1].strip(), comp_path)
+ if producer:
+ failed = failed.union(check_compile_unit(dso_path, producer, comp_path))
- if failed:
- print('%s failed check: %s' % (dso_path, ' '.join(failed)))
- return False
+ if failed:
+ print('%s failed check: %s' % (dso_path, ' '.join(failed)))
+ return False
- return True
+ return True
diff --git a/debug_info_test/check_exist.py b/debug_info_test/check_exist.py
index 5e7cce1..dbb8912 100644
--- a/debug_info_test/check_exist.py
+++ b/debug_info_test/check_exist.py
@@ -1,90 +1,102 @@
+# -*- coding: utf-8 -*-
# Copyright 2018 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+"""check whether intended components exists in the given dso."""
+
+from __future__ import print_function
+
import os
import subprocess
from whitelist import is_whitelisted
+
def check_debug_info(dso_path, readelf_content):
- """check whether debug info section exists in the elf file.
+ """Check whether debug info section exists in the elf file.
- Args:
- readelf: debug info dumped by command readelf
+ Args:
+ dso_path: path to the dso.
+ readelf_content: debug info dumped by command readelf.
- Returns:
- True if debug info section exists, otherwise False.
- """
+ Returns:
+ True if debug info section exists, otherwise False.
+ """
- # Return True if it is whitelisted
- if is_whitelisted('exist_debug_info', dso_path):
- return True
+ # Return True if it is whitelisted
+ if is_whitelisted('exist_debug_info', dso_path):
+ return True
- for l in readelf_content:
- if 'debug_info' in l:
- return True
- return False
+ for l in readelf_content:
+ if 'debug_info' in l:
+ return True
+ return False
+
def check_producer(dso_path, readelf_content):
- """check whether DW_AT_producer exists in each compile unit.
+ """Check whether DW_AT_producer exists in each compile unit.
- Args:
- readelf: debug info dumped by command readelf
+ Args:
+ dso_path: path to the dso.
+ readelf_content: debug info dumped by command readelf.
- Returns:
- True if DW_AT_producer exists in each compile unit, otherwise False.
- Notice: If no compile unit in DSO, also return True.
- """
+ Returns:
+ True if DW_AT_producer exists in each compile unit, otherwise False.
+ Notice: If no compile unit in DSO, also return True.
+ """
- # Return True if it is whitelisted
- if is_whitelisted('exist_producer', dso_path):
- return True
+ # Return True if it is whitelisted
+ if is_whitelisted('exist_producer', dso_path):
+ return True
- # Indicate if there is a producer under each cu
- cur_producer = False
+ # Indicate if there is a producer under each cu
+ cur_producer = False
- first_cu = True
- producer_exist = True
+ first_cu = True
+ producer_exist = True
- for l in readelf_content:
- if 'DW_TAG_compile_unit' in l:
- if not first_cu and not cur_producer:
- producer_exist = False
- break
- first_cu = False
- cur_producer = False
- elif 'DW_AT_producer' in l:
- cur_producer = True
-
- # Check whether last producer of compile unit exists in the elf,
- # also return True if no cu in the DSO.
- if not first_cu and not cur_producer:
+ for l in readelf_content:
+ if 'DW_TAG_compile_unit' in l:
+ if not first_cu and not cur_producer:
producer_exist = False
+ break
+ first_cu = False
+ cur_producer = False
+ elif 'DW_AT_producer' in l:
+ cur_producer = True
- return producer_exist
+ # Check whether last producer of compile unit exists in the elf,
+ # also return True if no cu in the DSO.
+ if not first_cu and not cur_producer:
+ producer_exist = False
+
+ return producer_exist
+
def check_exist_all(dso_path):
- """check whether intended components exists in the given dso.
+ """check whether intended components exists in the given dso.
- Args:
- dso_path: path to the dso
- Return:
- True if everything looks fine otherwise False.
- """
+ Args:
+ dso_path: path to the dso.
- readelf = subprocess.Popen(['readelf', '--debug-dump=info',
- '--dwarf-depth=1', dso_path],
- stdout=subprocess.PIPE,
- stderr=open(os.devnull, 'w'))
- readelf_content = list(readelf.stdout)
+ Returns:
+ True if everything looks fine otherwise False.
+ """
- exist_checks = [check_debug_info, check_producer]
+ readelf = subprocess.Popen(
+ ['readelf', '--debug-dump=info', '--dwarf-depth=1', dso_path],
+ stdout=subprocess.PIPE,
+ stderr=open(os.devnull, 'w'),
+ encoding='utf-8')
+ readelf_content = list(readelf.stdout)
- for e in exist_checks:
- if not e(dso_path, readelf_content):
- check_failed = e.__module__ + ': ' + e.__name__
- print('%s failed check: %s' % (dso_path, check_failed))
- return False
+ exist_checks = [check_debug_info, check_producer]
- return True
+ for e in exist_checks:
+ if not e(dso_path, readelf_content):
+ check_failed = e.__module__ + ': ' + e.__name__
+ print('%s failed check: %s' % (dso_path, check_failed))
+ return False
+
+ return True
diff --git a/debug_info_test/check_icf.py b/debug_info_test/check_icf.py
index 4ac67db..a46968e 100644
--- a/debug_info_test/check_icf.py
+++ b/debug_info_test/check_icf.py
@@ -1,47 +1,53 @@
+# -*- coding: utf-8 -*-
# Copyright 2018 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+"""check whether chrome was built with identical code folding."""
+
+from __future__ import print_function
+
import os
import re
import subprocess
+
def check_identical_code_folding(dso_path):
- """check whether chrome was built with identical code folding.
+ """check whether chrome was built with identical code folding.
- Args:
- dso_path: path to the dso
- Return:
- False if the dso is chrome and it was not built with icf,
- True otherwise.
- """
+ Args:
+ dso_path: path to the dso.
- if not dso_path.endswith('/chrome.debug'):
- return True
+ Returns:
+ False if the dso is chrome and it was not built with icf,
+ True otherwise.
+ """
- # Run 'nm' on the chrome binary and read the output.
- nm = subprocess.Popen(['nm', dso_path],
- stdout=subprocess.PIPE,
- stderr=open(os.devnull, 'w'))
- nm_output, _ = nm.communicate()
+ if not dso_path.endswith('/chrome.debug'):
+ return True
- # Search for addresses of text symbols.
- text_addresses = re.findall('^[0-9a-f]+[ ]+[tT] ',
- nm_output,
- re.MULTILINE)
+ # Run 'nm' on the chrome binary and read the output.
+ nm = subprocess.Popen(['nm', dso_path],
+ stdout=subprocess.PIPE,
+ stderr=open(os.devnull, 'w'),
+ encoding='utf-8')
+ nm_output, _ = nm.communicate()
- # Calculate number of text symbols in chrome binary.
- num_text_addresses = len(text_addresses)
+ # Search for addresses of text symbols.
+ text_addresses = re.findall('^[0-9a-f]+[ ]+[tT] ', nm_output, re.MULTILINE)
- # Calculate number of unique text symbols in chrome binary.
- num_unique_text_addresses = len(set(text_addresses))
+ # Calculate number of text symbols in chrome binary.
+ num_text_addresses = len(text_addresses)
- # Check that the number of duplicate symbols is at least 10,000.
- # - https://crbug.com/813272#c18
- if num_text_addresses-num_unique_text_addresses >= 10000:
- return True
+ # Calculate number of unique text symbols in chrome binary.
+ num_unique_text_addresses = len(set(text_addresses))
- print('%s was not built with ICF' % dso_path)
- print(' num_text_addresses = %d' % num_text_addresses)
- print(' num_unique_text_addresses = %d' % num_unique_text_addresses)
- return False
+ # Check that the number of duplicate symbols is at least 10,000.
+ # - https://crbug.com/813272#c18
+ if num_text_addresses - num_unique_text_addresses >= 10000:
+ return True
+
+ print('%s was not built with ICF' % dso_path)
+ print(' num_text_addresses = %d' % num_text_addresses)
+ print(' num_unique_text_addresses = %d' % num_unique_text_addresses)
+ return False
diff --git a/debug_info_test/check_ngcc.py b/debug_info_test/check_ngcc.py
index eecbb85..501bb98 100644
--- a/debug_info_test/check_ngcc.py
+++ b/debug_info_test/check_ngcc.py
@@ -1,26 +1,30 @@
+# -*- coding: utf-8 -*-
# Copyright 2018 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+"""Check whether the compile unit is not built by gcc."""
+
+from __future__ import print_function
+
from whitelist import is_whitelisted
+
def not_by_gcc(dso_path, producer, comp_path):
- """Check whether the compile unit is not built by gcc.
+ """Check whether the compile unit is not built by gcc.
- Args:
- dso_path: path to the elf/dso
- producer: DW_AT_producer contains the compiler command line.
- comp_path: DW_AT_comp_dir + DW_AT_name
+ Args:
+ dso_path: path to the elf/dso.
+ producer: DW_AT_producer contains the compiler command line.
+ comp_path: DW_AT_comp_dir + DW_AT_name.
- Returns:
- False if compiled by gcc otherwise True
- """
- if is_whitelisted('ngcc_comp_path', comp_path):
- return True
-
- if is_whitelisted('ngcc_dso_path', dso_path):
- return True
-
- if 'GNU C' in producer:
- return False
+ Returns:
+ False if compiled by gcc otherwise True.
+ """
+ if is_whitelisted('ngcc_comp_path', comp_path):
return True
+
+ if is_whitelisted('ngcc_dso_path', dso_path):
+ return True
+
+ return 'GNU C' not in producer
diff --git a/debug_info_test/debug_info_test.py b/debug_info_test/debug_info_test.py
index 4839e69..ae7e9f4 100755
--- a/debug_info_test/debug_info_test.py
+++ b/debug_info_test/debug_info_test.py
@@ -1,9 +1,13 @@
-#!/usr/bin/python2
-
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
# Copyright 2018 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+"""Test for debug info."""
+
+from __future__ import print_function
+
import os
import subprocess
import sys
@@ -12,46 +16,52 @@
import check_cus
import check_exist
-elf_checks = [check_exist.check_exist_all,
- check_cus.check_compile_units,
- check_icf.check_identical_code_folding]
+elf_checks = [
+ check_exist.check_exist_all, check_cus.check_compile_units,
+ check_icf.check_identical_code_folding
+]
+
def scanelf(root):
- """find ELFs in root
+ """Find ELFs in root.
- Args:
- root: root dir to start with the search.
- Returns:
- Filenames of ELFs in root.
- """
- p = subprocess.Popen(['scanelf', '-y', '-B', '-F', '%F', '-R', root],
- stdout=subprocess.PIPE)
- return [l.strip() for l in p.stdout]
+ Args:
+ root: root dir to start with the search.
+
+ Returns:
+ Filenames of ELFs in root.
+ """
+ p = subprocess.Popen(['scanelf', '-y', '-B', '-F', '%F', '-R', root],
+ stdout=subprocess.PIPE,
+ encoding='utf-8')
+ return [l.strip() for l in p.stdout]
+
def Main(argv):
- if len(argv) < 2:
- print('usage: %s [file|dir]')
- return 1
+ if len(argv) < 2:
+ print('usage: %s [file|dir]')
+ return 1
- files = []
- cand = argv[1]
- if os.path.isfile(cand):
- files = [cand]
- elif os.path.isdir(cand):
- files = scanelf(cand)
- else:
- print('usage: %s [file|dir]')
- return 1
+ files = []
+ cand = argv[1]
+ if os.path.isfile(cand):
+ files = [cand]
+ elif os.path.isdir(cand):
+ files = scanelf(cand)
+ else:
+ print('usage: %s [file|dir]')
+ return 1
- failed = False
- for f in files:
- for c in elf_checks:
- if not c(f):
- failed = True
+ failed = False
+ for f in files:
+ for c in elf_checks:
+ if not c(f):
+ failed = True
- if failed:
- return 1
- return 0
+ if failed:
+ return 1
+ return 0
+
if __name__ == '__main__':
- sys.exit(Main(sys.argv))
+ sys.exit(Main(sys.argv))
diff --git a/debug_info_test/whitelist.py b/debug_info_test/whitelist.py
index 383fcc3..b53387a 100644
--- a/debug_info_test/whitelist.py
+++ b/debug_info_test/whitelist.py
@@ -1,11 +1,17 @@
+# -*- coding: utf-8 -*-
# Copyright 2018 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+"""Whitelist functions."""
+
+from __future__ import print_function
+
import os
import glob
import re
+
# Matching a string of length m in an NFA of size n is O(mn^2), but the
# performance also depends largely on the implementation. It appears to be fast
# enough according to the tests.
@@ -13,45 +19,50 @@
# The performance bottleneck of this script is readelf. Unless this becomes
# slower than readelf, don't waste time here.
def is_whitelisted(list_name, pattern):
- """chech whether the given pattern is specified in the whitelist.
+ """Check whether the given pattern is specified in the whitelist.
- Args:
- list_name: name of the whitelist
- pattern: the target string
- Returns:
- True if matched otherwise False
- """
- return pattern and whitelists[list_name].match(pattern)
+ Args:
+ list_name: name of the whitelist.
+ pattern: the target string.
+
+ Returns:
+ True if matched otherwise False.
+ """
+ return pattern and whitelists[list_name].match(pattern)
+
def prepare_whitelist(patterns):
- """Join and compile the re patterns.
+ """Join and compile the re patterns.
- Args:
- patterns: regex patterns.
- Return:
- A compiled re object
- """
- return re.compile('|'.join(patterns))
+ Args:
+ patterns: regex patterns.
+
+ Returns:
+ A compiled re object.
+ """
+ return re.compile('|'.join(patterns))
+
def load_whitelists(dirname):
- """Load whitelists under dirname.
+ """Load whitelists under dirname.
- A whitelist ends with .whitelist.
+ A whitelist ends with .whitelist.
- Args:
- dirname: path to the dir.
- Returns:
- A dictionary of 'filename' -> whitelist matcher.
- """
- wlist = {}
- for fn in glob.glob(os.path.join(dirname, '*.whitelist')):
- key = os.path.splitext(os.path.basename(fn))[0]
- with open(fn, 'r') as f:
- patterns = f.read().splitlines()
- patterns = [l for l in patterns if l != '']
- patterns = [l for l in patterns if l[0] != '#']
- wlist[key] = prepare_whitelist(patterns)
- return wlist
+ Args:
+ dirname: path to the dir.
+
+ Returns:
+ A dictionary of 'filename' -> whitelist matcher.
+ """
+ wlist = {}
+ for fn in glob.glob(os.path.join(dirname, '*.whitelist')):
+ key = os.path.splitext(os.path.basename(fn))[0]
+ with open(fn, 'r', encoding='utf-8') as f:
+ patterns = f.read().splitlines()
+ patterns = [l for l in patterns if l != '']
+ patterns = [l for l in patterns if l[0] != '#']
+ wlist[key] = prepare_whitelist(patterns)
+ return wlist
whitelists = load_whitelists(os.path.dirname(__file__))
diff --git a/automation/PRESUBMIT.py b/deprecated/automation/PRESUBMIT.py
similarity index 100%
rename from automation/PRESUBMIT.py
rename to deprecated/automation/PRESUBMIT.py
diff --git a/automation/__init__.py b/deprecated/automation/__init__.py
similarity index 100%
rename from automation/__init__.py
rename to deprecated/automation/__init__.py
diff --git a/automation/all_tests.py b/deprecated/automation/all_tests.py
similarity index 100%
rename from automation/all_tests.py
rename to deprecated/automation/all_tests.py
diff --git a/automation/clients/__init__.py b/deprecated/automation/clients/__init__.py
similarity index 100%
rename from automation/clients/__init__.py
rename to deprecated/automation/clients/__init__.py
diff --git a/automation/clients/android.py b/deprecated/automation/clients/android.py
similarity index 100%
rename from automation/clients/android.py
rename to deprecated/automation/clients/android.py
diff --git a/automation/clients/chromeos.py b/deprecated/automation/clients/chromeos.py
similarity index 100%
rename from automation/clients/chromeos.py
rename to deprecated/automation/clients/chromeos.py
diff --git a/automation/clients/crosstool.py b/deprecated/automation/clients/crosstool.py
similarity index 100%
rename from automation/clients/crosstool.py
rename to deprecated/automation/clients/crosstool.py
diff --git a/automation/clients/dejagnu_compiler.py b/deprecated/automation/clients/dejagnu_compiler.py
similarity index 100%
rename from automation/clients/dejagnu_compiler.py
rename to deprecated/automation/clients/dejagnu_compiler.py
diff --git a/automation/clients/helper/__init__.py b/deprecated/automation/clients/helper/__init__.py
similarity index 100%
rename from automation/clients/helper/__init__.py
rename to deprecated/automation/clients/helper/__init__.py
diff --git a/automation/clients/helper/android.py b/deprecated/automation/clients/helper/android.py
similarity index 100%
rename from automation/clients/helper/android.py
rename to deprecated/automation/clients/helper/android.py
diff --git a/automation/clients/helper/chromeos.py b/deprecated/automation/clients/helper/chromeos.py
similarity index 100%
rename from automation/clients/helper/chromeos.py
rename to deprecated/automation/clients/helper/chromeos.py
diff --git a/automation/clients/helper/crosstool.py b/deprecated/automation/clients/helper/crosstool.py
similarity index 100%
rename from automation/clients/helper/crosstool.py
rename to deprecated/automation/clients/helper/crosstool.py
diff --git a/automation/clients/helper/jobs.py b/deprecated/automation/clients/helper/jobs.py
similarity index 100%
rename from automation/clients/helper/jobs.py
rename to deprecated/automation/clients/helper/jobs.py
diff --git a/automation/clients/helper/perforce.py b/deprecated/automation/clients/helper/perforce.py
similarity index 100%
rename from automation/clients/helper/perforce.py
rename to deprecated/automation/clients/helper/perforce.py
diff --git a/automation/clients/nightly.py b/deprecated/automation/clients/nightly.py
similarity index 100%
rename from automation/clients/nightly.py
rename to deprecated/automation/clients/nightly.py
diff --git a/automation/clients/output_test.py b/deprecated/automation/clients/output_test.py
similarity index 100%
rename from automation/clients/output_test.py
rename to deprecated/automation/clients/output_test.py
diff --git a/automation/clients/pwd_test.py b/deprecated/automation/clients/pwd_test.py
similarity index 100%
rename from automation/clients/pwd_test.py
rename to deprecated/automation/clients/pwd_test.py
diff --git a/automation/clients/report/dejagnu.sh b/deprecated/automation/clients/report/dejagnu.sh
similarity index 100%
rename from automation/clients/report/dejagnu.sh
rename to deprecated/automation/clients/report/dejagnu.sh
diff --git a/automation/clients/report/dejagnu/__init__.py b/deprecated/automation/clients/report/dejagnu/__init__.py
similarity index 100%
rename from automation/clients/report/dejagnu/__init__.py
rename to deprecated/automation/clients/report/dejagnu/__init__.py
diff --git a/automation/clients/report/dejagnu/main.py b/deprecated/automation/clients/report/dejagnu/main.py
similarity index 100%
rename from automation/clients/report/dejagnu/main.py
rename to deprecated/automation/clients/report/dejagnu/main.py
diff --git a/automation/clients/report/dejagnu/manifest.py b/deprecated/automation/clients/report/dejagnu/manifest.py
similarity index 100%
rename from automation/clients/report/dejagnu/manifest.py
rename to deprecated/automation/clients/report/dejagnu/manifest.py
diff --git a/automation/clients/report/dejagnu/report.html b/deprecated/automation/clients/report/dejagnu/report.html
similarity index 100%
rename from automation/clients/report/dejagnu/report.html
rename to deprecated/automation/clients/report/dejagnu/report.html
diff --git a/automation/clients/report/dejagnu/report.py b/deprecated/automation/clients/report/dejagnu/report.py
similarity index 100%
rename from automation/clients/report/dejagnu/report.py
rename to deprecated/automation/clients/report/dejagnu/report.py
diff --git a/automation/clients/report/dejagnu/summary.py b/deprecated/automation/clients/report/dejagnu/summary.py
similarity index 100%
rename from automation/clients/report/dejagnu/summary.py
rename to deprecated/automation/clients/report/dejagnu/summary.py
diff --git a/automation/clients/report/validate_failures.py b/deprecated/automation/clients/report/validate_failures.py
similarity index 100%
rename from automation/clients/report/validate_failures.py
rename to deprecated/automation/clients/report/validate_failures.py
diff --git a/automation/common/__init__.py b/deprecated/automation/common/__init__.py
similarity index 100%
rename from automation/common/__init__.py
rename to deprecated/automation/common/__init__.py
diff --git a/automation/common/command.py b/deprecated/automation/common/command.py
similarity index 100%
rename from automation/common/command.py
rename to deprecated/automation/common/command.py
diff --git a/automation/common/command_executer.py b/deprecated/automation/common/command_executer.py
similarity index 100%
rename from automation/common/command_executer.py
rename to deprecated/automation/common/command_executer.py
diff --git a/automation/common/command_executer_test.py b/deprecated/automation/common/command_executer_test.py
similarity index 100%
rename from automation/common/command_executer_test.py
rename to deprecated/automation/common/command_executer_test.py
diff --git a/automation/common/events.py b/deprecated/automation/common/events.py
similarity index 100%
rename from automation/common/events.py
rename to deprecated/automation/common/events.py
diff --git a/automation/common/job.py b/deprecated/automation/common/job.py
similarity index 100%
rename from automation/common/job.py
rename to deprecated/automation/common/job.py
diff --git a/automation/common/job_group.py b/deprecated/automation/common/job_group.py
similarity index 100%
rename from automation/common/job_group.py
rename to deprecated/automation/common/job_group.py
diff --git a/automation/common/logger.py b/deprecated/automation/common/logger.py
similarity index 100%
rename from automation/common/logger.py
rename to deprecated/automation/common/logger.py
diff --git a/automation/common/machine.py b/deprecated/automation/common/machine.py
similarity index 100%
rename from automation/common/machine.py
rename to deprecated/automation/common/machine.py
diff --git a/automation/common/machine_test.py b/deprecated/automation/common/machine_test.py
similarity index 100%
rename from automation/common/machine_test.py
rename to deprecated/automation/common/machine_test.py
diff --git a/automation/common/state_machine.py b/deprecated/automation/common/state_machine.py
similarity index 100%
rename from automation/common/state_machine.py
rename to deprecated/automation/common/state_machine.py
diff --git a/automation/server/__init__.py b/deprecated/automation/server/__init__.py
similarity index 100%
rename from automation/server/__init__.py
rename to deprecated/automation/server/__init__.py
diff --git a/automation/server/job_executer.py b/deprecated/automation/server/job_executer.py
similarity index 100%
rename from automation/server/job_executer.py
rename to deprecated/automation/server/job_executer.py
diff --git a/automation/server/job_group_manager.py b/deprecated/automation/server/job_group_manager.py
similarity index 100%
rename from automation/server/job_group_manager.py
rename to deprecated/automation/server/job_group_manager.py
diff --git a/automation/server/job_manager.py b/deprecated/automation/server/job_manager.py
similarity index 100%
rename from automation/server/job_manager.py
rename to deprecated/automation/server/job_manager.py
diff --git a/automation/server/machine_manager.py b/deprecated/automation/server/machine_manager.py
similarity index 100%
rename from automation/server/machine_manager.py
rename to deprecated/automation/server/machine_manager.py
diff --git a/automation/server/machine_manager_test.py b/deprecated/automation/server/machine_manager_test.py
similarity index 100%
rename from automation/server/machine_manager_test.py
rename to deprecated/automation/server/machine_manager_test.py
diff --git a/automation/server/monitor/__init__.py b/deprecated/automation/server/monitor/__init__.py
similarity index 100%
rename from automation/server/monitor/__init__.py
rename to deprecated/automation/server/monitor/__init__.py
diff --git a/automation/server/monitor/dashboard.py b/deprecated/automation/server/monitor/dashboard.py
similarity index 100%
rename from automation/server/monitor/dashboard.py
rename to deprecated/automation/server/monitor/dashboard.py
diff --git a/automation/server/monitor/manage.py b/deprecated/automation/server/monitor/manage.py
similarity index 100%
rename from automation/server/monitor/manage.py
rename to deprecated/automation/server/monitor/manage.py
diff --git a/automation/server/monitor/settings.py b/deprecated/automation/server/monitor/settings.py
similarity index 100%
rename from automation/server/monitor/settings.py
rename to deprecated/automation/server/monitor/settings.py
diff --git a/automation/server/monitor/start.sh b/deprecated/automation/server/monitor/start.sh
similarity index 100%
rename from automation/server/monitor/start.sh
rename to deprecated/automation/server/monitor/start.sh
diff --git a/automation/server/monitor/static/style.css b/deprecated/automation/server/monitor/static/style.css
similarity index 100%
rename from automation/server/monitor/static/style.css
rename to deprecated/automation/server/monitor/static/style.css
diff --git a/automation/server/monitor/templates/base.html b/deprecated/automation/server/monitor/templates/base.html
similarity index 100%
rename from automation/server/monitor/templates/base.html
rename to deprecated/automation/server/monitor/templates/base.html
diff --git a/automation/server/monitor/templates/job.html b/deprecated/automation/server/monitor/templates/job.html
similarity index 100%
rename from automation/server/monitor/templates/job.html
rename to deprecated/automation/server/monitor/templates/job.html
diff --git a/automation/server/monitor/templates/job_group.html b/deprecated/automation/server/monitor/templates/job_group.html
similarity index 100%
rename from automation/server/monitor/templates/job_group.html
rename to deprecated/automation/server/monitor/templates/job_group.html
diff --git a/automation/server/monitor/templates/job_group_list.html b/deprecated/automation/server/monitor/templates/job_group_list.html
similarity index 100%
rename from automation/server/monitor/templates/job_group_list.html
rename to deprecated/automation/server/monitor/templates/job_group_list.html
diff --git a/automation/server/monitor/templates/job_log.html b/deprecated/automation/server/monitor/templates/job_log.html
similarity index 100%
rename from automation/server/monitor/templates/job_log.html
rename to deprecated/automation/server/monitor/templates/job_log.html
diff --git a/automation/server/monitor/templates/machine_list.html b/deprecated/automation/server/monitor/templates/machine_list.html
similarity index 100%
rename from automation/server/monitor/templates/machine_list.html
rename to deprecated/automation/server/monitor/templates/machine_list.html
diff --git a/automation/server/monitor/templates/snippet_attribute_table.html b/deprecated/automation/server/monitor/templates/snippet_attribute_table.html
similarity index 100%
rename from automation/server/monitor/templates/snippet_attribute_table.html
rename to deprecated/automation/server/monitor/templates/snippet_attribute_table.html
diff --git a/automation/server/monitor/templates/snippet_code.html b/deprecated/automation/server/monitor/templates/snippet_code.html
similarity index 100%
rename from automation/server/monitor/templates/snippet_code.html
rename to deprecated/automation/server/monitor/templates/snippet_code.html
diff --git a/automation/server/monitor/templates/snippet_links.html b/deprecated/automation/server/monitor/templates/snippet_links.html
similarity index 100%
rename from automation/server/monitor/templates/snippet_links.html
rename to deprecated/automation/server/monitor/templates/snippet_links.html
diff --git a/automation/server/monitor/urls.py b/deprecated/automation/server/monitor/urls.py
similarity index 100%
rename from automation/server/monitor/urls.py
rename to deprecated/automation/server/monitor/urls.py
diff --git a/automation/server/server.py b/deprecated/automation/server/server.py
similarity index 100%
rename from automation/server/server.py
rename to deprecated/automation/server/server.py
diff --git a/automation/server/server_test.py b/deprecated/automation/server/server_test.py
similarity index 100%
rename from automation/server/server_test.py
rename to deprecated/automation/server/server_test.py
diff --git a/automation/server/test_pool.csv b/deprecated/automation/server/test_pool.csv
similarity index 100%
rename from automation/server/test_pool.csv
rename to deprecated/automation/server/test_pool.csv
diff --git a/build-binutils/opts.sh b/deprecated/build-binutils/opts.sh
similarity index 100%
rename from build-binutils/opts.sh
rename to deprecated/build-binutils/opts.sh
diff --git a/build-gcc/opts.sh b/deprecated/build-gcc/opts.sh
similarity index 100%
rename from build-gcc/opts.sh
rename to deprecated/build-gcc/opts.sh
diff --git a/build_chrome_browser.py b/deprecated/build_chrome_browser.py
similarity index 100%
rename from build_chrome_browser.py
rename to deprecated/build_chrome_browser.py
diff --git a/build_tool.py b/deprecated/build_tool.py
similarity index 100%
rename from build_tool.py
rename to deprecated/build_tool.py
diff --git a/crb/autotest_gatherer.py b/deprecated/crb/autotest_gatherer.py
similarity index 100%
rename from crb/autotest_gatherer.py
rename to deprecated/crb/autotest_gatherer.py
diff --git a/crb/autotest_run.py b/deprecated/crb/autotest_run.py
similarity index 100%
rename from crb/autotest_run.py
rename to deprecated/crb/autotest_run.py
diff --git a/crb/crb_driver.py b/deprecated/crb/crb_driver.py
similarity index 100%
rename from crb/crb_driver.py
rename to deprecated/crb/crb_driver.py
diff --git a/crb/machine_manager_singleton.py b/deprecated/crb/machine_manager_singleton.py
similarity index 100%
rename from crb/machine_manager_singleton.py
rename to deprecated/crb/machine_manager_singleton.py
diff --git a/crb/table_formatter.py b/deprecated/crb/table_formatter.py
similarity index 100%
rename from crb/table_formatter.py
rename to deprecated/crb/table_formatter.py
diff --git a/cros_login.py b/deprecated/cros_login.py
similarity index 100%
rename from cros_login.py
rename to deprecated/cros_login.py
diff --git a/cwp/bartlett/app.yaml b/deprecated/cwp/bartlett/app.yaml
similarity index 100%
rename from cwp/bartlett/app.yaml
rename to deprecated/cwp/bartlett/app.yaml
diff --git a/cwp/bartlett/server.py b/deprecated/cwp/bartlett/server.py
similarity index 100%
rename from cwp/bartlett/server.py
rename to deprecated/cwp/bartlett/server.py
diff --git a/cwp/bartlett/static/favicon.ico b/deprecated/cwp/bartlett/static/favicon.ico
similarity index 100%
rename from cwp/bartlett/static/favicon.ico
rename to deprecated/cwp/bartlett/static/favicon.ico
Binary files differ
diff --git a/cwp/bartlett/test/server_tester.py b/deprecated/cwp/bartlett/test/server_tester.py
similarity index 100%
rename from cwp/bartlett/test/server_tester.py
rename to deprecated/cwp/bartlett/test/server_tester.py
diff --git a/cwp/bartlett/update_appengine_server b/deprecated/cwp/bartlett/update_appengine_server
similarity index 100%
rename from cwp/bartlett/update_appengine_server
rename to deprecated/cwp/bartlett/update_appengine_server
diff --git a/cwp/demo_pipeline.sh b/deprecated/cwp/demo_pipeline.sh
similarity index 100%
rename from cwp/demo_pipeline.sh
rename to deprecated/cwp/demo_pipeline.sh
diff --git a/cwp/interpreter/app_engine_pull.py b/deprecated/cwp/interpreter/app_engine_pull.py
similarity index 100%
rename from cwp/interpreter/app_engine_pull.py
rename to deprecated/cwp/interpreter/app_engine_pull.py
diff --git a/cwp/interpreter/symbolizer.py b/deprecated/cwp/interpreter/symbolizer.py
similarity index 100%
rename from cwp/interpreter/symbolizer.py
rename to deprecated/cwp/interpreter/symbolizer.py
diff --git a/cwp/performance/experiment_gen.py b/deprecated/cwp/performance/experiment_gen.py
similarity index 100%
rename from cwp/performance/experiment_gen.py
rename to deprecated/cwp/performance/experiment_gen.py
diff --git a/dejagnu/__init__.py b/deprecated/dejagnu/__init__.py
similarity index 100%
rename from dejagnu/__init__.py
rename to deprecated/dejagnu/__init__.py
diff --git a/dejagnu/boards/chromeos-machine.exp b/deprecated/dejagnu/boards/chromeos-machine.exp
similarity index 100%
rename from dejagnu/boards/chromeos-machine.exp
rename to deprecated/dejagnu/boards/chromeos-machine.exp
diff --git a/dejagnu/boards/gdb.exp.in b/deprecated/dejagnu/boards/gdb.exp.in
similarity index 100%
rename from dejagnu/boards/gdb.exp.in
rename to deprecated/dejagnu/boards/gdb.exp.in
diff --git a/dejagnu/boards/gdbserver.sh.in b/deprecated/dejagnu/boards/gdbserver.sh.in
similarity index 100%
rename from dejagnu/boards/gdbserver.sh.in
rename to deprecated/dejagnu/boards/gdbserver.sh.in
diff --git a/dejagnu/chromeos.exp.in b/deprecated/dejagnu/chromeos.exp.in
similarity index 100%
rename from dejagnu/chromeos.exp.in
rename to deprecated/dejagnu/chromeos.exp.in
diff --git a/dejagnu/gdb_baseline/armv7a-cros-linux-gnueabi b/deprecated/dejagnu/gdb_baseline/armv7a-cros-linux-gnueabi
similarity index 100%
rename from dejagnu/gdb_baseline/armv7a-cros-linux-gnueabi
rename to deprecated/dejagnu/gdb_baseline/armv7a-cros-linux-gnueabi
diff --git a/dejagnu/gdb_baseline/i686-pc-linux-gnu b/deprecated/dejagnu/gdb_baseline/i686-pc-linux-gnu
similarity index 100%
rename from dejagnu/gdb_baseline/i686-pc-linux-gnu
rename to deprecated/dejagnu/gdb_baseline/i686-pc-linux-gnu
diff --git a/dejagnu/gdb_baseline/x86_64-cros-linux-gnu b/deprecated/dejagnu/gdb_baseline/x86_64-cros-linux-gnu
similarity index 100%
rename from dejagnu/gdb_baseline/x86_64-cros-linux-gnu
rename to deprecated/dejagnu/gdb_baseline/x86_64-cros-linux-gnu
diff --git a/dejagnu/gdb_dejagnu.py b/deprecated/dejagnu/gdb_dejagnu.py
similarity index 100%
rename from dejagnu/gdb_dejagnu.py
rename to deprecated/dejagnu/gdb_dejagnu.py
diff --git a/dejagnu/run_dejagnu.py b/deprecated/dejagnu/run_dejagnu.py
similarity index 100%
rename from dejagnu/run_dejagnu.py
rename to deprecated/dejagnu/run_dejagnu.py
diff --git a/dejagnu/site.exp b/deprecated/dejagnu/site.exp
similarity index 100%
rename from dejagnu/site.exp
rename to deprecated/dejagnu/site.exp
diff --git a/fdo_scripts/divide_and_merge_profiles.py b/deprecated/fdo_scripts/divide_and_merge_profiles.py
similarity index 100%
rename from fdo_scripts/divide_and_merge_profiles.py
rename to deprecated/fdo_scripts/divide_and_merge_profiles.py
diff --git a/fdo_scripts/divide_and_merge_profiles_test.py b/deprecated/fdo_scripts/divide_and_merge_profiles_test.py
similarity index 100%
rename from fdo_scripts/divide_and_merge_profiles_test.py
rename to deprecated/fdo_scripts/divide_and_merge_profiles_test.py
diff --git a/fdo_scripts/profile_cycler.py b/deprecated/fdo_scripts/profile_cycler.py
similarity index 100%
rename from fdo_scripts/profile_cycler.py
rename to deprecated/fdo_scripts/profile_cycler.py
diff --git a/fdo_scripts/summarize_hot_blocks.py b/deprecated/fdo_scripts/summarize_hot_blocks.py
similarity index 100%
rename from fdo_scripts/summarize_hot_blocks.py
rename to deprecated/fdo_scripts/summarize_hot_blocks.py
diff --git a/fdo_scripts/vanilla_vs_fdo.py b/deprecated/fdo_scripts/vanilla_vs_fdo.py
similarity index 100%
rename from fdo_scripts/vanilla_vs_fdo.py
rename to deprecated/fdo_scripts/vanilla_vs_fdo.py
diff --git a/generate-waterfall-reports.py b/deprecated/generate-waterfall-reports.py
similarity index 100%
rename from generate-waterfall-reports.py
rename to deprecated/generate-waterfall-reports.py
diff --git a/get_common_image_version.py b/deprecated/get_common_image_version.py
similarity index 100%
rename from get_common_image_version.py
rename to deprecated/get_common_image_version.py
diff --git a/mem_tests/README.md b/deprecated/mem_tests/README.md
similarity index 100%
rename from mem_tests/README.md
rename to deprecated/mem_tests/README.md
diff --git a/mem_tests/clean_data.py b/deprecated/mem_tests/clean_data.py
similarity index 100%
rename from mem_tests/clean_data.py
rename to deprecated/mem_tests/clean_data.py
diff --git a/mem_tests/mem_groups.py b/deprecated/mem_tests/mem_groups.py
similarity index 100%
rename from mem_tests/mem_groups.py
rename to deprecated/mem_tests/mem_groups.py
diff --git a/mem_tests/total_mem_actual.py b/deprecated/mem_tests/total_mem_actual.py
similarity index 100%
rename from mem_tests/total_mem_actual.py
rename to deprecated/mem_tests/total_mem_actual.py
diff --git a/mem_tests/total_mem_sampled.py b/deprecated/mem_tests/total_mem_sampled.py
similarity index 100%
rename from mem_tests/total_mem_sampled.py
rename to deprecated/mem_tests/total_mem_sampled.py
diff --git a/mem_tests/utils.py b/deprecated/mem_tests/utils.py
similarity index 100%
rename from mem_tests/utils.py
rename to deprecated/mem_tests/utils.py
diff --git a/new-generate-waterfall-reports.py b/deprecated/new-generate-waterfall-reports.py
similarity index 100%
rename from new-generate-waterfall-reports.py
rename to deprecated/new-generate-waterfall-reports.py
diff --git a/produce_output.py b/deprecated/produce_output.py
similarity index 100%
rename from produce_output.py
rename to deprecated/produce_output.py
diff --git a/remote_gcc_build.py b/deprecated/remote_gcc_build.py
similarity index 100%
rename from remote_gcc_build.py
rename to deprecated/remote_gcc_build.py
diff --git a/repo_to_repo.py b/deprecated/repo_to_repo.py
similarity index 100%
rename from repo_to_repo.py
rename to deprecated/repo_to_repo.py
diff --git a/test_gcc_dejagnu.py b/deprecated/test_gcc_dejagnu.py
similarity index 100%
rename from test_gcc_dejagnu.py
rename to deprecated/test_gcc_dejagnu.py
diff --git a/test_gdb_dejagnu.py b/deprecated/test_gdb_dejagnu.py
similarity index 100%
rename from test_gdb_dejagnu.py
rename to deprecated/test_gdb_dejagnu.py
diff --git a/test_toolchains.py b/deprecated/test_toolchains.py
similarity index 100%
rename from test_toolchains.py
rename to deprecated/test_toolchains.py
diff --git a/verify_compiler.py b/deprecated/verify_compiler.py
similarity index 100%
rename from verify_compiler.py
rename to deprecated/verify_compiler.py
diff --git a/weekly_report.py b/deprecated/weekly_report.py
similarity index 98%
rename from weekly_report.py
rename to deprecated/weekly_report.py
index 01db867..476b9f0 100755
--- a/weekly_report.py
+++ b/deprecated/weekly_report.py
@@ -1,6 +1,8 @@
#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
#
# Copyright Google Inc. 2014
+
"""Module to generate the 7-day crosperf reports."""
from __future__ import print_function
@@ -205,7 +207,7 @@
'%s.%s' % (timestamp, options.board), 'weekly_tests')
for day in WEEKDAYS:
- startdir = os.path.join(constants.CROSTC_WORKSPACE, day)
+ startdir = os.path.join(constants.CROSTC_WORKSPACE, 'nightly-tests', day)
num_dirs = os.listdir(startdir)
for d in num_dirs:
exp_file = os.path.join(startdir, d, 'toolchain_experiment.txt')
diff --git a/go/chromeos/setup_chromeos_testing.py b/go/chromeos/setup_chromeos_testing.py
index b679ddf..8b53553 100755
--- a/go/chromeos/setup_chromeos_testing.py
+++ b/go/chromeos/setup_chromeos_testing.py
@@ -1,7 +1,9 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
-#
-# Copyright 2018 Google Inc. All Rightes Reserved
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Generate board-specific scripts for Go compiler testing."""
from __future__ import print_function
@@ -73,7 +75,7 @@
def WriteFile(file_content, file_name):
- with open(file_name, 'w') as out_file:
+ with open(file_name, 'w', encoding='utf-8') as out_file:
out_file.write(file_content)
@@ -82,7 +84,7 @@
names = {
'x86_64': x86_board,
'arm64': arm_board,
- 'arm32': ("%s32" % arm_board)
+ 'arm32': ('%s32' % arm_board)
}
toolchain_dir = os.path.join(chromeos_root, 'src', 'third_party',
@@ -93,17 +95,17 @@
toolchain = CROS_TOOLCHAIN_DATA[k]
glibc = GLIBC_DATA[k]
- base_file = os.path.join(toolchain_dir, ("go_%s" % name))
+ base_file = os.path.join(toolchain_dir, ('go_%s' % name))
base_file_content = BASE_TEMPLATE % (name, arch, arch, toolchain, toolchain,
toolchain)
WriteFile(base_file_content, base_file)
- cmd = "chmod 755 %s" % base_file
+ cmd = 'chmod 755 %s' % base_file
ce.RunCommand(cmd)
- exec_file = os.path.join(toolchain_dir, ("go_%s_exec" % name))
+ exec_file = os.path.join(toolchain_dir, ('go_%s_exec' % name))
exec_file_content = EXEC_TEMPLATE % (name, arch, glibc, name)
WriteFile(exec_file_content, exec_file)
- cmd = "chmod 755 %s" % exec_file
+ cmd = 'chmod 755 %s' % exec_file
ce.RunCommand(cmd)
return 0
@@ -111,7 +113,7 @@
def UpdateChrootSshConfig(ce, arm_board, arm_dut, x86_board, x86_dut,
chromeos_root):
- log("Entering UpdateChrootSshConfig")
+ log('Entering UpdateChrootSshConfig')
# Copy testing_rsa to .ssh and set file protections properly.
user = getpass.getuser()
ssh_dir = os.path.join(chromeos_root, 'chroot', 'home', user, '.ssh')
@@ -133,22 +135,22 @@
print('Cannot find %s; you will need to update testing_rsa by hand.' %
src_file)
else:
- log("testing_rsa exists already.")
+ log('testing_rsa exists already.')
# Save ~/.ssh/config file, if not already done.
- config_file = os.path.expanduser("~/.ssh/config")
+ config_file = os.path.expanduser('~/.ssh/config')
saved_config_file = os.path.join(
- os.path.expanduser("~/.ssh"), "config.save.go-scripts")
+ os.path.expanduser('~/.ssh'), 'config.save.go-scripts')
if not os.path.exists(saved_config_file):
- cmd = "cp %s %s" % (config_file, saved_config_file)
+ cmd = 'cp %s %s' % (config_file, saved_config_file)
ret = ce.RunCommand(cmd)
if ret != SUCCESS:
- print("Error making save copy of ~/.ssh/config. Exiting...")
+ print('Error making save copy of ~/.ssh/config. Exiting...')
sys.exit(1)
# Update ~/.ssh/config file
- log("Reading ssh config file")
- with open(config_file, "r") as input_file:
+ log('Reading ssh config file')
+ with open(config_file, 'r') as input_file:
config_lines = input_file.read()
x86_host_config = CONFIG_TEMPLATE % (x86_board, x86_dut)
@@ -158,7 +160,7 @@
config_lines += x86_host_config
config_lines += arm_host_config
- log("Writing ~/.ssh/config")
+ log('Writing ~/.ssh/config')
WriteFile(config_lines, config_file)
return 0
@@ -170,27 +172,27 @@
names = {
'x86_64': x86_board,
'arm64': arm_board,
- 'arm32': ("%s32" % arm_board)
+ 'arm32': ('%s32' % arm_board)
}
toolchain_dir = os.path.join(chromeos_root, 'src', 'third_party',
'toolchain-utils', 'go', 'chromeos')
for k in keys:
name = names[k]
- base_file = os.path.join(toolchain_dir, ("go_%s" % name))
- exec_file = os.path.join(toolchain_dir, ("go_%s_exec" % name))
+ base_file = os.path.join(toolchain_dir, ('go_%s' % name))
+ exec_file = os.path.join(toolchain_dir, ('go_%s_exec' % name))
cmd = ('rm -f %s; rm -f %s' % (base_file, exec_file))
ce.RunCommand(cmd)
# Restore saved config_file
- config_file = os.path.expanduser("~/.ssh/config")
+ config_file = os.path.expanduser('~/.ssh/config')
saved_config_file = os.path.join(
- os.path.expanduser("~/.ssh"), "config.save.go-scripts")
+ os.path.expanduser('~/.ssh'), 'config.save.go-scripts')
if not os.path.exists(saved_config_file):
- print("Could not find file: %s; unable to restore ~/.ssh/config ." %
+ print('Could not find file: %s; unable to restore ~/.ssh/config .' %
saved_config_file)
else:
- cmd = "mv %s %s" % (saved_config_file, config_file)
+ cmd = 'mv %s %s' % (saved_config_file, config_file)
ce.RunCommand(cmd)
return 0
@@ -219,7 +221,7 @@
DEBUG = True
if not os.path.exists(options.chromeos_root):
- print("Invalid ChromeOS Root: %s" % options.chromeos_root)
+ print('Invalid ChromeOS Root: %s' % options.chromeos_root)
ce = command_executer.GetCommandExecuter()
all_good = True
@@ -245,6 +247,6 @@
return 0
-if __name__ == "__main__":
+if __name__ == '__main__':
val = Main(sys.argv)
sys.exit(val)
diff --git a/heatmaps/heat_map.py b/heatmaps/heat_map.py
index 2fd742d..a989ab7 100755
--- a/heatmaps/heat_map.py
+++ b/heatmaps/heat_map.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2015 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -15,7 +15,7 @@
import tempfile
from cros_utils import command_executer
-import heatmap_generator
+from heatmaps import heatmap_generator
def IsARepoRoot(directory):
diff --git a/heatmaps/heat_map_test.py b/heatmaps/heat_map_test.py
index 21f90d4..ad62cd9 100755
--- a/heatmaps/heat_map_test.py
+++ b/heatmaps/heat_map_test.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 The Chromium OS Authors. All rights reserved.
@@ -9,14 +9,15 @@
from __future__ import print_function
-import mock
+import unittest.mock as mock
import unittest
import os
from cros_utils import command_executer
-import heat_map
+from heatmaps import heat_map
+from heatmaps import heatmap_generator
def make_heatmap(chromeos_root='/path/to/fake/chromeos_root/',
@@ -41,7 +42,7 @@
class HeatmapTest(unittest.TestCase):
"""All of our tests for heat_map."""
- #pylint: disable=protected-access
+ # pylint: disable=protected-access
@mock.patch('shutil.copy2')
@mock.patch('tempfile.mkdtemp')
def test_EnsureFileInChrootAlreadyInside(self, mock_mkdtemp, mock_copy):
@@ -81,7 +82,7 @@
self.assertEqual(heatmap.perf_report,
'/fake/chroot/inchroot_path/perf_report.txt')
- @mock.patch('heatmap_generator.HeatmapGenerator')
+ @mock.patch.object(heatmap_generator, 'HeatmapGenerator')
def test_GetHeatMap(self, mock_heatmap_generator):
heatmap = make_heatmap()
heatmap._GetHeatMap(10)
@@ -107,7 +108,7 @@
new=fake_generate_perf_report_exception)
@mock.patch.object(heat_map.HeatMapProducer, '_GetHeatMap')
@mock.patch.object(heat_map.HeatMapProducer, '_RemoveFiles')
- @mock.patch('__builtin__.print')
+ @mock.patch('builtins.print')
def test_Run_with_exception(self, mock_print, mock_remove_files,
mock_get_heatmap, mock_ensure_file_in_chroot):
heatmap = make_heatmap()
diff --git a/heatmaps/heatmap_generator.py b/heatmaps/heatmap_generator.py
index 42fd635..0dd6ad2 100644
--- a/heatmaps/heatmap_generator.py
+++ b/heatmaps/heatmap_generator.py
@@ -13,7 +13,7 @@
the symbol names in hot pages.
"""
-from __future__ import print_function
+from __future__ import division, print_function
import bisect
import collections
@@ -93,10 +93,10 @@
self.max_addr = 1024 * 1024 * 1024
self.ce = command_executer.GetCommandExecuter(log_level=log_level)
self.dir = os.path.dirname(os.path.realpath(__file__))
- with open(perf_report) as f:
+ with open(perf_report, 'r', encoding='utf-8') as f:
self.perf_report_contents = f.readlines()
# Write histogram results to a text file, in order to use gnu plot to draw
- self.hist_temp_output = open('out.txt', 'w')
+ self.hist_temp_output = open('out.txt', 'w', encoding='utf-8')
self.processes = {}
self.deleted_processes = {}
self.count = 0
@@ -277,7 +277,7 @@
if address < self.max_addr:
self.count += 1
line = '%d/%d: %d %d' % (pid[0], pid[1], self.count,
- address / self.page_size * self.page_size)
+ address // self.page_size * self.page_size)
if self.hugepage:
if self.hugepage.start <= address < self.hugepage.end:
line += ' hugepage'
@@ -321,7 +321,7 @@
names = [x for x in os.listdir('.') if 'inst-histo' in x and '.txt' in x]
hist = {}
for n in names:
- with open(n) as f:
+ with open(n, encoding='utf-8') as f:
for l in f.readlines():
num, addr = l.strip().split(' ')
assert int(addr) not in hist
@@ -354,7 +354,7 @@
if 't' not in symbol_type and 'T' not in symbol_type:
# Filter out symbols not in text sections
continue
- if len(self.symbol_addresses) == 0:
+ if not self.symbol_addresses:
# The first symbol in text sections
text_section_start = addr
self.symbol_addresses.append(0)
@@ -374,9 +374,9 @@
def _map_addr_to_symbol(self, addr):
# Find out the symbol name
- assert len(self.symbol_addresses) > 0
+ assert self.symbol_addresses
index = bisect.bisect(self.symbol_addresses, addr)
- assert index > 0 and index <= len(self.symbol_names), \
+ assert 0 < index <= len(self.symbol_names), \
'Failed to find an index (%d) in the list (len=%d)' % (
index, len(self.symbol_names))
return self.symbol_names[index - 1]
@@ -387,7 +387,7 @@
print(
'----------------------------------------------------------', file=fp)
print(
- 'Page Offset: %d MB, Count: %d' % (page_num / 1024 / 1024,
+ 'Page Offset: %d MB, Count: %d' % (page_num // 1024 // 1024,
sample_num),
file=fp)
@@ -400,8 +400,8 @@
if pid is None:
# The sampling is not on Chrome
continue
- if addr / self.page_size != (
- self.processes[pid].start_address + page_num) / self.page_size:
+ if addr // self.page_size != (
+ self.processes[pid].start_address + page_num) // self.page_size:
# Sampling not in the current page
continue
@@ -410,14 +410,14 @@
assert name, 'Failed to find symbol name of addr %x' % addr
symbol_counts[name] += 1
- assert sum(symbol_counts.itervalues()) == sample_num, \
+ assert sum(symbol_counts.values()) == sample_num, \
'Symbol name matching missing for some addresses: %d vs %d' % (
- sum(symbol_counts.itervalues()), sample_num)
+ sum(symbol_counts.values()), sample_num)
# Print out the symbol names sorted by the number of samples in
# the page
for name, count in sorted(
- symbol_counts.iteritems(), key=lambda kv: kv[1], reverse=True):
+ symbol_counts.items(), key=lambda kv: kv[1], reverse=True):
if count == 0:
break
print('> %s : %d' % (name, count), file=fp)
@@ -434,20 +434,19 @@
# Read histogram from histo.txt
hist = self._restore_histogram()
# Sort the pages in histogram
- sorted_hist = sorted(
- hist.iteritems(), key=lambda value: value[1], reverse=True)
+ sorted_hist = sorted(hist.items(), key=lambda value: value[1], reverse=True)
# Generate symbolizations
self._read_symbols_from_binary(binary)
# Write hottest pages
- with open('addr2symbol.txt', 'w') as fp:
+ with open('addr2symbol.txt', 'w', encoding='utf-8') as fp:
if self.hugepage:
# Print hugepage region first
print(
'Hugepage top %d hot pages (%d MB - %d MB):' %
- (top_n, self.hugepage.start / 1024 / 1024,
- self.hugepage.end / 1024 / 1024),
+ (top_n, self.hugepage.start // 1024 // 1024,
+ self.hugepage.end // 1024 // 1024),
file=fp)
pages_to_print = [(k, v)
for k, v in sorted_hist
@@ -464,5 +463,5 @@
self._print_symbols_in_hot_pages(fp, pages_to_print)
else:
# Print top_n hottest pages.
- pages_to_print = [(k, v) for k, v in sorted_hist][:top_n]
+ pages_to_print = sorted_hist[:top_n]
self._print_symbols_in_hot_pages(fp, pages_to_print)
diff --git a/heatmaps/heatmap_generator_test.py b/heatmaps/heatmap_generator_test.py
index 0c0bbfc..5008c65 100755
--- a/heatmaps/heatmap_generator_test.py
+++ b/heatmaps/heatmap_generator_test.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2018 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -8,11 +8,12 @@
from __future__ import division, print_function
-import mock
-import os
+import unittest.mock as mock
import unittest
-import heatmap_generator
+import os
+
+from heatmaps import heatmap_generator
def _write_perf_mmap(pid, tid, addr, size, fp):
diff --git a/llvm_extra/create_ebuild_file.py b/llvm_extra/create_ebuild_file.py
index 459e702..058a270 100755
--- a/llvm_extra/create_ebuild_file.py
+++ b/llvm_extra/create_ebuild_file.py
@@ -1,25 +1,21 @@
-#!/usr/bin/env python2
-
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
# Copyright 2018 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
-from __future__ import print_function
+"""Create llvm ebuild file.
-import os
-import sys
+This script takes an existing host llvm compiler ebuild and
+creates another build that should be installable in a prefixed location.
+The script patches a few lines in the llvm ebuild to make that happen.
-# This script takes an existing host llvm compiler ebuild and
-# creates another build that should be installable in a prefixed location.
-# The script patches a few lines in the llvm ebuild to make that happen.
-#
-# Since the script is based on the current llvm ebuild patterns,
-# it may need to be updated if those patterns change.
-#
-# This script should normally be invoked by the shell script
-# create_llvm_extra.sh .
+Since the script is based on the current llvm ebuild patterns,
+it may need to be updated if those patterns change.
-"""
+This script should normally be invoked by the shell script
+create_llvm_extra.sh .
+
Below is an example of the expected diff of the newly generated ebuild with
some explanation of the diffs.
@@ -40,8 +36,8 @@
# Change USE flags to match llvm ebuild installtion. To see the set of flags
enabled in llvm compiler ebuild, run $ sudo emerge -pv llvm
--IUSE="debug +default-compiler-rt +default-libcxx doc libedit +libffi multitarget
-+IUSE="debug +default-compiler-rt +default-libcxx doc libedit +libffi +multitarget
+-IUSE="debug +default-compiler-rt +default-libcxx doc libedit +libffi"
++IUSE="debug +default-compiler-rt +default-libcxx doc libedit +libffi
ncurses ocaml python llvm-next llvm-tot test xml video_cards_radeon"
COMMON_DEPEND="
@@ -59,8 +55,8 @@
# Allow custom cmake build types (like 'Gentoo')
# Convert use of PN to llvm in epatch commands.
-- epatch "${FILESDIR}"/cmake/${PN}-3.8-allow_custom_cmake_build_types.patch
-+ epatch "${FILESDIR}"/cmake/llvm-3.8-allow_custom_cmake_build_types.patch
+- epatch "${FILESDIR}"/cmake/${PN}-3.8-allow_custom_cmake_build.patch
++ epatch "${FILESDIR}"/cmake/llvm-3.8-allow_custom_cmake_build.patch
# crbug/591436
epatch "${FILESDIR}"/clang-executable-detection.patch
@@ -94,6 +90,12 @@
# some users may find it useful
"""
+from __future__ import print_function
+
+import os
+import sys
+
+
def process_line(line, text):
# Process the line and append to the text we want to generate.
# Check if line has any patterns that we want to handle.
@@ -103,7 +105,7 @@
text.append(line)
elif line.startswith('SLOT='):
# Change SLOT to "${PV%%_p[[:digit:]]*}"
- SLOT_STRING='SLOT="${PV%%_p[[:digit:]]*}"\n'
+ SLOT_STRING = 'SLOT="${PV%%_p[[:digit:]]*}"\n'
text.append(SLOT_STRING)
elif line.startswith('IUSE') and 'multitarget' in line:
# Enable multitarget USE flag.
@@ -137,9 +139,9 @@
def main():
if len(sys.argv) != 3:
- filename = os.path.basename(__file__)
- print ('Usage: ', filename,' <input.ebuild> <output.ebuild>')
- return 1
+ filename = os.path.basename(__file__)
+ print('Usage: ', filename, ' <input.ebuild> <output.ebuild>')
+ return 1
text = []
with open(sys.argv[1], 'r') as infile:
@@ -147,10 +149,10 @@
process_line(line, text)
with open(sys.argv[2], 'w') as outfile:
- outfile.write("".join(text))
+ outfile.write(''.join(text))
return 0
-if __name__== "__main__":
+if __name__ == '__main__':
sys.exit(main())
diff --git a/llvm_tools/custom_script_example.py b/llvm_tools/custom_script_example.py
index 7e107ad..38dff00 100755
--- a/llvm_tools/custom_script_example.py
+++ b/llvm_tools/custom_script_example.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
diff --git a/lock_machine.py b/lock_machine.py
index 244edfb..06c0e80 100755
--- a/lock_machine.py
+++ b/lock_machine.py
@@ -59,9 +59,9 @@
# set it long enough to cover the period to finish nightly rotation tests.
LEASE_MINS = 1440
- SKYLAB_CREDENTIAL = '/usr/local/google/home/mobiletc-prebuild' \
- '/sheriff_utils/skylab_credential' \
- '/chromeos-swarming-credential.json'
+ SKYLAB_CREDENTIAL = ('/usr/local/google/home/mobiletc-prebuild'
+ '/sheriff_utils/credentials/skylab'
+ '/chromeos-swarming-credential.json')
SWARMING = 'chromite/third_party/swarming.client/swarming.py'
SUCCESS = 0
diff --git a/orderfile/post_process_orderfile.py b/orderfile/post_process_orderfile.py
index e24ab1c..3db0b3b 100755
--- a/orderfile/post_process_orderfile.py
+++ b/orderfile/post_process_orderfile.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -79,7 +79,7 @@
options = parser.parse_args(argv)
if not os.path.exists(options.input_file):
- sys.exit('Input orderfile doesn\'t exist.')
+ sys.exit("Input orderfile doesn\'t exist.")
with open(options.input_file) as in_stream, \
open(options.chrome_nm) as chrome_nm_stream, \
diff --git a/orderfile/post_process_orderfile_test.py b/orderfile/post_process_orderfile_test.py
index 2532b8b..a5fb2c7 100755
--- a/orderfile/post_process_orderfile_test.py
+++ b/orderfile/post_process_orderfile_test.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -41,7 +41,7 @@
class Tests(unittest.TestCase):
"""All of our tests for post_process_orderfile."""
- #pylint: disable=protected-access
+ # pylint: disable=protected-access
def test__parse_nm_output(self):
temp_dir = tempfile.mkdtemp()
self.addCleanup(_cleanup, [temp_dir])
diff --git a/pgo_tools/merge_profdata_and_upload.py b/pgo_tools/merge_profdata_and_upload.py
index dddc7f1..ea95289 100755
--- a/pgo_tools/merge_profdata_and_upload.py
+++ b/pgo_tools/merge_profdata_and_upload.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -56,7 +56,8 @@
def _get_gs_latest(remote_lastest):
assert remote_lastest.startswith(_GS_PREFIX)
try:
- return subprocess.check_output(['gsutil', 'cat', remote_lastest])
+ return subprocess.check_output(['gsutil', 'cat', remote_lastest],
+ encoding='utf-8')
except subprocess.CalledProcessError:
raise RuntimeError('Lastest artifacts not found: %s' % remote_lastest)
@@ -142,7 +143,8 @@
print('Uploading tarball to gs.\nCMD: %s\n' % upload_cmd)
# gsutil prints all status to stderr, oddly enough.
- gs_output = subprocess.check_output(upload_cmd, stderr=subprocess.STDOUT)
+ gs_output = subprocess.check_output(
+ upload_cmd, stderr=subprocess.STDOUT, encoding='utf-8')
print(gs_output)
# gsutil exits successfully even if it uploaded nothing. It prints a summary
@@ -227,7 +229,7 @@
for tryjob in args.tryjob:
fetch_and_append_artifacts(tryjob)
- assert heads, 'Didn\'t fetch anything?'
+ assert heads, "Didn't fetch anything?"
def die_with_head_complaint(complaint):
extra = ' (HEADs found: %s)' % sorted(heads)
@@ -239,7 +241,7 @@
die_with_head_complaint(
'%d LLVM HEADs were found, which is more than one. You probably '
'want a consistent set of HEADs for a profile. If you know you '
- 'don\'t, please specify --llvm_hash, and note that *all* profiles '
+ "don't, please specify --llvm_hash, and note that *all* profiles "
'will be merged into this final profile, regardless of their '
'reported HEAD.' % len(heads))
llvm_hash, = heads
@@ -247,7 +249,7 @@
if llvm_hash not in heads:
assert llvm_hash == args.llvm_hash
die_with_head_complaint(
- 'HEAD %s wasn\'t found in any fetched artifacts.' % llvm_hash)
+ "HEAD %s wasn't found in any fetched artifacts." % llvm_hash)
print('Using LLVM hash: %s' % llvm_hash)
diff --git a/remote_kill_test.py b/remote_kill_test.py
deleted file mode 100755
index e0f29d0..0000000
--- a/remote_kill_test.py
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/usr/bin/env python2
-#
-# Copyright 2010 Google Inc. All Rights Reserved.
-"""Script to wrap test_that script.
-
-Run this script and kill it. Then run ps -ef to see if sleep
-is still running,.
-"""
-
-from __future__ import print_function
-
-__author__ = 'asharif@google.com (Ahmad Sharif)'
-
-import argparse
-import os
-import sys
-
-from cros_utils import command_executer
-
-
-def Usage(parser, message):
- print('ERROR: %s' % message)
- parser.print_help()
- sys.exit(0)
-
-
-def Main(argv):
- parser = argparse.ArgumentParser()
- parser.add_argument(
- '-c',
- '--chromeos_root',
- dest='chromeos_root',
- help='ChromeOS root checkout directory')
- parser.add_argument(
- '-r', '--remote', dest='remote', help='Remote chromeos device.')
-
- _ = parser.parse_args(argv)
- ce = command_executer.GetCommandExecuter()
- ce.RunCommand('ls; sleep 10000', machine=os.uname()[1])
- return 0
-
-
-if __name__ == '__main__':
- Main(sys.argv[1:])
diff --git a/remote_test.py b/remote_test.py
index 62598d5..98ff62a 100755
--- a/remote_test.py
+++ b/remote_test.py
@@ -1,6 +1,10 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
#
-# Copyright 2010 Google Inc. All Rights Reserved.
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Script to wrap test_that script.
This script can login to the chromeos machine using the test private key.
diff --git a/run_tests_for.py b/run_tests_for.py
index 6f77b12..19f8172 100755
--- a/run_tests_for.py
+++ b/run_tests_for.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 The Chromium OS Authors. All rights reserved.
@@ -37,6 +37,12 @@
TestSpec = collections.namedtuple('TestSpec', ['directory', 'command'])
+# List of python scripts that are not test with relative path to
+# toolchain-utils.
+non_test_py_files = {
+ 'debug_info_test/debug_info_test.py',
+}
+
def _make_relative_to_toolchain_utils(toolchain_utils, path):
"""Cleans & makes a path relative to toolchain_utils.
@@ -52,13 +58,26 @@
return result
-def _gather_python_tests_in(subdir):
+def _filter_python_tests(test_files, toolchain_utils):
+ """Returns all files that are real python tests."""
+ python_tests = []
+ for test_file in test_files:
+ rel_path = _make_relative_to_toolchain_utils(toolchain_utils, test_file)
+ if rel_path not in non_test_py_files:
+ python_tests.append(_python_test_to_spec(test_file))
+ else:
+ print('## %s ... NON_TEST_PY_FILE' % rel_path)
+ return python_tests
+
+
+def _gather_python_tests_in(rel_subdir, toolchain_utils):
"""Returns all files that appear to be Python tests in a given directory."""
+ subdir = os.path.join(toolchain_utils, rel_subdir)
test_files = (
os.path.join(subdir, file_name)
for file_name in os.listdir(subdir)
if file_name.endswith('_test.py') or file_name.endswith('_unittest.py'))
- return [_python_test_to_spec(test_file) for test_file in test_files]
+ return _filter_python_tests(test_files, toolchain_utils)
def _run_test(test_spec):
@@ -68,7 +87,8 @@
cwd=test_spec.directory,
stdin=open('/dev/null'),
stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT)
+ stderr=subprocess.STDOUT,
+ encoding='utf-8')
stdout, _ = p.communicate()
exit_code = p.wait()
return exit_code, stdout
@@ -84,13 +104,13 @@
if os.access(test_file, os.X_OK):
command = ['./' + file_name]
else:
- # Assume the user wanted py2.
- command = ['python2', file_name]
+ # Assume the user wanted py3.
+ command = ['python3', file_name]
return TestSpec(directory=test_directory, command=command)
-def _autodetect_python_tests_for(test_file):
+def _autodetect_python_tests_for(test_file, toolchain_utils):
"""Given a test file, detect if there may be related tests."""
if not test_file.endswith('.py'):
return []
@@ -102,8 +122,7 @@
base = test_file[:-3]
candidates = (base + x for x in test_suffixes)
test_files = (x for x in candidates if os.path.exists(x))
-
- return [_python_test_to_spec(test_file) for test_file in test_files]
+ return _filter_python_tests(test_files, toolchain_utils)
def _run_test_scripts(all_tests, show_successful_output=False):
@@ -194,7 +213,7 @@
results = []
for d in sorted(gather_test_dirs):
- results += _gather_python_tests_in(os.path.join(toolchain_utils, d))
+ results += _gather_python_tests_in(d, toolchain_utils)
return results
@@ -240,7 +259,7 @@
tests_to_run = _find_forced_subdir_python_tests(modified_files,
toolchain_utils)
for f in modified_files:
- tests_to_run += _autodetect_python_tests_for(f)
+ tests_to_run += _autodetect_python_tests_for(f, toolchain_utils)
tests_to_run += _find_go_tests(modified_files)
# TestSpecs have lists, so we can't use a set. We'd likely want to keep them
diff --git a/setup_chromeos.py b/setup_chromeos.py
index 0b51d83..6320649 100755
--- a/setup_chromeos.py
+++ b/setup_chromeos.py
@@ -1,8 +1,9 @@
-#!/usr/bin/env python2
-#
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
# Copyright 2010 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+
"""Script to checkout the ChromeOS source.
This script sets up the ChromeOS source in the given directory, matching a
@@ -85,7 +86,7 @@
def TimeToCommonVersion(timestamp):
"""Convert timestamp to common image version."""
tdt = datetime.fromtimestamp(float(timestamp))
- with open(COMMON_VERSIONS, 'r') as f:
+ with open(COMMON_VERSIONS, 'r', encoding='utf-8') as f:
common_list = pickle.load(f)
for sv in common_list:
sdt = datetime.strptime(sv['date'], '%Y-%m-%d %H:%M:%S.%f')
@@ -157,7 +158,7 @@
if version not in ('lkgm', 'common'):
parser.print_help()
logger.GetLogger().LogFatal('timestamp option only applies for '
- "versions \"lkgm\" or \"common\"")
+ 'versions "lkgm" or "common"')
if not options.directory:
parser.print_help()
@@ -181,8 +182,8 @@
manifests = manifest_versions.ManifestVersions()
version = manifests.TimeToVersionChromeOS(time.mktime(time.gmtime()))
version, manifest = version.split('.', 1)
- logger.GetLogger().LogOutput('found version %s.%s for latest LKGM' %
- (version, manifest))
+ logger.GetLogger().LogOutput(
+ 'found version %s.%s for latest LKGM' % (version, manifest))
init = ('repo init -u %s -m paladin/buildspecs/%s/%s.xml' %
(versions_repo, version, manifest))
del manifests
@@ -193,17 +194,16 @@
manifests = manifest_versions.ManifestVersions()
version = manifests.TimeToVersion(timestamp)
version, manifest = version.split('.', 1)
- logger.GetLogger().LogOutput(
- 'found version %s.%s for LKGM at timestamp %s' % (version, manifest,
- timestamp))
+ logger.GetLogger().LogOutput('found version %s.%s for LKGM at timestamp %s'
+ % (version, manifest, timestamp))
init = ('repo init -u %s -m paladin/buildspecs/%s/%s.xml' %
(versions_repo, version, manifest))
del manifests
elif version == 'latest_common':
version = TimeToCommonVersion(time.mktime(time.gmtime()))
version, manifest = version.split('.', 1)
- logger.GetLogger().LogOutput('found version %s.%s for latest Common image' %
- (version, manifest))
+ logger.GetLogger().LogOutput(
+ 'found version %s.%s for latest Common image' % (version, manifest))
init = ('repo init -u %s -m buildspecs/%s/%s.xml' % (versions_repo, version,
manifest))
elif version == 'common':
@@ -212,9 +212,9 @@
logger.GetLogger().LogFatal('No timestamp specified for version=lkgm')
version = TimeToCommonVersion(timestamp)
version, manifest = version.split('.', 1)
- logger.GetLogger().LogOutput('found version %s.%s for latest common image '
- 'at timestamp %s' % (version, manifest,
- timestamp))
+ logger.GetLogger().LogOutput(
+ 'found version %s.%s for latest common image '
+ 'at timestamp %s' % (version, manifest, timestamp))
init = ('repo init -u %s -m buildspecs/%s/%s.xml' % (versions_repo, version,
manifest))
else:
diff --git a/tc_enter_chroot.py b/tc_enter_chroot.py
index d919c96..3a7538a 100755
--- a/tc_enter_chroot.py
+++ b/tc_enter_chroot.py
@@ -1,6 +1,9 @@
-#!/usr/bin/env python2
-#
-# Copyright 2010 Google Inc. All Rights Reserved.
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2010 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Script to enter the ChromeOS chroot with mounted sources.
This script enters the chroot with mounted sources.
@@ -191,10 +194,10 @@
'%s/../../../third_party' % os.path.dirname(__file__))
if os.path.isdir(third_party_dir):
- mount_point = MountPoint(third_party_dir,
- ('%s/%s' % (full_mounted_tc_root,
- os.path.basename(third_party_dir))),
- getpass.getuser())
+ mount_point = MountPoint(
+ third_party_dir,
+ ('%s/%s' % (full_mounted_tc_root, os.path.basename(third_party_dir))),
+ getpass.getuser())
mount_points.append(mount_point)
output = options.output
@@ -243,16 +246,15 @@
inner_command = inner_command[3:]
command_file = 'tc_enter_chroot.cmd'
command_file_path = chromeos_root + '/src/scripts/' + command_file
- retv = command_executer.GetCommandExecuter().RunCommand(
- 'sudo rm -f ' + command_file_path)
+ retv = command_executer.GetCommandExecuter().RunCommand('sudo rm -f ' +
+ command_file_path)
if retv != 0:
return retv
- f = open(command_file_path, 'w')
- f.write(inner_command)
- f.close()
+ with open(command_file_path, 'w', encoding='utf-8') as f:
+ f.write(inner_command)
logger.GetLogger().LogCmd(inner_command)
- retv = command_executer.GetCommandExecuter().RunCommand(
- 'chmod +x ' + command_file_path)
+ retv = command_executer.GetCommandExecuter().RunCommand('chmod +x ' +
+ command_file_path)
if retv != 0:
return retv
diff --git a/toolchain_utils_githooks/check-presubmit.py b/toolchain_utils_githooks/check-presubmit.py
index 2fea102..fc6ec9f 100755
--- a/toolchain_utils_githooks/check-presubmit.py
+++ b/toolchain_utils_githooks/check-presubmit.py
@@ -226,7 +226,7 @@
# pylint+golint.
def try_run_cros_lint(cros_binary):
exit_code, output = run_command_unchecked(
- [cros_binary, 'lint', '--'] + files,
+ [cros_binary, 'lint', '--py3', '--'] + files,
toolchain_utils_root,
env=fixed_env)
diff --git a/update_telemetry_defaults.py b/update_telemetry_defaults.py
index 943dc26..c070aeb 100755
--- a/update_telemetry_defaults.py
+++ b/update_telemetry_defaults.py
@@ -1,6 +1,9 @@
-#!/usr/bin/env python2
-#
-# Copyright 2013 Google Inc. All Rights Reserved.
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
"""Script to maintain the Telemetry benchmark default results file.
This script allows the user to see and update the set of default
@@ -36,20 +39,20 @@
def ReadDefaultsFile(self):
if os.path.exists(self._filename):
- with open(self._filename, 'r') as fp:
+ with open(self._filename, 'r', encoding='utf-8') as fp:
self._defaults = json.load(fp)
def WriteDefaultsFile(self):
- with open(self._filename, 'w') as fp:
+ with open(self._filename, 'w', encoding='utf-8') as fp:
json.dump(self._defaults, fp, indent=2)
- def ListCurrentDefaults(self, benchmark=all):
+ def ListCurrentDefaults(self, benchmark='all'):
# Show user current defaults. By default, show all. The user
# can specify the name of a particular benchmark to see only that
# benchmark's default values.
if len(self._defaults) == 0:
print('The benchmark default results are currently empty.')
- if benchmark == all:
+ if benchmark == 'all':
for b in self._defaults.keys():
results = self._defaults[b]
out_str = b + ' : '
@@ -83,8 +86,8 @@
print("Updated results set for '%s': " % benchmark)
print('%s : %s' % (benchmark, repr(self._defaults[benchmark])))
else:
- print("'%s' is not in '%s's default results list." % (result,
- benchmark))
+ print(
+ "'%s' is not in '%s's default results list." % (result, benchmark))
else:
print("Cannot find benchmark named '%s'" % benchmark)
@@ -145,14 +148,14 @@
words = inp.split(' ')
option = words[0]
option = option.lower()
- if option == 'h' or option == 'help':
+ if option in ('h', 'help'):
self.ShowOptions()
- elif option == 'l' or option == 'list':
+ elif option in ('l', 'list'):
if len(words) == 1:
self.ListCurrentDefaults()
else:
self.ListCurrentDefaults(benchmark=words[1])
- elif option == 'a' or option == 'add':
+ elif option in ('a', 'add'):
if len(words) < 3:
self.UsageError(inp)
else:
@@ -160,31 +163,30 @@
resultList = words[2:]
for r in resultList:
self.AddDefault(benchmark, r)
- elif option == 'd' or option == 'delete':
+ elif option in ('d', 'delete'):
if len(words) != 3:
self.UsageError(inp)
else:
benchmark = words[1]
result = words[2]
self.RemoveDefault(benchmark, result)
- elif option == 'r' or option == 'remove':
+ elif option in ('r', 'remove'):
if len(words) != 2:
self.UsageError(inp)
else:
benchmark = words[1]
self.RemoveBenchmark(benchmark)
- elif option == 'm' or option == 'move':
+ elif option in ('m', 'move'):
if len(words) != 3:
self.UsageError(inp)
else:
old_name = words[1]
new_name = words[2]
self.RenameBenchmark(old_name, new_name)
- elif option == 'q' or option == 'quit':
+ elif option in ('q', 'quit'):
self.WriteDefaultsFile()
- return (option == 'q' or option == 'quit' or option == 't' or
- option == 'terminate')
+ return option in ('q', 'quit', 't', 'terminate')
def Main():
