| # -*- coding:utf-8 -*- |
| # Copyright 2016 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| """Functions that implement the actual checks.""" |
| |
| from __future__ import print_function |
| |
| import json |
| import os |
| import platform |
| import re |
| import sys |
| |
| _path = os.path.realpath(__file__ + '/../..') |
| if sys.path[0] != _path: |
| sys.path.insert(0, _path) |
| del _path |
| |
| # pylint: disable=wrong-import-position |
| import rh.git |
| import rh.results |
| from rh.sixish import string_types |
| import rh.utils |
| |
| |
| class Placeholders(object): |
| """Holder class for replacing ${vars} in arg lists. |
| |
| To add a new variable to replace in config files, just add it as a @property |
| to this class using the form. So to add support for BIRD: |
| @property |
| def var_BIRD(self): |
| return <whatever this is> |
| |
| You can return either a string or an iterable (e.g. a list or tuple). |
| """ |
| |
| def __init__(self, diff=()): |
| """Initialize. |
| |
| Args: |
| diff: The list of files that changed. |
| """ |
| self.diff = diff |
| |
| def expand_vars(self, args): |
| """Perform place holder expansion on all of |args|. |
| |
| Args: |
| args: The args to perform expansion on. |
| |
| Returns: |
| The updated |args| list. |
| """ |
| all_vars = set(self.vars()) |
| replacements = dict((var, self.get(var)) for var in all_vars) |
| |
| ret = [] |
| for arg in args: |
| # First scan for exact matches |
| for key, val in replacements.items(): |
| var = '${%s}' % (key,) |
| if arg == var: |
| if isinstance(val, string_types): |
| ret.append(val) |
| else: |
| ret.extend(val) |
| # We break on first hit to avoid double expansion. |
| break |
| else: |
| # If no exact matches, do an inline replacement. |
| def replace(m): |
| val = self.get(m.group(1)) |
| if isinstance(val, string_types): |
| return val |
| return ' '.join(val) |
| ret.append(re.sub(r'\$\{(%s)\}' % ('|'.join(all_vars),), |
| replace, arg)) |
| |
| return ret |
| |
| @classmethod |
| def vars(cls): |
| """Yield all replacement variable names.""" |
| for key in dir(cls): |
| if key.startswith('var_'): |
| yield key[4:] |
| |
| def get(self, var): |
| """Helper function to get the replacement |var| value.""" |
| return getattr(self, 'var_%s' % (var,)) |
| |
| @property |
| def var_PREUPLOAD_COMMIT_MESSAGE(self): |
| """The git commit message.""" |
| return os.environ.get('PREUPLOAD_COMMIT_MESSAGE', '') |
| |
| @property |
| def var_PREUPLOAD_COMMIT(self): |
| """The git commit sha1.""" |
| return os.environ.get('PREUPLOAD_COMMIT', '') |
| |
| @property |
| def var_PREUPLOAD_FILES(self): |
| """List of files modified in this git commit.""" |
| return [x.file for x in self.diff if x.status != 'D'] |
| |
| @property |
| def var_REPO_ROOT(self): |
| """The root of the repo checkout.""" |
| return rh.git.find_repo_root() |
| |
| @property |
| def var_BUILD_OS(self): |
| """The build OS (see _get_build_os_name for details).""" |
| return _get_build_os_name() |
| |
| |
| class HookOptions(object): |
| """Holder class for hook options.""" |
| |
| def __init__(self, name, args, tool_paths): |
| """Initialize. |
| |
| Args: |
| name: The name of the hook. |
| args: The override commandline arguments for the hook. |
| tool_paths: A dictionary with tool names to paths. |
| """ |
| self.name = name |
| self._args = args |
| self._tool_paths = tool_paths |
| |
| @staticmethod |
| def expand_vars(args, diff=()): |
| """Perform place holder expansion on all of |args|.""" |
| replacer = Placeholders(diff=diff) |
| return replacer.expand_vars(args) |
| |
| def args(self, default_args=(), diff=()): |
| """Gets the hook arguments, after performing place holder expansion. |
| |
| Args: |
| default_args: The list to return if |self._args| is empty. |
| diff: The list of files that changed in the current commit. |
| |
| Returns: |
| A list with arguments. |
| """ |
| args = self._args |
| if not args: |
| args = default_args |
| |
| return self.expand_vars(args, diff=diff) |
| |
| def tool_path(self, tool_name): |
| """Gets the path in which the |tool_name| executable can be found. |
| |
| This function performs expansion for some place holders. If the tool |
| does not exist in the overridden |self._tool_paths| dictionary, the tool |
| name will be returned and will be run from the user's $PATH. |
| |
| Args: |
| tool_name: The name of the executable. |
| |
| Returns: |
| The path of the tool with all optional place holders expanded. |
| """ |
| assert tool_name in TOOL_PATHS |
| if tool_name not in self._tool_paths: |
| return TOOL_PATHS[tool_name] |
| |
| tool_path = os.path.normpath(self._tool_paths[tool_name]) |
| return self.expand_vars([tool_path])[0] |
| |
| |
| def _run(cmd, **kwargs): |
| """Helper command for checks that tend to gather output.""" |
| kwargs.setdefault('redirect_stderr', True) |
| kwargs.setdefault('combine_stdout_stderr', True) |
| kwargs.setdefault('capture_output', True) |
| kwargs.setdefault('check', False) |
| return rh.utils.run(cmd, **kwargs) |
| |
| |
| def _match_regex_list(subject, expressions): |
| """Try to match a list of regular expressions to a string. |
| |
| Args: |
| subject: The string to match regexes on. |
| expressions: An iterable of regular expressions to check for matches with. |
| |
| Returns: |
| Whether the passed in subject matches any of the passed in regexes. |
| """ |
| for expr in expressions: |
| if re.search(expr, subject): |
| return True |
| return False |
| |
| |
| def _filter_diff(diff, include_list, exclude_list=()): |
| """Filter out files based on the conditions passed in. |
| |
| Args: |
| diff: list of diff objects to filter. |
| include_list: list of regex that when matched with a file path will cause |
| it to be added to the output list unless the file is also matched with |
| a regex in the exclude_list. |
| exclude_list: list of regex that when matched with a file will prevent it |
| from being added to the output list, even if it is also matched with a |
| regex in the include_list. |
| |
| Returns: |
| A list of filepaths that contain files matched in the include_list and not |
| in the exclude_list. |
| """ |
| filtered = [] |
| for d in diff: |
| if (d.status != 'D' and |
| _match_regex_list(d.file, include_list) and |
| not _match_regex_list(d.file, exclude_list)): |
| # We've got a match! |
| filtered.append(d) |
| return filtered |
| |
| |
| def _get_build_os_name(): |
| """Gets the build OS name. |
| |
| Returns: |
| A string in a format usable to get prebuilt tool paths. |
| """ |
| system = platform.system() |
| if 'Darwin' in system or 'Macintosh' in system: |
| return 'darwin-x86' |
| |
| # TODO: Add more values if needed. |
| return 'linux-x86' |
| |
| |
| def _fixup_func_caller(cmd, **kwargs): |
| """Wraps |cmd| around a callable automated fixup. |
| |
| For hooks that support automatically fixing errors after running (e.g. code |
| formatters), this function provides a way to run |cmd| as the |fixup_func| |
| parameter in HookCommandResult. |
| """ |
| def wrapper(): |
| result = _run(cmd, **kwargs) |
| if result.returncode not in (None, 0): |
| return result.stdout |
| return None |
| return wrapper |
| |
| |
| def _check_cmd(hook_name, project, commit, cmd, fixup_func=None, **kwargs): |
| """Runs |cmd| and returns its result as a HookCommandResult.""" |
| return [rh.results.HookCommandResult(hook_name, project, commit, |
| _run(cmd, **kwargs), |
| fixup_func=fixup_func)] |
| |
| |
| # Where helper programs exist. |
| TOOLS_DIR = os.path.realpath(__file__ + '/../../tools') |
| |
| def get_helper_path(tool): |
| """Return the full path to the helper |tool|.""" |
| return os.path.join(TOOLS_DIR, tool) |
| |
| |
| def check_custom(project, commit, _desc, diff, options=None, **kwargs): |
| """Run a custom hook.""" |
| return _check_cmd(options.name, project, commit, options.args((), diff), |
| **kwargs) |
| |
| |
| def check_bpfmt(project, commit, _desc, diff, options=None): |
| """Checks that Blueprint files are formatted with bpfmt.""" |
| filtered = _filter_diff(diff, [r'\.bp$']) |
| if not filtered: |
| return None |
| |
| bpfmt = options.tool_path('bpfmt') |
| cmd = [bpfmt, '-l'] + options.args((), filtered) |
| ret = [] |
| for d in filtered: |
| data = rh.git.get_file_content(commit, d.file) |
| result = _run(cmd, input=data) |
| if result.stdout: |
| ret.append(rh.results.HookResult( |
| 'bpfmt', project, commit, error=result.stdout, |
| files=(d.file,))) |
| return ret |
| |
| |
| def check_checkpatch(project, commit, _desc, diff, options=None): |
| """Run |diff| through the kernel's checkpatch.pl tool.""" |
| tool = get_helper_path('checkpatch.pl') |
| cmd = ([tool, '-', '--root', project.dir] + |
| options.args(('--ignore=GERRIT_CHANGE_ID',), diff)) |
| return _check_cmd('checkpatch.pl', project, commit, cmd, |
| input=rh.git.get_patch(commit)) |
| |
| |
| def check_clang_format(project, commit, _desc, diff, options=None): |
| """Run git clang-format on the commit.""" |
| tool = get_helper_path('clang-format.py') |
| clang_format = options.tool_path('clang-format') |
| git_clang_format = options.tool_path('git-clang-format') |
| tool_args = (['--clang-format', clang_format, '--git-clang-format', |
| git_clang_format] + |
| options.args(('--style', 'file', '--commit', commit), diff)) |
| cmd = [tool] + tool_args |
| fixup_func = _fixup_func_caller([tool, '--fix'] + tool_args) |
| return _check_cmd('clang-format', project, commit, cmd, |
| fixup_func=fixup_func) |
| |
| |
| def check_google_java_format(project, commit, _desc, _diff, options=None): |
| """Run google-java-format on the commit.""" |
| |
| tool = get_helper_path('google-java-format.py') |
| google_java_format = options.tool_path('google-java-format') |
| google_java_format_diff = options.tool_path('google-java-format-diff') |
| tool_args = ['--google-java-format', google_java_format, |
| '--google-java-format-diff', google_java_format_diff, |
| '--commit', commit] + options.args() |
| cmd = [tool] + tool_args |
| fixup_func = _fixup_func_caller([tool, '--fix'] + tool_args) |
| return _check_cmd('google-java-format', project, commit, cmd, |
| fixup_func=fixup_func) |
| |
| |
| def check_commit_msg_bug_field(project, commit, desc, _diff, options=None): |
| """Check the commit message for a 'Bug:' line.""" |
| field = 'Bug' |
| regex = r'^%s: (None|[0-9]+(, [0-9]+)*)$' % (field,) |
| check_re = re.compile(regex) |
| |
| if options.args(): |
| raise ValueError('commit msg %s check takes no options' % (field,)) |
| |
| found = [] |
| for line in desc.splitlines(): |
| if check_re.match(line): |
| found.append(line) |
| |
| if not found: |
| error = ('Commit message is missing a "%s:" line. It must match the\n' |
| 'following case-sensitive regex:\n\n %s') % (field, regex) |
| else: |
| return None |
| |
| return [rh.results.HookResult('commit msg: "%s:" check' % (field,), |
| project, commit, error=error)] |
| |
| |
| def check_commit_msg_changeid_field(project, commit, desc, _diff, options=None): |
| """Check the commit message for a 'Change-Id:' line.""" |
| field = 'Change-Id' |
| regex = r'^%s: I[a-f0-9]+$' % (field,) |
| check_re = re.compile(regex) |
| |
| if options.args(): |
| raise ValueError('commit msg %s check takes no options' % (field,)) |
| |
| found = [] |
| for line in desc.splitlines(): |
| if check_re.match(line): |
| found.append(line) |
| |
| if not found: |
| error = ('Commit message is missing a "%s:" line. It must match the\n' |
| 'following case-sensitive regex:\n\n %s') % (field, regex) |
| elif len(found) > 1: |
| error = ('Commit message has too many "%s:" lines. There can be only ' |
| 'one.') % (field,) |
| else: |
| return None |
| |
| return [rh.results.HookResult('commit msg: "%s:" check' % (field,), |
| project, commit, error=error)] |
| |
| |
| PREBUILT_APK_MSG = """Commit message is missing required prebuilt APK |
| information. To generate the information, use the aapt tool to dump badging |
| information of the APKs being uploaded, specify where the APK was built, and |
| specify whether the APKs are suitable for release: |
| |
| for apk in $(find . -name '*.apk' | sort); do |
| echo "${apk}" |
| ${AAPT} dump badging "${apk}" | |
| grep -iE "(package: |sdkVersion:|targetSdkVersion:)" | |
| sed -e "s/' /'\\n/g" |
| echo |
| done |
| |
| It must match the following case-sensitive multiline regex searches: |
| |
| %s |
| |
| For more information, see go/platform-prebuilt and go/android-prebuilt. |
| |
| """ |
| |
| |
| def check_commit_msg_prebuilt_apk_fields(project, commit, desc, diff, |
| options=None): |
| """Check that prebuilt APK commits contain the required lines.""" |
| |
| if options.args(): |
| raise ValueError('prebuilt apk check takes no options') |
| |
| filtered = _filter_diff(diff, [r'\.apk$']) |
| if not filtered: |
| return None |
| |
| regexes = [ |
| r'^package: .*$', |
| r'^sdkVersion:.*$', |
| r'^targetSdkVersion:.*$', |
| r'^Built here:.*$', |
| (r'^This build IS( NOT)? suitable for' |
| r'( preview|( preview or)? public) release' |
| r'( but IS NOT suitable for public release)?\.$') |
| ] |
| |
| missing = [] |
| for regex in regexes: |
| if not re.search(regex, desc, re.MULTILINE): |
| missing.append(regex) |
| |
| if missing: |
| error = PREBUILT_APK_MSG % '\n '.join(missing) |
| else: |
| return None |
| |
| return [rh.results.HookResult('commit msg: "prebuilt apk:" check', |
| project, commit, error=error)] |
| |
| |
| TEST_MSG = """Commit message is missing a "Test:" line. It must match the |
| following case-sensitive regex: |
| |
| %s |
| |
| The Test: stanza is free-form and should describe how you tested your change. |
| As a CL author, you'll have a consistent place to describe the testing strategy |
| you use for your work. As a CL reviewer, you'll be reminded to discuss testing |
| as part of your code review, and you'll more easily replicate testing when you |
| patch in CLs locally. |
| |
| Some examples below: |
| |
| Test: make WITH_TIDY=1 mmma art |
| Test: make test-art |
| Test: manual - took a photo |
| Test: refactoring CL. Existing unit tests still pass. |
| |
| Check the git history for more examples. It's a free-form field, so we urge |
| you to develop conventions that make sense for your project. Note that many |
| projects use exact test commands, which are perfectly fine. |
| |
| Adding good automated tests with new code is critical to our goals of keeping |
| the system stable and constantly improving quality. Please use Test: to |
| highlight this area of your development. And reviewers, please insist on |
| high-quality Test: descriptions. |
| """ |
| |
| |
| def check_commit_msg_test_field(project, commit, desc, _diff, options=None): |
| """Check the commit message for a 'Test:' line.""" |
| field = 'Test' |
| regex = r'^%s: .*$' % (field,) |
| check_re = re.compile(regex) |
| |
| if options.args(): |
| raise ValueError('commit msg %s check takes no options' % (field,)) |
| |
| found = [] |
| for line in desc.splitlines(): |
| if check_re.match(line): |
| found.append(line) |
| |
| if not found: |
| error = TEST_MSG % (regex) |
| else: |
| return None |
| |
| return [rh.results.HookResult('commit msg: "%s:" check' % (field,), |
| project, commit, error=error)] |
| |
| |
| def check_cpplint(project, commit, _desc, diff, options=None): |
| """Run cpplint.""" |
| # This list matches what cpplint expects. We could run on more (like .cxx), |
| # but cpplint would just ignore them. |
| filtered = _filter_diff(diff, [r'\.(cc|h|cpp|cu|cuh)$']) |
| if not filtered: |
| return None |
| |
| cpplint = options.tool_path('cpplint') |
| cmd = [cpplint] + options.args(('${PREUPLOAD_FILES}',), filtered) |
| return _check_cmd('cpplint', project, commit, cmd) |
| |
| |
| def check_gofmt(project, commit, _desc, diff, options=None): |
| """Checks that Go files are formatted with gofmt.""" |
| filtered = _filter_diff(diff, [r'\.go$']) |
| if not filtered: |
| return None |
| |
| gofmt = options.tool_path('gofmt') |
| cmd = [gofmt, '-l'] + options.args((), filtered) |
| ret = [] |
| for d in filtered: |
| data = rh.git.get_file_content(commit, d.file) |
| result = _run(cmd, input=data) |
| if result.stdout: |
| ret.append(rh.results.HookResult( |
| 'gofmt', project, commit, error=result.stdout, |
| files=(d.file,))) |
| return ret |
| |
| |
| def check_json(project, commit, _desc, diff, options=None): |
| """Verify json files are valid.""" |
| if options.args(): |
| raise ValueError('json check takes no options') |
| |
| filtered = _filter_diff(diff, [r'\.json$']) |
| if not filtered: |
| return None |
| |
| ret = [] |
| for d in filtered: |
| data = rh.git.get_file_content(commit, d.file) |
| try: |
| json.loads(data) |
| except ValueError as e: |
| ret.append(rh.results.HookResult( |
| 'json', project, commit, error=str(e), |
| files=(d.file,))) |
| return ret |
| |
| |
| def _check_pylint(project, commit, _desc, diff, extra_args=None, options=None): |
| """Run pylint.""" |
| filtered = _filter_diff(diff, [r'\.py$']) |
| if not filtered: |
| return None |
| |
| if extra_args is None: |
| extra_args = [] |
| |
| pylint = options.tool_path('pylint') |
| cmd = [ |
| get_helper_path('pylint.py'), |
| '--executable-path', pylint, |
| ] + extra_args + options.args(('${PREUPLOAD_FILES}',), filtered) |
| return _check_cmd('pylint', project, commit, cmd) |
| |
| |
| def check_pylint2(project, commit, desc, diff, options=None): |
| """Run pylint through Python 2.""" |
| return _check_pylint(project, commit, desc, diff, options=options) |
| |
| |
| def check_pylint3(project, commit, desc, diff, options=None): |
| """Run pylint through Python 3.""" |
| return _check_pylint(project, commit, desc, diff, |
| extra_args=['--executable-path=pylint3'], |
| options=options) |
| |
| |
| def check_xmllint(project, commit, _desc, diff, options=None): |
| """Run xmllint.""" |
| # XXX: Should we drop most of these and probe for <?xml> tags? |
| extensions = frozenset(( |
| 'dbus-xml', # Generated DBUS interface. |
| 'dia', # File format for Dia. |
| 'dtd', # Document Type Definition. |
| 'fml', # Fuzzy markup language. |
| 'form', # Forms created by IntelliJ GUI Designer. |
| 'fxml', # JavaFX user interfaces. |
| 'glade', # Glade user interface design. |
| 'grd', # GRIT translation files. |
| 'iml', # Android build modules? |
| 'kml', # Keyhole Markup Language. |
| 'mxml', # Macromedia user interface markup language. |
| 'nib', # OS X Cocoa Interface Builder. |
| 'plist', # Property list (for OS X). |
| 'pom', # Project Object Model (for Apache Maven). |
| 'rng', # RELAX NG schemas. |
| 'sgml', # Standard Generalized Markup Language. |
| 'svg', # Scalable Vector Graphics. |
| 'uml', # Unified Modeling Language. |
| 'vcproj', # Microsoft Visual Studio project. |
| 'vcxproj', # Microsoft Visual Studio project. |
| 'wxs', # WiX Transform File. |
| 'xhtml', # XML HTML. |
| 'xib', # OS X Cocoa Interface Builder. |
| 'xlb', # Android locale bundle. |
| 'xml', # Extensible Markup Language. |
| 'xsd', # XML Schema Definition. |
| 'xsl', # Extensible Stylesheet Language. |
| )) |
| |
| filtered = _filter_diff(diff, [r'\.(%s)$' % '|'.join(extensions)]) |
| if not filtered: |
| return None |
| |
| # TODO: Figure out how to integrate schema validation. |
| # XXX: Should we use python's XML libs instead? |
| cmd = ['xmllint'] + options.args(('${PREUPLOAD_FILES}',), filtered) |
| |
| return _check_cmd('xmllint', project, commit, cmd) |
| |
| |
| def check_android_test_mapping(project, commit, _desc, diff, options=None): |
| """Verify Android TEST_MAPPING files are valid.""" |
| if options.args(): |
| raise ValueError('Android TEST_MAPPING check takes no options') |
| filtered = _filter_diff(diff, [r'(^|.*/)TEST_MAPPING$']) |
| if not filtered: |
| return None |
| |
| testmapping_format = options.tool_path('android-test-mapping-format') |
| testmapping_args = ['--commit', commit] |
| cmd = [testmapping_format] + options.args( |
| (project.dir, '${PREUPLOAD_FILES}'), filtered) + testmapping_args |
| return _check_cmd('android-test-mapping-format', project, commit, cmd) |
| |
| |
| # Hooks that projects can opt into. |
| # Note: Make sure to keep the top level README.md up to date when adding more! |
| BUILTIN_HOOKS = { |
| 'android_test_mapping_format': check_android_test_mapping, |
| 'bpfmt': check_bpfmt, |
| 'checkpatch': check_checkpatch, |
| 'clang_format': check_clang_format, |
| 'commit_msg_bug_field': check_commit_msg_bug_field, |
| 'commit_msg_changeid_field': check_commit_msg_changeid_field, |
| 'commit_msg_prebuilt_apk_fields': check_commit_msg_prebuilt_apk_fields, |
| 'commit_msg_test_field': check_commit_msg_test_field, |
| 'cpplint': check_cpplint, |
| 'gofmt': check_gofmt, |
| 'google_java_format': check_google_java_format, |
| 'jsonlint': check_json, |
| 'pylint': check_pylint2, |
| 'pylint2': check_pylint2, |
| 'pylint3': check_pylint3, |
| 'xmllint': check_xmllint, |
| } |
| |
| # Additional tools that the hooks can call with their default values. |
| # Note: Make sure to keep the top level README.md up to date when adding more! |
| TOOL_PATHS = { |
| 'android-test-mapping-format': |
| os.path.join(TOOLS_DIR, 'android_test_mapping_format.py'), |
| 'bpfmt': 'bpfmt', |
| 'clang-format': 'clang-format', |
| 'cpplint': os.path.join(TOOLS_DIR, 'cpplint.py'), |
| 'git-clang-format': 'git-clang-format', |
| 'gofmt': 'gofmt', |
| 'google-java-format': 'google-java-format', |
| 'google-java-format-diff': 'google-java-format-diff.py', |
| 'pylint': 'pylint', |
| } |