Skip ab/6749736 in stage.
Merged-In: Ica61bbf83dd6bfa47cd3d1b84986f7cbadde1b7a
Change-Id: I575e0f9bef1f12cd790475e3fdee7e47aba551e5
diff --git a/Android.bp b/Android.bp
index 960d19c..532bb46 100644
--- a/Android.bp
+++ b/Android.bp
@@ -27,7 +27,6 @@
name: "external_updater_notifier",
main: "notifier.py",
srcs: [
- "git_utils.py",
"notifier.py",
],
}
@@ -42,7 +41,9 @@
"git_updater.py",
"git_utils.py",
"github_archive_updater.py",
+ "hashtags.py",
"metadata.proto",
+ "reviewers.py",
"updater_utils.py",
],
libs: [
@@ -54,6 +55,7 @@
},
data: [
"update_package.sh",
+ "regen_bp.sh",
],
version: {
py2: {
@@ -67,13 +69,37 @@
},
}
+python_defaults {
+ name: "external_updater_defaults",
+ libs: [
+ "external_updater_lib",
+ ],
+ version: {
+ py2: {
+ enabled: false,
+ },
+ py3: {
+ enabled: true,
+ },
+ },
+}
+
python_test_host {
name: "external_updater_test",
+ defaults: ["external_updater_defaults"],
main: "external_updater_test.py",
srcs: [
"external_updater_test.py",
],
- libs: [
- "external_updater_lib",
+ test_suites: ["general-tests"],
+}
+
+python_test_host {
+ name: "external_updater_reviewers_test",
+ defaults: ["external_updater_defaults"],
+ main: "external_updater_reviewers_test.py",
+ srcs: [
+ "external_updater_reviewers_test.py",
],
+ test_suites: ["general-tests"],
}
diff --git a/PREUPLOAD.cfg b/PREUPLOAD.cfg
index f52b2bd..9e97695 100644
--- a/PREUPLOAD.cfg
+++ b/PREUPLOAD.cfg
@@ -1,5 +1,2 @@
[Builtin Hooks]
-pylint = true
-
-[Builtin Hooks Options]
-pylint = --executable-path pylint3 ${PREUPLOAD_FILES}
+pylint3 = true
diff --git a/TEST_MAPPING b/TEST_MAPPING
new file mode 100644
index 0000000..00c285b
--- /dev/null
+++ b/TEST_MAPPING
@@ -0,0 +1,9 @@
+{
+ "presubmit": [
+ // "external_updater_test" runs alone but not with atest
+ {
+ "name": "external_updater_reviewers_test",
+ "host": true
+ }
+ ]
+}
diff --git a/archive_utils.py b/archive_utils.py
index 4a10392..b8386aa 100644
--- a/archive_utils.py
+++ b/archive_utils.py
@@ -90,7 +90,7 @@
return func
# crates.io download url does not have file suffix
# e.g., https://crates.io/api/v1/crates/syn/1.0.16/download
- if url.find('/crates.io/api/') > 0:
+ if url.find('/crates.io/api/') > 0 or url.find('/static.crates.io/crates/'):
return untar
return None
diff --git a/base_updater.py b/base_updater.py
index 74b688d..8cf3255 100644
--- a/base_updater.py
+++ b/base_updater.py
@@ -16,6 +16,7 @@
from pathlib import Path
import fileutils
+# pylint: disable=import-error
import metadata_pb2 # type: ignore
@@ -70,3 +71,8 @@
def latest_url(self) -> metadata_pb2.URL:
"""Gets URL for latest version."""
return self._new_url
+
+ def use_current_as_latest(self):
+ """Uses current version/url as the latest to refresh project."""
+ self._new_ver = self._old_ver
+ self._new_url = self._old_url
diff --git a/crates_updater.py b/crates_updater.py
index 77fec67..78fa9b5 100644
--- a/crates_updater.py
+++ b/crates_updater.py
@@ -14,23 +14,39 @@
"""Module to check updates from crates.io."""
import json
+import os
+# pylint: disable=g-importing-member
+from pathlib import Path
import re
import urllib.request
import archive_utils
from base_updater import Updater
+# pylint: disable=import-error
import metadata_pb2 # type: ignore
import updater_utils
-CRATES_IO_URL_PATTERN: str = (r'^https:\/\/crates.io\/crates\/([-\w]+)')
+CRATES_IO_URL_PATTERN: str = (r"^https:\/\/crates.io\/crates\/([-\w]+)")
CRATES_IO_URL_RE: re.Pattern = re.compile(CRATES_IO_URL_PATTERN)
+ALPHA_BETA_PATTERN: str = (r"^.*[0-9]+\.[0-9]+\.[0-9]+-(alpha|beta).*")
+
+ALPHA_BETA_RE: re.Pattern = re.compile(ALPHA_BETA_PATTERN)
+
+VERSION_PATTERN: str = (r"([0-9]+)\.([0-9]+)\.([0-9]+)")
+
+VERSION_MATCHER: re.Pattern = re.compile(VERSION_PATTERN)
+
+DESCRIPTION_PATTERN: str = (r"^description *= *(\".+\")")
+
+DESCRIPTION_MATCHER: re.Pattern = re.compile(DESCRIPTION_PATTERN)
+
class CratesUpdater(Updater):
"""Updater for crates.io packages."""
- dl_path: str
+ download_url: str
package: str
def is_supported_url(self) -> bool:
@@ -42,16 +58,55 @@
self.package = match.group(1)
return True
+ def _get_version_numbers(self, version: str) -> (int, int, int):
+ match = VERSION_MATCHER.match(version)
+ if match is not None:
+ return tuple(int(match.group(i)) for i in range(1, 4))
+ return (0, 0, 0)
+
+ def _is_newer_version(self, prev_version: str, prev_id: int,
+ check_version: str, check_id: int):
+ """Return true if check_version+id is newer than prev_version+id."""
+ return ((self._get_version_numbers(check_version), check_id) >
+ (self._get_version_numbers(prev_version), prev_id))
+
+ def _find_latest_non_test_version(self) -> None:
+ url = "https://crates.io/api/v1/crates/{}/versions".format(self.package)
+ with urllib.request.urlopen(url) as request:
+ data = json.loads(request.read().decode())
+ last_id = 0
+ self._new_ver = ""
+ for v in data["versions"]:
+ version = v["num"]
+ if (not v["yanked"] and not ALPHA_BETA_RE.match(version) and
+ self._is_newer_version(
+ self._new_ver, last_id, version, int(v["id"]))):
+ last_id = int(v["id"])
+ self._new_ver = version
+ self.download_url = "https://crates.io" + v["dl_path"]
+
def check(self) -> None:
"""Checks crates.io and returns whether a new version is available."""
url = "https://crates.io/api/v1/crates/" + self.package
with urllib.request.urlopen(url) as request:
data = json.loads(request.read().decode())
self._new_ver = data["crate"]["max_version"]
- url = url + "/" + self._new_ver
- with urllib.request.urlopen(url) as request:
- data = json.loads(request.read().decode())
- self.dl_path = data["version"]["dl_path"]
+ # Skip d.d.d-{alpha,beta}* versions
+ if ALPHA_BETA_RE.match(self._new_ver):
+ print("Ignore alpha or beta release: {}-{}."
+ .format(self.package, self._new_ver))
+ self._find_latest_non_test_version()
+ else:
+ url = url + "/" + self._new_ver
+ with urllib.request.urlopen(url) as request:
+ data = json.loads(request.read().decode())
+ self.download_url = "https://crates.io" + data["version"]["dl_path"]
+
+ def use_current_as_latest(self):
+ Updater.use_current_as_latest(self)
+ # A shortcut to use the static download path.
+ self.download_url = "https://static.crates.io/crates/{}/{}-{}.crate".format(
+ self.package, self.package, self._new_ver)
def update(self) -> None:
"""Updates the package.
@@ -59,9 +114,52 @@
Has to call check() before this function.
"""
try:
- url = 'https://crates.io' + self.dl_path
- temporary_dir = archive_utils.download_and_extract(url)
+ temporary_dir = archive_utils.download_and_extract(self.download_url)
package_dir = archive_utils.find_archive_root(temporary_dir)
updater_utils.replace_package(package_dir, self._proj_path)
finally:
urllib.request.urlcleanup()
+
+ # pylint: disable=no-self-use
+ def update_metadata(self, metadata: metadata_pb2.MetaData,
+ full_path: Path) -> None:
+ """Updates METADATA content."""
+ # copy only HOMEPAGE url, and then add new ARCHIVE url.
+ new_url_list = []
+ for url in metadata.third_party.url:
+ if url.type == metadata_pb2.URL.HOMEPAGE:
+ new_url_list.append(url)
+ new_url = metadata_pb2.URL()
+ new_url.type = metadata_pb2.URL.ARCHIVE
+ new_url.value = "https://static.crates.io/crates/{}/{}-{}.crate".format(
+ metadata.name, metadata.name, metadata.third_party.version)
+ new_url_list.append(new_url)
+ del metadata.third_party.url[:]
+ metadata.third_party.url.extend(new_url_list)
+ # copy description from Cargo.toml to METADATA
+ cargo_toml = os.path.join(full_path, "Cargo.toml")
+ description = self._get_cargo_description(cargo_toml)
+ if description and description != metadata.description:
+ print("New METADATA description:", description)
+ metadata.description = description
+
+ def _toml2str(self, line: str) -> str:
+ """Convert a quoted toml string to a Python str without quotes."""
+ if line.startswith("\"\"\""):
+ return "" # cannot handle broken multi-line description
+ # TOML string escapes: \b \t \n \f \r \" \\ (no unicode escape)
+ line = line[1:-1].replace("\\\\", "\n").replace("\\b", "")
+ line = line.replace("\\t", " ").replace("\\n", " ").replace("\\f", " ")
+ line = line.replace("\\r", "").replace("\\\"", "\"").replace("\n", "\\")
+ # replace a unicode quotation mark, used in the libloading crate
+ return line.replace("’", "'").strip()
+
+ def _get_cargo_description(self, cargo_toml: str) -> str:
+ """Return the description in Cargo.toml or empty string."""
+ if os.path.isfile(cargo_toml) and os.access(cargo_toml, os.R_OK):
+ with open(cargo_toml, "r") as toml_file:
+ for line in toml_file:
+ match = DESCRIPTION_MATCHER.match(line)
+ if match:
+ return self._toml2str(match.group(1))
+ return ""
diff --git a/external_updater.py b/external_updater.py
index 31cdbbd..006ad34 100644
--- a/external_updater.py
+++ b/external_updater.py
@@ -17,6 +17,7 @@
Example usage:
updater.sh checkall
updater.sh update kotlinc
+updater.sh update --refresh --keep_date rust/crates/libc
"""
import argparse
@@ -35,6 +36,7 @@
from github_archive_updater import GithubArchiveUpdater
import fileutils
import git_utils
+# pylint: disable=import-error
import metadata_pb2 # type: ignore
import updater_utils
@@ -105,11 +107,15 @@
for metadata_url in updated_metadata.third_party.url:
if metadata_url == updater.current_url:
metadata_url.CopyFrom(updater.latest_url)
- fileutils.write_metadata(full_path, updated_metadata)
+ # For Rust crates, replace GIT url with ARCHIVE url
+ if isinstance(updater, CratesUpdater):
+ updater.update_metadata(updated_metadata, full_path)
+ fileutils.write_metadata(full_path, updated_metadata, args.keep_date)
git_utils.add_file(full_path, 'METADATA')
if args.branch_and_commit:
- msg = 'Upgrade {} to {}\n'.format(args.path, updater.latest_version)
+ msg = 'Upgrade {} to {}\n\nTest: make\n'.format(
+ args.path, updater.latest_version)
git_utils.add_file(full_path, '*')
git_utils.commit(full_path, msg)
@@ -149,9 +155,13 @@
else:
print(color_string(' Up to date.', Color.FRESH))
- if update_lib and (has_new_version or args.force):
+ if update_lib and args.refresh:
+ print('Refreshing the current version')
+ updater.use_current_as_latest()
+ if update_lib and (has_new_version or args.force or args.refresh):
_do_update(args, updater, metadata)
return updater
+ # pylint: disable=broad-except
except Exception as err:
print('{} {}.'.format(color_string('Failed.', Color.ERROR), err))
return str(err)
@@ -237,6 +247,14 @@
'--force',
help='Run update even if there\'s no new version.',
action='store_true')
+ update_parser.add_argument(
+ '--refresh',
+ help='Run update and refresh to the current version.',
+ action='store_true')
+ update_parser.add_argument(
+ '--keep_date',
+ help='Run update and do not change date in METADATA.',
+ action='store_true')
update_parser.add_argument('--branch_and_commit',
action='store_true',
help='Starts a new branch and commit changes.')
diff --git a/external_updater_reviewers_test.py b/external_updater_reviewers_test.py
new file mode 100644
index 0000000..f9c2014
--- /dev/null
+++ b/external_updater_reviewers_test.py
@@ -0,0 +1,148 @@
+# Copyright (C) 2020 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Unit tests for external updater reviewers."""
+
+from typing import List, Mapping, Set
+import unittest
+
+import reviewers
+
+
+class ExternalUpdaterReviewersTest(unittest.TestCase):
+ """Unit tests for external updater reviewers."""
+
+ def setUp(self):
+ super().setUp()
+ # save constants in reviewers
+ self.saved_proj_reviewers = reviewers.PROJ_REVIEWERS
+ self.saved_rust_reviewers = reviewers.RUST_REVIEWERS
+ self.saved_rust_reviewer_list = reviewers.RUST_REVIEWER_LIST
+ self.saved_num_rust_projects = reviewers.NUM_RUST_PROJECTS
+ self.saved_rust_crate_owners = reviewers.RUST_CRATE_OWNERS
+
+ def tearDown(self):
+ super().tearDown()
+ # restore constants in reviewers
+ reviewers.PROJ_REVIEWERS = self.saved_proj_reviewers
+ reviewers.RUST_REVIEWERS = self.saved_rust_reviewers
+ reviewers.RUST_REVIEWER_LIST = self.saved_rust_reviewer_list
+ reviewers.NUM_RUST_PROJECTS = self.saved_num_rust_projects
+ reviewers.RUST_CRATE_OWNERS = self.saved_rust_crate_owners
+
+ # pylint: disable=no-self-use
+ def _collect_reviewers(self, num_runs, proj_path):
+ counters = {}
+ for _ in range(num_runs):
+ name = reviewers.find_reviewers(proj_path)
+ if name in counters:
+ counters[name] += 1
+ else:
+ counters[name] = 1
+ return counters
+
+ def test_reviewers_types(self):
+ """Check the types of PROJ_REVIEWERS and RUST_REVIEWERS."""
+ # Check type of PROJ_REVIEWERS
+ self.assertIsInstance(reviewers.PROJ_REVIEWERS, Mapping)
+ for key, value in reviewers.PROJ_REVIEWERS.items():
+ self.assertIsInstance(key, str)
+ # pylint: disable=isinstance-second-argument-not-valid-type
+ # https://github.com/PyCQA/pylint/issues/3507
+ if isinstance(value, (List, Set)):
+ for x in value:
+ self.assertIsInstance(x, str)
+ else:
+ self.assertIsInstance(value, str)
+ # Check element types of the reviewers list and map.
+ self.assertIsInstance(reviewers.RUST_REVIEWERS, Mapping)
+ for (name, quota) in reviewers.RUST_REVIEWERS.items():
+ self.assertIsInstance(name, str)
+ self.assertIsInstance(quota, int)
+
+ def test_reviewers_constants(self):
+ """Check the constants associated to the reviewers."""
+ # There should be enough people in the reviewers pool.
+ self.assertGreaterEqual(len(reviewers.RUST_REVIEWERS), 3)
+ # The NUM_RUST_PROJECTS should not be too small.
+ self.assertGreaterEqual(reviewers.NUM_RUST_PROJECTS, 50)
+ self.assertGreaterEqual(reviewers.NUM_RUST_PROJECTS,
+ len(reviewers.RUST_CRATE_OWNERS))
+ # chh@ should be in many projects and not in RUST_REVIEWER_LIST
+ self.assertGreaterEqual(len(reviewers.RUST_REVIEWER_LIST), 3)
+ self.assertNotIn("chh@google.com", reviewers.RUST_REVIEWER_LIST)
+ self.assertIn("chh@google.com", reviewers.RUST_REVIEWERS)
+ # Assume no project reviewers and recreate RUST_REVIEWER_LIST
+ reviewers.PROJ_REVIEWERS = {}
+ reviewers.RUST_REVIEWER_LIST = reviewers.create_rust_reviewer_list()
+ sum_projects = sum(reviewers.RUST_REVIEWERS.values())
+ self.assertEqual(sum_projects, len(reviewers.RUST_REVIEWER_LIST))
+ self.assertGreaterEqual(sum_projects, reviewers.NUM_RUST_PROJECTS)
+
+ def test_reviewers_randomness(self):
+ """Check random selection of reviewers."""
+ # This might fail when the random.choice function is extremely unfair.
+ # With N * 20 tries, each reviewer should be picked at least twice.
+ # Assume no project reviewers and recreate RUST_REVIEWER_LIST
+ reviewers.PROJ_REVIEWERS = {}
+ reviewers.RUST_REVIEWER_LIST = reviewers.create_rust_reviewer_list()
+ num_tries = len(reviewers.RUST_REVIEWERS) * 20
+ counters = self._collect_reviewers(num_tries, "rust/crates/libc")
+ self.assertEqual(len(counters), len(reviewers.RUST_REVIEWERS))
+ for n in counters.values():
+ self.assertGreaterEqual(n, 10)
+ self.assertEqual(sum(counters.values()), num_tries)
+
+ def test_project_reviewers(self):
+ """For specific projects, select only the specified reviewers."""
+ reviewers.PROJ_REVIEWERS = {
+ "rust/crates/p1": "x@g.com",
+ "rust/crates/p_any": ["x@g.com", "y@g.com"],
+ "rust/crates/p_all": {"z@g", "x@g.com", "y@g.com"},
+ }
+ counters = self._collect_reviewers(20, "external/rust/crates/p1")
+ self.assertEqual(len(counters), 1)
+ self.assertTrue(counters["r=x@g.com"], 20)
+ counters = self._collect_reviewers(20, "external/rust/crates/p_any")
+ self.assertEqual(len(counters), 2)
+ self.assertGreater(counters["r=x@g.com"], 2)
+ self.assertGreater(counters["r=y@g.com"], 2)
+ self.assertTrue(counters["r=x@g.com"] + counters["r=y@g.com"], 20)
+ counters = self._collect_reviewers(20, "external/rust/crates/p_all")
+ # {x, y, z} reviewers should be sorted
+ self.assertEqual(counters["r=x@g.com,r=y@g.com,r=z@g"], 20)
+
+ def test_weighted_reviewers(self):
+ """Test create_rust_reviewer_list."""
+ reviewers.PROJ_REVIEWERS = {
+ "any_p1": "x@g", # 1 for x@g
+ "any_p2": {"xyz", "x@g"}, # 1 for x@g, xyz is not a rust reviewer
+ "any_p3": {"abc", "x@g"}, # 0.5 for "abc" and "x@g"
+ }
+ reviewers.RUST_REVIEWERS = {
+ "x@g": 5, # ceil(5 - 2.5) = 3
+ "abc": 2, # ceil(2 - 0.5) = 2
+ }
+ reviewer_list = reviewers.create_rust_reviewer_list()
+ self.assertEqual(reviewer_list, ["x@g", "x@g", "x@g", "abc", "abc"])
+ # Error case: if nobody has project quota, reset everyone to 1.
+ reviewers.RUST_REVIEWERS = {
+ "x@g": 1, # ceil(1 - 2.5) = -1
+ "abc": 0, # ceil(0 - 0.5) = 0
+ }
+ reviewer_list = reviewers.create_rust_reviewer_list()
+ self.assertEqual(reviewer_list, ["x@g", "abc"]) # everyone got 1
+
+
+if __name__ == "__main__":
+ unittest.main(verbosity=2)
diff --git a/external_updater_test.py b/external_updater_test.py
index b834fed..c274be5 100644
--- a/external_updater_test.py
+++ b/external_updater_test.py
@@ -20,6 +20,7 @@
class ExternalUpdaterTest(unittest.TestCase):
"""Unit tests for external updater."""
+
def test_url_selection(self):
"""Tests that GithubArchiveUpdater can choose the right url."""
prefix = "https://github.com/author/project/"
@@ -43,5 +44,5 @@
self.assertEqual(url, expected_url)
-if __name__ == '__main__':
- unittest.main()
+if __name__ == "__main__":
+ unittest.main(verbosity=2)
diff --git a/fileutils.py b/fileutils.py
index d7dd0fa..f6a51cc 100644
--- a/fileutils.py
+++ b/fileutils.py
@@ -66,7 +66,7 @@
return text_format.Parse(metadata, metadata_pb2.MetaData())
-def write_metadata(proj_path: Path, metadata: metadata_pb2.MetaData) -> None:
+def write_metadata(proj_path: Path, metadata: metadata_pb2.MetaData, keep_date: bool) -> None:
"""Writes updated METADATA file for a project.
This function updates last_upgrade_date in metadata and write to the project
@@ -75,13 +75,15 @@
Args:
proj_path: Path to the project.
metadata: The MetaData proto to write.
+ keep_date: Do not change date.
"""
- date = metadata.third_party.last_upgrade_date
- now = datetime.datetime.now()
- date.year = now.year
- date.month = now.month
- date.day = now.day
+ if not keep_date:
+ date = metadata.third_party.last_upgrade_date
+ now = datetime.datetime.now()
+ date.year = now.year
+ date.month = now.month
+ date.day = now.day
text_metadata = text_format.MessageToString(metadata)
with get_metadata_path(proj_path).open('w') as metadata_file:
metadata_file.write(text_metadata)
diff --git a/git_utils.py b/git_utils.py
index 69dcfba..0304417 100644
--- a/git_utils.py
+++ b/git_utils.py
@@ -19,6 +19,8 @@
from pathlib import Path
from typing import Dict, List, Tuple
+import hashtags
+import reviewers
def _run(cmd: List[str], cwd: Path) -> str:
"""Runs a command and returns its output."""
@@ -84,6 +86,7 @@
return out.splitlines()
+# pylint: disable=redefined-outer-name
def get_commit_time(proj_path: Path, commit: str) -> datetime.datetime:
"""Gets commit time of one commit."""
out = _run(['git', 'show', '-s', '--format=%ct', commit], cwd=proj_path)
@@ -103,11 +106,9 @@
def list_remote_tags(proj_path: Path, remote_name: str) -> List[str]:
"""Lists all tags for a remote."""
+ regex = re.compile(r".*refs/tags/(?P<tag>[^\^]*).*")
def parse_remote_tag(line: str) -> str:
- tag_prefix = 'refs/tags/'
- tag_suffix = '^{}'
- line = line[line.index(tag_prefix):]
- return line.lstrip(tag_prefix).rstrip(tag_suffix)
+ return regex.match(line).group("tag")
lines = _run(['git', "ls-remote", "--tags", remote_name],
cwd=proj_path).splitlines()
@@ -119,6 +120,7 @@
COMMIT_RE = re.compile(COMMIT_PATTERN)
+# pylint: disable=redefined-outer-name
def is_commit(commit: str) -> bool:
"""Whether a string looks like a SHA1 hash."""
return bool(COMMIT_RE.match(commit))
@@ -161,4 +163,9 @@
def push(proj_path: Path, remote_name: str) -> None:
"""Pushes change to remote."""
- _run(['git', 'push', remote_name, 'HEAD:refs/for/master'], cwd=proj_path)
+ cmd = ['git', 'push', remote_name, 'HEAD:refs/for/master']
+ if revs := reviewers.find_reviewers(str(proj_path)):
+ cmd.extend(['-o', revs])
+ if tag := hashtags.find_hashtag(proj_path):
+ cmd.extend(['-o', 't=' + tag])
+ _run(cmd, cwd=proj_path)
diff --git a/github_archive_updater.py b/github_archive_updater.py
index cc3840f..fc3e362 100644
--- a/github_archive_updater.py
+++ b/github_archive_updater.py
@@ -23,6 +23,7 @@
import archive_utils
from base_updater import Updater
import git_utils
+# pylint: disable=import-error
import metadata_pb2 # type: ignore
import updater_utils
@@ -88,6 +89,7 @@
return True
def _fetch_latest_release(self) -> Optional[Tuple[str, List[str]]]:
+ # pylint: disable=line-too-long
url = f'https://api.github.com/repos/{self.owner}/{self.repo}/releases/latest'
try:
with urllib.request.urlopen(url) as request:
@@ -109,6 +111,7 @@
for page in range(1, 21):
# Sleeps 10s to avoid rate limit.
time.sleep(10)
+ # pylint: disable=line-too-long
url = f'https://api.github.com/repos/{self.owner}/{self.repo}/tags?page={page}'
with urllib.request.urlopen(url) as request:
data = json.loads(request.read().decode())
@@ -133,11 +136,13 @@
def _fetch_latest_commit(self) -> None:
"""Checks upstream and gets the latest commit to master."""
+ # pylint: disable=line-too-long
url = f'https://api.github.com/repos/{self.owner}/{self.repo}/commits/master'
with urllib.request.urlopen(url) as request:
data = json.loads(request.read().decode())
self._new_ver = data['sha']
self._new_url.value = (
+ # pylint: disable=line-too-long
f'https://github.com/{self.owner}/{self.repo}/archive/{self._new_ver}.zip'
)
diff --git a/hashtags.py b/hashtags.py
new file mode 100644
index 0000000..9b043dd
--- /dev/null
+++ b/hashtags.py
@@ -0,0 +1,22 @@
+# Copyright (C) 2020 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Find main reviewers for git push commands."""
+
+from pathlib import Path
+
+def find_hashtag(proj_path: Path) -> str:
+ """Returns an empty string or a hashtag for git push."""
+ if str(proj_path).find('/external/rust/') != -1:
+ return 'external_updater_rust'
+ return 'external_updater'
diff --git a/notifier.py b/notifier.py
index d061181..7e2e4c7 100644
--- a/notifier.py
+++ b/notifier.py
@@ -29,8 +29,7 @@
import subprocess
import time
-import git_utils
-
+# pylint: disable=invalid-name
def parse_args():
"""Parses commandline arguments."""
@@ -96,6 +95,15 @@
encoding='ascii')
+COMMIT_PATTERN = r'^[a-f0-9]{40}$'
+COMMIT_RE = re.compile(COMMIT_PATTERN)
+
+
+def is_commit(commit: str) -> bool:
+ """Whether a string looks like a SHA1 hash."""
+ return bool(COMMIT_RE.match(commit))
+
+
NOTIFIED_TIME_KEY_NAME = 'latest_notified_time'
@@ -106,7 +114,7 @@
timestamp = proj_history.get(NOTIFIED_TIME_KEY_NAME, 0)
time_diff = datetime.today() - datetime.fromtimestamp(timestamp)
- if git_utils.is_commit(latest_ver) and time_diff <= timedelta(days=30):
+ if is_commit(latest_ver) and time_diff <= timedelta(days=30):
return False
return True
@@ -156,6 +164,7 @@
def _upgrade(proj):
+ # pylint: disable=subprocess-run-check
out = subprocess.run([
'out/soong/host/linux-x86/bin/external_updater', 'update',
'--branch_and_commit', '--push_change', proj
@@ -188,6 +197,7 @@
params += args.paths
print(_get_android_top())
+ # pylint: disable=subprocess-run-check
subprocess.run(params, cwd=_get_android_top())
diff --git a/regen_bp.sh b/regen_bp.sh
new file mode 100755
index 0000000..2ade8e4
--- /dev/null
+++ b/regen_bp.sh
@@ -0,0 +1,84 @@
+#!/bin/bash
+#
+# Copyright (C) 2020 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is used by external_updater to replace a package. Don't
+# invoke directly.
+
+set -e
+
+# Call this in two ways:
+# (1) in a .../external/* rust directory with .bp and Cargo.toml,
+# development/scripts/cargo2android.py must be in PATH
+# (2) in a tmp new directory with .bp and Cargo.toml,
+# and $1 equals to the rust Android source tree root,
+# and $2 equals to the rust sub-directory path name under external.
+if [ "$1" == "" ]; then
+ external_dir=`pwd`
+ C2A=`which cargo2android.py`
+ if [ "$C2A" == "" ]; then
+ echo "ERROR: cannot find cargo2android.py in PATH"
+ exit 1
+ fi
+else
+ external_dir="$2" # e.g. rust/crates/bytes
+ C2A="$1/development/scripts/cargo2android.py"
+ if [ ! -f $C2A ]; then
+ echo "ERROR: cannot find $C2A"
+ exit 1
+ fi
+fi
+
+# Save Cargo.lock if it existed before this update.
+if [ -f Cargo.lock ]; then
+ mv Cargo.lock Cargo.lock.saved
+fi
+
+LINE1=`head -1 Android.bp`
+FLAGS=`echo $LINE1 | sed -e 's:^.*cargo2android.py ::;s:\.$::'`
+CMD="$C2A $FLAGS"
+echo "Updating Android.bp: $CMD"
+$CMD
+
+if [ -d $2/out ]; then
+ # copy files generated by cargo build to out directory
+ PKGNAME=`basename $2`
+ for f in $2/out/*
+ do
+ OUTF=`basename $f`
+ SRC=`ls ./target.tmp/x86_64-unknown-linux-gnu/debug/build/$PKGNAME-*/out/$OUTF ||
+ ls ./target.tmp/debug/build/$PKGNAME-*/out/$OUTF || true`
+ if [ "$SRC" != "" ]; then
+ echo "Copying $SRC to out/$OUTF"
+ mkdir -p out
+ cp $SRC out/$OUTF
+ fi
+ done
+fi
+rm -rf target.tmp cargo.out Cargo.lock
+
+# Restore Cargo.lock if it existed before this update.
+if [ -f Cargo.lock.saved ]; then
+ mv Cargo.lock.saved Cargo.lock
+fi
+
+# Some .bp files have manual changes that cannot be fixed by post_update.sh.
+# Add a note to force a manual edit.
+case $external_dir in
+ */libloading|*/libsqlite3-sys|*/unicode-xid)
+ echo "FIXME: Copy manual changes from old version!" >> Android.bp
+esac
+
+exit 0
diff --git a/reviewers.py b/reviewers.py
new file mode 100644
index 0000000..c6f48d6
--- /dev/null
+++ b/reviewers.py
@@ -0,0 +1,169 @@
+# Copyright (C) 2020 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Find main reviewers for git push commands."""
+
+import math
+import random
+from typing import List, Mapping, Set, Union
+
+# To randomly pick one of multiple reviewers, we put them in a List[str]
+# to work with random.choice efficiently.
+# To pick all of multiple reviewers, we use a Set[str].
+
+# A ProjMapping maps a project path string to
+# (1) a single reviewer email address as a string, or
+# (2) a List of multiple reviewers to be randomly picked, or
+# (3) a Set of multiple reviewers to be all added.
+ProjMapping = Mapping[str, Union[str, List[str], Set[str]]]
+
+# Upgrades of Rust futures-* crates should be reviewed/tested together,
+# not split to radomly different people. In additional to one rust-dev
+# reviewer, we need one crosvm-dev reviewer to check the impact to crosvm.
+RUST_FUTURES_REVIEWERS: Set[str] = {'chh@google.com', 'natsu@google.com'}
+
+# Rust crate owners (reviewers).
+RUST_CRATE_OWNERS: ProjMapping = {
+ 'rust/crates/aho-corasick': 'chh@google.com',
+ 'rust/crates/anyhow': 'mmaurer@google.com',
+ 'rust/crates/bindgen': 'chh@google.com',
+ 'rust/crates/bytes': 'chh@google.com',
+ 'rust/crates/cexpr': 'chh@google.com',
+ 'rust/crates/cfg-if': 'chh@google.com',
+ 'rust/crates/clang-sys': 'chh@google.com',
+ 'rust/crates/futures': RUST_FUTURES_REVIEWERS,
+ 'rust/crates/futures-channel': RUST_FUTURES_REVIEWERS,
+ 'rust/crates/futures-core': RUST_FUTURES_REVIEWERS,
+ 'rust/crates/futures-executor': RUST_FUTURES_REVIEWERS,
+ 'rust/crates/futures-io': RUST_FUTURES_REVIEWERS,
+ 'rust/crates/futures-macro': RUST_FUTURES_REVIEWERS,
+ 'rust/crates/futures-sink': RUST_FUTURES_REVIEWERS,
+ 'rust/crates/futures-task': RUST_FUTURES_REVIEWERS,
+ 'rust/crates/futures-util': RUST_FUTURES_REVIEWERS,
+ 'rust/crates/glob': 'chh@google.com',
+ 'rust/crates/lazycell': 'chh@google.com',
+ 'rust/crates/lazy_static': 'chh@google.com',
+ 'rust/crates/libloading': 'chh@google.com',
+ 'rust/crates/log': 'chh@google.com',
+ 'rust/crates/nom': 'chh@google.com',
+ 'rust/crates/once_cell': 'chh@google.com',
+ 'rust/crates/peeking_take_while': 'chh@google.com',
+ 'rust/crates/pin-project': 'chh@google.com',
+ 'rust/crates/pin-project-internal': 'chh@google.com',
+ 'rust/crates/protobuf': 'chh@google.com',
+ 'rust/crates/protobuf-codegen': 'chh@google.com',
+ 'rust/crates/regex': 'chh@google.com',
+ 'rust/crates/regex-syntax': 'chh@google.com',
+ 'rust/crates/rustc-hash': 'chh@google.com',
+ 'rust/crates/shlex': 'chh@google.com',
+ 'rust/crates/thread_local': 'chh@google.com',
+ 'rust/crates/which': 'chh@google.com',
+ # more rust crate owners could be added later
+}
+
+PROJ_REVIEWERS: ProjMapping = {
+ # define non-rust project reviewers here
+}
+
+# Combine all roject reviewers.
+PROJ_REVIEWERS.update(RUST_CRATE_OWNERS)
+
+# Estimated number of rust projects, not the actual number.
+# It is only used to make random distribution "fair" among RUST_REVIEWERS.
+# It should not be too small, to spread nicely to multiple reviewers.
+# It should be larger or equal to len(RUST_CRATES_OWNERS).
+NUM_RUST_PROJECTS = 90
+
+# Reviewers for external/rust/crates projects not found in PROJ_REVIEWER.
+# Each person has a quota, the number of projects to review.
+# Sum of these numbers should be greater or equal to NUM_RUST_PROJECTS
+# to avoid error cases in the creation of RUST_REVIEWER_LIST.
+RUST_REVIEWERS: Mapping[str, int] = {
+ 'chh@google.com': 15,
+ 'ivanlozano@google.com': 15,
+ 'jeffv@google.com': 15,
+ 'jgalenson@google.com': 15,
+ 'mmaurer@google.com': 15,
+ 'srhines@google.com': 15,
+ 'tweek@google.com': 15,
+ # If a Rust reviewer needs to take a vacation, comment out the line,
+ # and distribute the quota to other reviewers.
+}
+
+
+# pylint: disable=invalid-name
+def add_proj_count(projects: Mapping[str, float], reviewer: str, n: float):
+ """Add n to the number of projects owned by the reviewer."""
+ if reviewer in projects:
+ projects[reviewer] += n
+ else:
+ projects[reviewer] = n
+
+
+# Random Rust reviewers are selected from RUST_REVIEWER_LIST,
+# which is created from RUST_REVIEWERS and PROJ_REVIEWERS.
+# A person P in RUST_REVIEWERS will occur in the RUST_REVIEWER_LIST N times,
+# if N = RUST_REVIEWERS[P] - (number of projects owned by P in PROJ_REVIEWERS)
+# is greater than 0. N is rounded up by math.ceil.
+def create_rust_reviewer_list() -> List[str]:
+ """Create a list of duplicated reviewers for weighted random selection."""
+ # Count number of projects owned by each reviewer.
+ rust_reviewers = set(RUST_REVIEWERS.keys())
+ projects = {} # map from owner to number of owned projects
+ for value in PROJ_REVIEWERS.values():
+ if isinstance(value, str): # single reviewer for a project
+ add_proj_count(projects, value, 1)
+ continue
+ # multiple reviewers share one project, count only rust_reviewers
+ # pylint: disable=bad-builtin
+ reviewers = set(filter(lambda x: x in rust_reviewers, value))
+ if reviewers:
+ count = 1.0 / len(reviewers) # shared among all reviewers
+ for name in reviewers:
+ add_proj_count(projects, name, count)
+ result = []
+ for (x, n) in RUST_REVIEWERS.items():
+ if x in projects: # reduce x's quota by the number of assigned ones
+ n = n - projects[x]
+ if n > 0:
+ result.extend([x] * math.ceil(n))
+ if result:
+ return result
+ # Something was wrong or quotas were too small so that nobody
+ # was selected from the RUST_REVIEWERS. Select everyone!!
+ return list(RUST_REVIEWERS.keys())
+
+
+RUST_REVIEWER_LIST: List[str] = create_rust_reviewer_list()
+
+
+def find_reviewers(proj_path: str) -> str:
+ """Returns an empty string or a reviewer parameter(s) for git push."""
+ index = proj_path.find('/external/')
+ if index >= 0: # full path
+ proj_path = proj_path[(index + len('/external/')):]
+ elif proj_path.startswith('external/'): # relative path
+ proj_path = proj_path[len('external/'):]
+ if proj_path in PROJ_REVIEWERS:
+ reviewers = PROJ_REVIEWERS[proj_path]
+ # pylint: disable=isinstance-second-argument-not-valid-type
+ if isinstance(reviewers, List): # pick any one reviewer
+ return 'r=' + random.choice(reviewers)
+ if isinstance(reviewers, Set): # add all reviewers in sorted order
+ # pylint: disable=bad-builtin
+ return ','.join(map(lambda x: 'r=' + x, sorted(reviewers)))
+ # reviewers must be a string
+ return 'r=' + reviewers
+ if proj_path.startswith('rust/crates/'):
+ return 'r=' + random.choice(RUST_REVIEWER_LIST)
+ return ''
diff --git a/update_package.sh b/update_package.sh
index d703466..663f647 100644
--- a/update_package.sh
+++ b/update_package.sh
@@ -22,6 +22,9 @@
tmp_dir=$1
external_dir=$2
+# root of Android source tree
+root_dir=`pwd`
+
echo "Entering $tmp_dir..."
cd $tmp_dir
@@ -47,6 +50,13 @@
CopyIfPresent "OWNERS"
CopyIfPresent "README.android"
+if [ -f $tmp_dir/Cargo.toml -a -f $tmp_dir/Android.bp ]
+then
+ # regenerate Android.bp before local patches, so it is
+ # possible to patch the generated Android.bp after this.
+ /bin/bash `dirname $0`/regen_bp.sh $root_dir $external_dir
+fi
+
echo "Applying patches..."
for p in $tmp_dir/patches/*.diff
do
diff --git a/updater_utils.py b/updater_utils.py
index 3301e8f..1bcd567 100644
--- a/updater_utils.py
+++ b/updater_utils.py
@@ -21,6 +21,7 @@
from typing import List, Tuple, Type
from base_updater import Updater
+# pylint: disable=import-error
import metadata_pb2 # type: ignore
@@ -80,6 +81,7 @@
versions = [int(v) for v in VERSION_SPLITTER_RE.split(version)]
return (versions, str(prefix), str(suffix))
except IndexError:
+ # pylint: disable=raise-missing-from
raise ValueError('Invalid version.')
