rust_uprev: fetch distfiles from local mirror
This adds a few extra steps to rust_uprev where rust sources and
rust-bootstrap prebuilts are fetched from the local mirror. This
ensures that these files exist and are readable, and that the
local versions (that will be used to compute the digests for the
manifest) match the ones on the mirror.
BUG=None
TEST=Ran the script with existing and non-existing rust versions
Change-Id: I9868b027356cf14fc10d9e56665c74ad26555345
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3182632
Tested-by: Bob Haarman <inglorion@chromium.org>
Reviewed-by: George Burgess <gbiv@chromium.org>
Commit-Queue: Bob Haarman <inglorion@chromium.org>
diff --git a/rust_tools/rust_uprev.py b/rust_tools/rust_uprev.py
index 492729e..011639d 100755
--- a/rust_tools/rust_uprev.py
+++ b/rust_tools/rust_uprev.py
@@ -49,6 +49,7 @@
EQUERY = 'equery'
GSUTIL = 'gsutil.py'
+MIRROR_PATH = 'gs://chromeos-localmirror/distfiles'
RUST_PATH = Path(
'/mnt/host/source/src/third_party/chromiumos-overlay/dev-lang/rust')
@@ -58,6 +59,15 @@
**kwargs).strip()
+def get_command_output_unchecked(command: List[str], *args, **kwargs) -> str:
+ return subprocess.run(command,
+ check=False,
+ stdout=subprocess.PIPE,
+ encoding='utf-8',
+ *args,
+ **kwargs).stdout.strip()
+
+
class RustVersion(NamedTuple):
"""NamedTuple represents a Rust version"""
major: int
@@ -93,6 +103,14 @@
int(m.group('patch')))
+def compute_rustc_src_name(version: RustVersion) -> str:
+ return f'rustc-{version}-src.tar.gz'
+
+
+def compute_rust_bootstrap_prebuilt_name(version: RustVersion) -> str:
+ return f'rust-bootstrap-{version}.tbz2'
+
+
def find_ebuild_for_package(name: str) -> os.PathLike:
"""Returns the path to the ebuild for the named package."""
return get_command_output([EQUERY, 'w', name])
@@ -369,7 +387,96 @@
subprocess.check_call(cmd)
+def fetch_distfile_from_mirror(name: str) -> None:
+ """Gets the named file from the local mirror.
+
+ This ensures that the file exists on the mirror, and
+ that we can read it. We overwrite any existing distfile
+ to ensure the checksums that update_manifest() records
+ match the file as it exists on the mirror.
+
+ This function also attempts to verify the ACL for
+ the file (which is expected to have READER permission
+ for allUsers). We can only see the ACL if the user
+ gsutil runs with is the owner of the file. If not,
+ we get an access denied error. We also count this
+ as a success, because it means we were able to fetch
+ the file even though we don't own it.
+ """
+ mirror_file = MIRROR_PATH + '/' + name
+ local_file = Path(get_distdir(), name)
+ cmd = [GSUTIL, 'cp', mirror_file, local_file]
+ logging.info('Running %r', cmd)
+ rc = subprocess.call(cmd)
+ if rc != 0:
+ logging.error(
+ """Could not fetch %s
+
+If the file does not yet exist at %s
+please download the file, verify its integrity
+with something like:
+
+curl -O https://static.rust-lang.org/dist/%s
+gpg --verify %s.asc
+
+You may need to import the signing key first, e.g.:
+
+gpg --recv-keys 85AB96E6FA1BE5FE
+
+Once you have verify the integrity of the file, upload
+it to the local mirror using gsutil cp.
+""", mirror_file, MIRROR_PATH, name, name)
+ raise Exception(f'Could not fetch {mirror_file}')
+ # Check that the ACL allows allUsers READER access.
+ # If we get an AccessDeniedAcception here, that also
+ # counts as a success, because we were able to fetch
+ # the file as a non-owner.
+ cmd = [GSUTIL, 'acl', 'get', mirror_file]
+ logging.info('Running %r', cmd)
+ output = get_command_output_unchecked(cmd, stderr=subprocess.STDOUT)
+ acl_verified = False
+ if 'AccessDeniedException:' in output:
+ acl_verified = True
+ else:
+ acl = json.loads(output)
+ for x in acl:
+ if x['entity'] == 'allUsers' and x['role'] == 'READER':
+ acl_verified = True
+ break
+ if not acl_verified:
+ logging.error('Output from acl get:\n%s', output)
+ raise Exception('Could not verify that allUsers has READER permission')
+
+
+def fetch_bootstrap_distfiles(old_version: RustVersion,
+ new_version: RustVersion) -> None:
+ """Fetches rust-bootstrap distfiles from the local mirror
+
+ Fetches the distfiles for a rust-bootstrap ebuild to ensure they
+ are available on the mirror and the local copies are the same as
+ the ones on the mirror.
+ """
+ fetch_distfile_from_mirror(compute_rust_bootstrap_prebuilt_name(old_version))
+ fetch_distfile_from_mirror(compute_rustc_src_name(new_version))
+
+
+def fetch_rust_distfiles(version: RustVersion) -> None:
+ """Fetches rust distfiles from the local mirror
+
+ Fetches the distfiles for a rust ebuild to ensure they
+ are available on the mirror and the local copies are
+ the same as the ones on the mirror.
+ """
+ fetch_distfile_from_mirror(compute_rustc_src_name(version))
+
+
+def get_distdir() -> os.PathLike:
+ """Returns portage's distdir."""
+ return get_command_output(['portageq', 'distdir'])
+
+
def update_manifest(ebuild_file: os.PathLike) -> None:
+ """Updates the MANIFEST for the ebuild at the given path."""
ebuild = Path(ebuild_file)
logging.info('Added "mirror" to RESTRICT to %s', ebuild.name)
flip_mirror_in_ebuild(ebuild, add=True)
@@ -451,7 +558,7 @@
def create_rust_uprev(rust_version: RustVersion,
maybe_template_version: Optional[RustVersion],
skip_compile: bool, run_step: Callable[[], T]) -> None:
- template_version, template_ebuild, _old_bootstrap_version = run_step(
+ template_version, template_ebuild, old_bootstrap_version = run_step(
'prepare uprev',
lambda: prepare_uprev(rust_version, maybe_template_version),
result_from_json=prepare_uprev_from_json,
@@ -459,6 +566,14 @@
if template_ebuild is None:
return
+ # The fetch steps will fail (on purpose) if the files they check for
+ # are not available on the mirror. To make them pass, fetch the
+ # required files yourself, verify their checksums, then upload them
+ # to the mirror.
+ run_step(
+ 'fetch bootstrap distfiles', lambda: fetch_bootstrap_distfiles(
+ old_bootstrap_version, template_version))
+ run_step('fetch rust distfiles', lambda: fetch_rust_distfiles(rust_version))
run_step('update bootstrap ebuild', lambda: update_bootstrap_ebuild(
template_version))
run_step(
diff --git a/rust_tools/rust_uprev_test.py b/rust_tools/rust_uprev_test.py
index 18e8d9e..0076139 100755
--- a/rust_tools/rust_uprev_test.py
+++ b/rust_tools/rust_uprev_test.py
@@ -20,6 +20,53 @@
from rust_uprev import RustVersion
+def _fail_command(cmd, *_args, **_kwargs):
+ err = subprocess.CalledProcessError(returncode=1, cmd=cmd)
+ err.stderr = b'mock failure'
+ raise err
+
+
+class FetchDistfileTest(unittest.TestCase):
+ """Tests rust_uprev.fetch_distfile_from_mirror()"""
+
+ @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles')
+ @mock.patch.object(subprocess, 'call', side_effect=_fail_command)
+ def test_fetch_difstfile_fail(self, *_args) -> None:
+ with self.assertRaises(subprocess.CalledProcessError):
+ rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz')
+
+ @mock.patch.object(rust_uprev,
+ 'get_command_output_unchecked',
+ return_value='AccessDeniedException: Access denied.')
+ @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles')
+ @mock.patch.object(subprocess, 'call', return_value=0)
+ def test_fetch_distfile_acl_access_denied(self, *_args) -> None:
+ rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz')
+
+ @mock.patch.object(
+ rust_uprev,
+ 'get_command_output_unchecked',
+ return_value='[ { "entity": "allUsers", "role": "READER" } ]')
+ @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles')
+ @mock.patch.object(subprocess, 'call', return_value=0)
+ def test_fetch_distfile_acl_ok(self, *_args) -> None:
+ rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz')
+
+ @mock.patch.object(
+ rust_uprev,
+ 'get_command_output_unchecked',
+ return_value='[ { "entity": "___fake@google.com", "role": "OWNER" } ]')
+ @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles')
+ @mock.patch.object(subprocess, 'call', return_value=0)
+ def test_fetch_distfile_acl_wrong(self, *_args) -> None:
+ with self.assertRaisesRegex(Exception, 'allUsers.*READER'):
+ with self.assertLogs(level='ERROR') as log:
+ rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz')
+ self.assertIn(
+ '[ { "entity": "___fake@google.com", "role": "OWNER" } ]',
+ '\n'.join(log.output))
+
+
class FindEbuildPathTest(unittest.TestCase):
"""Tests for rust_uprev.find_ebuild_path()"""
