packager: Exclude pre-existing packager tarball and checksum
This patch prevents a pernicious error where repeated builds can lead
to the tar.bz2 files getting included in new tar.bz2 packages, leading
to exponential increases in tarball size and other weird errors in the
unpackaging of these files.
Also use this patch to add tags to some similar log prints, so it is
clear which stage of the build these are coming from.
Finally, the quoting on some of the exclude strings was incorrect. So,
this patch fixes those errors too.
BUG=chromium:718626
TEST=emerge-veyron_minnie chromeos-base/autotest-tests-wimax
Change-Id: Ibc8aceff2c608700e8bbe22f63b0ef7d0cd94d29
Reviewed-on: https://chromium-review.googlesource.com/498090
Reviewed-by: Mike Frysinger <vapier@chromium.org>
Reviewed-by: Ilja H. Friedel <ihf@chromium.org>
Reviewed-by: Aviv Keshet <akeshet@chromium.org>
Tested-by: Prashant Malani <pmalani@google.com>
Commit-Queue: Ilja H. Friedel <ihf@chromium.org>
Commit-Queue: Prashant Malani <pmalani@google.com>
Trybot-Ready: Prashant Malani <pmalani@google.com>
diff --git a/utils/packager.py b/utils/packager.py
index c219d44..fdf0abf 100755
--- a/utils/packager.py
+++ b/utils/packager.py
@@ -25,8 +25,8 @@
For profilers we need to exclude everything except the __init__.py
file so that the profilers can be imported.
'''
- exclude_string = ('--exclude=deps/* --exclude=tests/* '
- '--exclude=site_tests/* --exclude=**.pyc')
+ exclude_string = ('--exclude="deps/*" --exclude="tests/*" '
+ '--exclude="site_tests/*" --exclude="**.pyc"')
# Get the profilers directory
prof_dir = os.path.join(client_dir, 'profilers')
@@ -35,7 +35,7 @@
# subdirectories
for f in os.listdir(prof_dir):
if os.path.isdir(os.path.join(prof_dir, f)):
- exclude_string += ' --exclude=profilers/%s' % f
+ exclude_string += ' --exclude="profilers/%s"' % f
# The '.' here is needed to zip the files in the current
# directory. We use '-C' for tar to change to the required
@@ -122,7 +122,7 @@
exclude_string = ' .'
names = [p.strip() for p in pkg_names.split(',')]
for name in names:
- print "Processing %s ... " % name
+ print "process_packages: Processing %s ... " % name
if pkg_type == 'client':
pkg_dir = src_dir
exclude_string = get_exclude_string(pkg_dir)
@@ -137,6 +137,11 @@
pkg_name = pkgmgr.get_tarball_name(name, pkg_type)
if action == ACTION_TAR_ONLY:
+ # We don't want any pre-existing tarballs and checksums to
+ # be repackaged, so we should purge these.
+ exclude_string_tar = ((
+ ' --exclude="**%s" --exclude="**%s.checksum" ' %
+ (pkg_name, pkg_name)) + exclude_string)
build_dir = get_build_dir(name, dest_dir, pkg_type)
try:
packages.check_diskspace(build_dir)
@@ -145,7 +150,7 @@
"enough space available: %s" % (build_dir, e))
raise error.RepoDiskFullError(msg)
tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir,
- build_dir, exclude_string)
+ build_dir, exclude_string_tar)
# Create the md5 hash too.
md5sum = pkgmgr.compute_checksum(tarball_path)
@@ -168,7 +173,8 @@
# the effort.
tarball_path = os.path.join(pkg_dir, pkg_name);
if os.path.exists(tarball_path):
- print("Tarball %s already exists" % tarball_path)
+ print("process_packages: Tarball %s already exists" %
+ tarball_path)
else:
tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir,
temp_dir, exclude_string)
@@ -187,7 +193,7 @@
exclude_string = ' .'
names = [p.strip() for p in pkg_names.split(',')]
for name in names:
- print "Processing %s ... " % name
+ print "tar_packages: Processing %s ... " % name
if pkg_type == 'client':
pkg_dir = src_dir
exclude_string = get_exclude_string(pkg_dir)
@@ -201,14 +207,19 @@
pkg_name = pkgmgr.get_tarball_name(name, pkg_type)
+ # We don't want any pre-existing tarballs and checksums to
+ # be repackaged, so we should purge these.
+ exclude_string_tar = ((
+ ' --exclude="**%s" --exclude="**%s.checksum" ' %
+ (pkg_name, pkg_name)) + exclude_string)
# Check if tarball already exists. If it does, don't duplicate
# the effort.
tarball_path = os.path.join(pkg_dir, pkg_name);
if os.path.exists(tarball_path):
- print("Tarball %s already exists" % tarball_path);
+ print("tar_packages: Tarball %s already exists" % tarball_path);
else:
tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir,
- temp_dir, exclude_string)
+ temp_dir, exclude_string_tar)
tarballs.append(tarball_path)
return tarballs
