| #!/usr/bin/env python |
| # Copyright (C) 2017 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| import argparse |
| import hashlib |
| import logging |
| import os |
| import shutil |
| import subprocess |
| import sys |
| import tempfile |
| import zipfile |
| |
| from collections import namedtuple |
| from platform import system |
| |
| # The format for the deps below is the following: |
| # (target_folder, source_url, sha1, target_platform) |
| # |source_url| can be either a git repo or a http url. |
| # If a git repo, |sha1| is the committish that will be checked out. |
| # If a http url, |sha1| is the shasum of the original file. |
| # If the url is a .zip or .tgz file it will be automatically deflated under |
| # |target_folder|, taking care of stripping the root folder if it's a single |
| # root (to avoid ending up with buildtools/protobuf/protobuf-1.2.3/... and have |
| # instead just buildtools/protobuf). |
| # |target_platform| is either 'darwin', 'linux' or 'all' and applies the dep |
| # only on the given platform |
| Dependency = namedtuple( |
| 'Dependency', ['target_folder', 'source_url', 'sha1', 'target_platform']) |
| |
| # Dependencies required to build code on the host or when targeting desktop OS. |
| BUILD_DEPS_HOST = [ |
| # GN |
| Dependency('buildtools/mac/gn', |
| 'https://storage.googleapis.com/perfetto/gn-mac-1695-83dad00a', |
| '4c0d45772aea4146699772165e8112fa76ceb295', 'darwin'), |
| Dependency( |
| 'buildtools/linux64/gn', |
| 'https://storage.googleapis.com/perfetto/gn-linux64-1695-83dad00a', |
| 'fcabfc379bccaa65b4e2fc791594ba124dafc7d0', 'linux'), |
| |
| # clang-format |
| Dependency( |
| 'buildtools/mac/clang-format', |
| 'https://storage.googleapis.com/chromium-clang-format/62bde1baa7196ad9df969fc1f06b66360b1a927b', |
| '62bde1baa7196ad9df969fc1f06b66360b1a927b', 'darwin'), |
| Dependency( |
| 'buildtools/linux64/clang-format', |
| 'https://storage.googleapis.com/chromium-clang-format/1baf0089e895c989a311b6a38ed94d0e8be4c0a7', |
| '1baf0089e895c989a311b6a38ed94d0e8be4c0a7', 'linux'), |
| # Keep the SHA1 in sync with |clang_format_rev| in chromium //buildtools/DEPS. |
| Dependency( |
| 'buildtools/clang_format/script', |
| 'https://chromium.googlesource.com/chromium/llvm-project/cfe/tools/clang-format.git', |
| '96636aa0e9f047f17447f2d45a094d0b59ed7917', 'all'), |
| |
| # Ninja |
| Dependency( |
| 'buildtools/mac/ninja', |
| 'https://storage.googleapis.com/perfetto/ninja-mac-c15b0698da038b2bd2e8970c14c75fadc06b1add', |
| 'c15b0698da038b2bd2e8970c14c75fadc06b1add', 'darwin'), |
| Dependency( |
| 'buildtools/linux64/ninja', |
| 'https://storage.googleapis.com/perfetto/ninja-linux64-c866952bda50c29a669222477309287119bbb7e8', |
| 'c866952bda50c29a669222477309287119bbb7e8', 'linux'), |
| |
| # Keep in sync with Android's //external/googletest/README.version. |
| Dependency( |
| 'buildtools/googletest.zip', |
| 'https://github.com/google/googletest/archive/3f05f651ae3621db58468153e32016bc1397800b.zip', |
| '86384688f7c533ad325a505efc917e0cdf39a0ce', 'all'), |
| |
| # Keep in sync with Chromium's //third_party/protobuf. |
| Dependency( |
| 'buildtools/protobuf.zip', |
| 'https://github.com/protocolbuffers/protobuf/releases/download/v3.9.0/protobuf-cpp-3.9.0.zip', |
| 'c975536dffe9d9a3d362928aef4fb9f199012b98', 'all'), |
| |
| # libc++, libc++abi and libunwind for Linux where we need to rebuild the C++ |
| # lib from sources. Keep the SHA1s in sync with Chrome's src/buildtools/DEPS. |
| Dependency( |
| 'buildtools/libcxx', |
| 'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxx.git', |
| 'd9040c75cfea5928c804ab7c235fed06a63f743a', 'all'), |
| Dependency( |
| 'buildtools/libcxxabi', |
| 'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxxabi.git', |
| '196ba1aaa8ac285d94f4ea8d9836390a45360533', 'all'), |
| Dependency( |
| 'buildtools/libunwind', |
| 'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libunwind.git', |
| 'd999d54f4bca789543a2eb6c995af2d9b5a1f3ed', 'all'), |
| |
| # Keep the revision in sync with Chrome's PACKAGE_VERSION in |
| # tools/clang/scripts/update.py. |
| Dependency( |
| 'buildtools/clang.tgz', |
| 'https://commondatastorage.googleapis.com/chromium-browser-clang/Linux_x64/clang-llvmorg-12-init-5035-gd0abc757-3.tgz', |
| 'd5f013c57681a7b07dc17c18eaa927fab43fe5db', 'linux'), |
| |
| # Keep in sync with chromium DEPS. |
| Dependency( |
| 'buildtools/libfuzzer', |
| 'https://chromium.googlesource.com/chromium/llvm-project/compiler-rt/lib/fuzzer.git', |
| 'debe7d2d1982e540fbd6bd78604bf001753f9e74', 'linux'), |
| |
| # Benchmarking tool. |
| Dependency('buildtools/benchmark.zip', |
| 'https://github.com/google/benchmark/archive/v1.5.0.zip', |
| 'a9c9bd8a28db82f5ba02998197cfcc4db5a67507', 'all'), |
| |
| # Libbacktrace, for stacktraces in Linux/Android debug builds. |
| Dependency( |
| 'buildtools/libbacktrace.zip', |
| 'https://github.com/ianlancetaylor/libbacktrace/archive/177940370e4a6b2509e92a0aaa9749184e64af43.zip', |
| 'b723fe9d671d1ab54df1297f6afbf2893a41c3ea', 'all'), |
| |
| # Sqlite for the trace processing library. |
| # This is the amalgamated source whose compiled output is meant to be faster. |
| # We still pull the full source for the extensions (not amalgamated). |
| Dependency( |
| 'buildtools/sqlite.zip', |
| 'https://storage.googleapis.com/perfetto/sqlite-amalgamation-3320300.zip', |
| '0c805bea134712a903290a26b2a61c3a8a3bd8cc', 'all'), |
| Dependency( |
| 'buildtools/sqlite_src.zip', |
| 'https://storage.googleapis.com/perfetto/sqlite-src-3320300.zip', |
| 'd46f60e0fb2b1a959ae59bfa881fc95a510c4d21', 'all'), |
| |
| # JsonCpp for legacy json import. Used only by the trace processor in |
| # standalone builds. |
| Dependency( |
| 'buildtools/jsoncpp.zip', |
| 'https://github.com/open-source-parsers/jsoncpp/archive/1.9.3.zip', |
| 'ec1cf26bf4e60822dbb31576e1a83ce1b9fbc36a', 'all'), |
| |
| # These dependencies are for libunwindstack, which is used by src/profiling. |
| Dependency('buildtools/android-core', |
| 'https://android.googlesource.com/platform/system/core.git', |
| '278f11b57425d3a14369049e866c597980677988', 'all'), |
| Dependency('buildtools/libbase', |
| 'https://android.googlesource.com/platform/system/libbase.git', |
| '78f1c2f83e625bdf66d55b48bdb3a301c20d2fb3', 'all'), |
| Dependency('buildtools/lzma', |
| 'https://android.googlesource.com/platform/external/lzma.git', |
| '7851dce6f4ca17f5caa1c93a4e0a45686b1d56c3', 'all'), |
| Dependency('buildtools/zlib', |
| 'https://android.googlesource.com/platform/external/zlib.git', |
| 'dfa0646a03b4e1707469e04dc931b09774968fe6', 'all'), |
| Dependency('buildtools/bionic', |
| 'https://android.googlesource.com/platform/bionic.git', |
| 'a60488109cda997dfd83832731c8527feaa2825e', 'all'), |
| |
| # Example traces for regression tests. |
| Dependency( |
| 'buildtools/test_data.zip', |
| 'https://storage.googleapis.com/perfetto/test-data-20200910-141649.zip', |
| '8439e07570cbae4114f6495c68c4490c207bff58', |
| 'all', |
| ), |
| |
| # Linenoise, used only by trace_processor in standalone builds. |
| Dependency('buildtools/linenoise', |
| 'https://fuchsia.googlesource.com/third_party/linenoise.git', |
| 'c894b9e59f02203dbe4e2be657572cf88c4230c3', 'all'), |
| ] |
| |
| # Dependencies required to build Android code. |
| # URLs and SHA1s taken from: |
| # - https://dl.google.com/android/repository/repository-11.xml |
| # - https://dl.google.com/android/repository/sys-img/android/sys-img.xml |
| BUILD_DEPS_ANDROID = [ |
| # Android NDK |
| Dependency( |
| 'buildtools/ndk.zip', |
| 'https://dl.google.com/android/repository/android-ndk-r17b-darwin-x86_64.zip', |
| 'f990aafaffec0b583d2c5420bfa622e52ac14248', 'darwin'), |
| Dependency( |
| 'buildtools/ndk.zip', |
| 'https://dl.google.com/android/repository/android-ndk-r17b-linux-x86_64.zip', |
| 'dd5762ee7ef4995ad04fe0c45a608c344d99ca9f', 'linux'), |
| ] |
| |
| # Dependencies required to run Android tests. |
| TEST_DEPS_ANDROID = [ |
| # Android emulator images. |
| Dependency('buildtools/aosp-arm.zip', |
| 'https://storage.googleapis.com/perfetto/aosp-02022018-arm.zip', |
| 'a480d5e7d3ca888b0a58fe15ce76b1791537429a', 'all'), |
| |
| # platform-tools.zip contains adb binaries. |
| Dependency( |
| 'buildtools/android_sdk/platform-tools.zip', |
| 'https://dl.google.com/android/repository/platform-tools_r26.0.0-darwin.zip', |
| 'e75b6137dc444f777eb02f44a6d9819b3aabff82', 'darwin'), |
| Dependency( |
| 'buildtools/android_sdk/platform-tools.zip', |
| 'https://dl.google.com/android/repository/platform-tools_r26.0.0-linux.zip', |
| '00de8a6631405b617c10f68cd11ff2e1cd528e23', 'linux'), |
| |
| # Android emulator binaries. |
| Dependency( |
| 'buildtools/emulator', |
| 'https://android.googlesource.com/platform/prebuilts/android-emulator.git', |
| '4b260028dc27bc92c39bee9129cb2ba839970956', 'all'), |
| ] |
| |
| # This variable is updated by tools/roll-catapult-trace-viewer. |
| CATAPULT_SHA1 = '5f77256e1b24851a05e8580438b532e2480dd7fd' |
| |
| TYPEFACES_SHA1 = '4fb455de506f8a2859dc5264b8448c2559b08ab8' |
| |
| UI_DEPS = [ |
| Dependency( |
| 'buildtools/nodejs.tgz', |
| 'https://storage.googleapis.com/perfetto/node-v12.18.3-darwin-x64.tar.gz', |
| '8a7a901b7c1447235bfe8575a1c8a78eb14afbcf', 'darwin'), |
| Dependency( |
| 'buildtools/nodejs.tgz', |
| 'https://storage.googleapis.com/perfetto/node-v12.18.3-linux-x64.tar.gz', |
| 'c02138c2f9a813f88e81d31c4f9e5652dd7a9684', 'linux'), |
| Dependency( |
| 'buildtools/emsdk/emscripten.tgz', |
| 'https://storage.googleapis.com/perfetto/emscripten-1.37.40.tar.gz', |
| '588c28221321ebbdfc8e3a6f47ea6106f589669b', 'all'), |
| Dependency( |
| 'buildtools/emsdk/llvm.tgz', |
| 'https://storage.googleapis.com/perfetto/emscripten-llvm-e1.37.40-darwin.tar.gz', |
| '7a894ef0a52821c62f6abaac552dc4ce5d424607', 'darwin'), |
| Dependency( |
| 'buildtools/emsdk/llvm.tgz', |
| 'https://storage.googleapis.com/perfetto/emscripten-llvm-e1.37.40-static-linux.tar.gz', |
| '478501b9b7a14884e546c84efe209a90052cbb07', 'linux'), |
| Dependency( |
| 'buildtools/catapult_trace_viewer.tgz', |
| 'https://storage.googleapis.com/perfetto/catapult_trace_viewer-%s.tar.gz' |
| % CATAPULT_SHA1, CATAPULT_SHA1, 'all'), |
| Dependency( |
| 'buildtools/typefaces.tgz', |
| 'https://storage.googleapis.com/perfetto/typefaces-%s.tar.gz' % |
| TYPEFACES_SHA1, TYPEFACES_SHA1, 'all') |
| ] |
| |
| ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| UI_DIR = os.path.join(ROOT_DIR, 'ui') |
| NODE_MODULES_STATUS_FILE = os.path.join(UI_DIR, 'node_modules', '.last_install') |
| |
| |
| def DownloadURL(url, out_file): |
| subprocess.check_call(['curl', '-L', '-#', '-o', out_file, url]) |
| |
| |
| def ReadFile(path): |
| if not os.path.exists(path): |
| return None |
| with open(path) as f: |
| return f.read().strip() |
| |
| |
| def MkdirRecursive(path): |
| # Works with both relative and absolute paths |
| cwd = '/' if path.startswith('/') else ROOT_DIR |
| for part in path.split('/'): |
| cwd = os.path.join(cwd, part) |
| if not os.path.exists(cwd): |
| os.makedirs(cwd) |
| else: |
| assert (os.path.isdir(cwd)) |
| |
| |
| def HashLocalFile(path): |
| if not os.path.exists(path): |
| return None |
| with open(path, 'rb') as f: |
| return hashlib.sha1(f.read()).hexdigest() |
| |
| |
| def ExtractZipfilePreservePermissions(zf, info, path): |
| zf.extract(info.filename, path=path) |
| target_path = os.path.join(path, info.filename) |
| min_acls = 0o755 if info.filename.endswith('/') else 0o644 |
| os.chmod(target_path, (info.external_attr >> 16) | min_acls) |
| |
| |
| def IsGitRepoCheckoutOutAtRevision(path, revision): |
| return ReadFile(os.path.join(path, '.git', 'HEAD')) == revision |
| |
| |
| def CheckoutGitRepo(path, git_url, revision, check_only): |
| if IsGitRepoCheckoutOutAtRevision(path, revision): |
| return False |
| if check_only: |
| return True |
| if os.path.exists(path): |
| shutil.rmtree(path) |
| MkdirRecursive(path) |
| logging.info('Fetching %s @ %s into %s', git_url, revision, path) |
| subprocess.check_call(['git', 'init', path], cwd=path) |
| subprocess.check_call( |
| ['git', 'fetch', '--quiet', '--depth', '1', git_url, revision], cwd=path) |
| subprocess.check_call(['git', 'checkout', revision, '--quiet'], cwd=path) |
| assert (IsGitRepoCheckoutOutAtRevision(path, revision)) |
| return True |
| |
| |
| def InstallNodeModules(force_clean=False): |
| if force_clean: |
| node_modules = os.path.join(UI_DIR, 'node_modules') |
| logging.info('Clearing %s', node_modules) |
| subprocess.check_call(['git', 'clean', '-qxffd', node_modules], |
| cwd=ROOT_DIR) |
| logging.info("Running npm install in {0}".format(UI_DIR)) |
| subprocess.check_call([os.path.join(UI_DIR, 'npm'), 'install', '--no-save'], |
| cwd=UI_DIR) |
| with open(NODE_MODULES_STATUS_FILE, 'w') as f: |
| f.write(HashLocalFile(os.path.join(UI_DIR, 'package-lock.json'))) |
| |
| |
| def CheckNodeModules(): |
| """Returns True if the modules are up-to-date. |
| |
| There doesn't seem to be an easy way to check node modules versions. Instead |
| just check if package-lock.json changed since the last `npm install` call. |
| """ |
| if not os.path.exists(NODE_MODULES_STATUS_FILE): |
| return False |
| with open(NODE_MODULES_STATUS_FILE, 'r') as f: |
| actual = f.read() |
| expected = HashLocalFile(os.path.join(UI_DIR, 'package-lock.json')) |
| return expected == actual |
| |
| |
| def CheckHashes(): |
| for deps in [BUILD_DEPS_HOST, BUILD_DEPS_ANDROID, TEST_DEPS_ANDROID, UI_DEPS]: |
| for dep in deps: |
| if dep.source_url.endswith('.git'): |
| continue |
| logging.info('Downloading %s from %s', dep.target_platform, |
| dep.source_url) |
| with tempfile.NamedTemporaryFile(delete=False) as f: |
| f.close() |
| DownloadURL(url, f.name) |
| actual_sha1 = HashLocalFile(f.name) |
| os.unlink(f.name) |
| if (actual_sha1 != dep.sha1): |
| logging.fatal('SHA1 mismatch for {} expected {} was {}'.format( |
| dep.source_url, dep.sha1, actual_sha1)) |
| |
| |
| def Main(): |
| parser = argparse.ArgumentParser() |
| parser.add_argument('--android', action='store_true') |
| parser.add_argument('--ui', action='store_true') |
| parser.add_argument('--check-only') |
| parser.add_argument('--verify', help='Check all URLs', action='store_true') |
| args = parser.parse_args() |
| if args.verify: |
| CheckHashes() |
| return 0 |
| deps = BUILD_DEPS_HOST |
| if args.android: |
| deps += BUILD_DEPS_ANDROID + TEST_DEPS_ANDROID |
| if args.ui: |
| deps += UI_DEPS |
| deps_updated = False |
| nodejs_updated = False |
| |
| for dep in deps: |
| if (dep.target_platform != 'all' and |
| dep.target_platform != system().lower()): |
| continue |
| local_path = os.path.join(ROOT_DIR, dep.target_folder) |
| if dep.source_url.endswith('.git'): |
| deps_updated |= CheckoutGitRepo(local_path, dep.source_url, dep.sha1, |
| args.check_only) |
| continue |
| is_zip = local_path.endswith('.zip') or local_path.endswith('.tgz') |
| zip_target_dir = local_path[:-4] if is_zip else None |
| zip_dir_stamp = os.path.join(zip_target_dir, '.stamp') if is_zip else None |
| |
| if ((not is_zip and HashLocalFile(local_path) == dep.sha1) or |
| (is_zip and ReadFile(zip_dir_stamp) == dep.sha1)): |
| continue |
| deps_updated = True |
| if args.check_only: |
| continue |
| MkdirRecursive(os.path.dirname(dep.target_folder)) |
| if HashLocalFile(local_path) != dep.sha1: |
| download_path = local_path + '.tmp' |
| logging.info('Downloading %s from %s', local_path, dep.source_url) |
| DownloadURL(dep.source_url, download_path) |
| os.chmod(download_path, 0o755) |
| actual_sha1 = HashLocalFile(download_path) |
| if (actual_sha1 != dep.sha1): |
| os.remove(download_path) |
| logging.fatal('SHA1 mismatch for {} expected {} was {}'.format( |
| download_path, dep.sha1, actual_sha1)) |
| return 1 |
| os.rename(download_path, local_path) |
| if 'nodejs' in dep.target_folder: |
| nodejs_updated = True |
| |
| assert (HashLocalFile(local_path) == dep.sha1) |
| |
| if is_zip: |
| logging.info('Extracting %s into %s' % (local_path, zip_target_dir)) |
| assert (os.path.commonprefix((ROOT_DIR, zip_target_dir)) == ROOT_DIR) |
| if os.path.exists(zip_target_dir): |
| logging.info('Deleting stale dir %s' % zip_target_dir) |
| shutil.rmtree(zip_target_dir) |
| |
| # Decompress the archive. |
| if local_path.endswith('.tgz'): |
| MkdirRecursive(zip_target_dir) |
| subprocess.check_call(['tar', '-xf', local_path], cwd=zip_target_dir) |
| elif local_path.endswith('.zip'): |
| with zipfile.ZipFile(local_path, 'r') as zf: |
| for info in zf.infolist(): |
| ExtractZipfilePreservePermissions(zf, info, zip_target_dir) |
| |
| # If the zip contains one root folder, rebase one level up moving all |
| # its sub files and folders inside |target_dir|. |
| subdir = os.listdir(zip_target_dir) |
| if len(subdir) == 1: |
| subdir = os.path.join(zip_target_dir, subdir[0]) |
| if os.path.isdir(subdir): |
| for subf in os.listdir(subdir): |
| shutil.move(os.path.join(subdir, subf), zip_target_dir) |
| os.rmdir(subdir) |
| |
| # Create stamp and remove the archive. |
| with open(zip_dir_stamp, 'w') as stamp_file: |
| stamp_file.write(dep.sha1) |
| os.remove(local_path) |
| |
| if args.ui: |
| # Needs to happen after nodejs is installed above. |
| if args.check_only: |
| deps_updated = not CheckNodeModules() |
| else: |
| InstallNodeModules(force_clean=nodejs_updated) |
| |
| if args.check_only: |
| if not deps_updated: |
| with open(args.check_only, 'w') as f: |
| f.write('OK') # The content is irrelevant, just keep GN happy. |
| return 0 |
| argz = ' '.join([x for x in sys.argv[1:] if '--check-only' not in x]) |
| sys.stderr.write('\033[91mBuild deps are stale. ' + |
| 'Please run tools/install-build-deps %s\033[0m' % argz) |
| return 1 |
| |
| if deps_updated: |
| # Stale binary files may be compiled against old sysroot headers that aren't |
| # tracked by gn. |
| logging.warning('Remember to run "gn clean <output_directory>" ' + |
| 'to avoid stale binary files.') |
| |
| |
| if __name__ == '__main__': |
| logging.basicConfig(level=logging.INFO) |
| sys.exit(Main()) |
