| #!/usr/bin/env python2.7 |
| # Copyright 2017 gRPC authors. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| """Build and upload docker images to Google Container Registry per matrix.""" |
| |
| from __future__ import print_function |
| |
| import argparse |
| import atexit |
| import multiprocessing |
| import os |
| import shutil |
| import subprocess |
| import sys |
| import tempfile |
| |
| # Langauage Runtime Matrix |
| import client_matrix |
| |
| python_util_dir = os.path.abspath( |
| os.path.join(os.path.dirname(__file__), '../run_tests/python_utils')) |
| sys.path.append(python_util_dir) |
| import dockerjob |
| import jobset |
| |
| _IMAGE_BUILDER = 'tools/run_tests/dockerize/build_interop_image.sh' |
| _LANGUAGES = client_matrix.LANG_RUNTIME_MATRIX.keys() |
| # All gRPC release tags, flattened, deduped and sorted. |
| _RELEASES = sorted( |
| list( |
| set( |
| client_matrix.get_release_tag_name(info) |
| for lang in client_matrix.LANG_RELEASE_MATRIX.values() |
| for info in lang))) |
| |
| # Destination directory inside docker image to keep extra info from build time. |
| _BUILD_INFO = '/var/local/build_info' |
| |
| argp = argparse.ArgumentParser(description='Run interop tests.') |
| argp.add_argument( |
| '--gcr_path', |
| default='gcr.io/grpc-testing', |
| help='Path of docker images in Google Container Registry') |
| |
| argp.add_argument( |
| '--release', |
| default='master', |
| choices=['all', 'master'] + _RELEASES, |
| help='github commit tag to checkout. When building all ' |
| 'releases defined in client_matrix.py, use "all". Valid only ' |
| 'with --git_checkout.') |
| |
| argp.add_argument( |
| '-l', |
| '--language', |
| choices=['all'] + sorted(_LANGUAGES), |
| nargs='+', |
| default=['all'], |
| help='Test languages to build docker images for.') |
| |
| argp.add_argument( |
| '--git_checkout', |
| action='store_true', |
| help='Use a separate git clone tree for building grpc stack. ' |
| 'Required when using --release flag. By default, current' |
| 'tree and the sibling will be used for building grpc stack.') |
| |
| argp.add_argument( |
| '--git_checkout_root', |
| default='/export/hda3/tmp/grpc_matrix', |
| help='Directory under which grpc-go/java/main repo will be ' |
| 'cloned. Valid only with --git_checkout.') |
| |
| argp.add_argument( |
| '--keep', |
| action='store_true', |
| help='keep the created local images after uploading to GCR') |
| |
| argp.add_argument( |
| '--reuse_git_root', |
| default=False, |
| action='store_const', |
| const=True, |
| help='reuse the repo dir. If False, the existing git root ' |
| 'directory will removed before a clean checkout, because ' |
| 'reusing the repo can cause git checkout error if you switch ' |
| 'between releases.') |
| |
| argp.add_argument( |
| '--upload_images', |
| action='store_true', |
| help='If set, images will be uploaded to container registry after building.' |
| ) |
| |
| args = argp.parse_args() |
| |
| |
| def add_files_to_image(image, with_files, label=None): |
| """Add files to a docker image. |
| |
| image: docker image name, i.e. grpc_interop_java:26328ad8 |
| with_files: additional files to include in the docker image. |
| label: label string to attach to the image. |
| """ |
| tag_idx = image.find(':') |
| if tag_idx == -1: |
| jobset.message( |
| 'FAILED', 'invalid docker image %s' % image, do_newline=True) |
| sys.exit(1) |
| orig_tag = '%s_' % image |
| subprocess.check_output(['docker', 'tag', image, orig_tag]) |
| |
| lines = ['FROM ' + orig_tag] |
| if label: |
| lines.append('LABEL %s' % label) |
| |
| temp_dir = tempfile.mkdtemp() |
| atexit.register(lambda: subprocess.call(['rm', '-rf', temp_dir])) |
| |
| # Copy with_files inside the tmp directory, which will be the docker build |
| # context. |
| for f in with_files: |
| shutil.copy(f, temp_dir) |
| lines.append('COPY %s %s/' % (os.path.basename(f), _BUILD_INFO)) |
| |
| # Create a Dockerfile. |
| with open(os.path.join(temp_dir, 'Dockerfile'), 'w') as f: |
| f.write('\n'.join(lines)) |
| |
| jobset.message('START', 'Repackaging %s' % image, do_newline=True) |
| build_cmd = ['docker', 'build', '--rm', '--tag', image, temp_dir] |
| subprocess.check_output(build_cmd) |
| dockerjob.remove_image(orig_tag, skip_nonexistent=True) |
| |
| |
| def build_image_jobspec(runtime, env, gcr_tag, stack_base): |
| """Build interop docker image for a language with runtime. |
| |
| runtime: a <lang><version> string, for example go1.8. |
| env: dictionary of env to passed to the build script. |
| gcr_tag: the tag for the docker image (i.e. v1.3.0). |
| stack_base: the local gRPC repo path. |
| """ |
| basename = 'grpc_interop_%s' % runtime |
| tag = '%s/%s:%s' % (args.gcr_path, basename, gcr_tag) |
| build_env = {'INTEROP_IMAGE': tag, 'BASE_NAME': basename, 'TTY_FLAG': '-t'} |
| build_env.update(env) |
| image_builder_path = _IMAGE_BUILDER |
| if client_matrix.should_build_docker_interop_image_from_release_tag(lang): |
| image_builder_path = os.path.join(stack_base, _IMAGE_BUILDER) |
| build_job = jobset.JobSpec( |
| cmdline=[image_builder_path], |
| environ=build_env, |
| shortname='build_docker_%s' % runtime, |
| timeout_seconds=30 * 60) |
| build_job.tag = tag |
| return build_job |
| |
| |
| def build_all_images_for_lang(lang): |
| """Build all docker images for a language across releases and runtimes.""" |
| if not args.git_checkout: |
| if args.release != 'master': |
| print( |
| 'Cannot use --release without also enabling --git_checkout.\n') |
| sys.exit(1) |
| releases = [args.release] |
| else: |
| if args.release == 'all': |
| releases = client_matrix.get_release_tags(lang) |
| else: |
| # Build a particular release. |
| if args.release not in ['master' |
| ] + client_matrix.get_release_tags(lang): |
| jobset.message( |
| 'SKIPPED', |
| '%s for %s is not defined' % (args.release, lang), |
| do_newline=True) |
| return [] |
| releases = [args.release] |
| |
| images = [] |
| for release in releases: |
| images += build_all_images_for_release(lang, release) |
| jobset.message( |
| 'SUCCESS', |
| 'All docker images built for %s at %s.' % (lang, releases), |
| do_newline=True) |
| return images |
| |
| |
| def build_all_images_for_release(lang, release): |
| """Build all docker images for a release across all runtimes.""" |
| docker_images = [] |
| build_jobs = [] |
| |
| env = {} |
| # If we not using current tree or the sibling for grpc stack, do checkout. |
| stack_base = '' |
| if args.git_checkout: |
| stack_base = checkout_grpc_stack(lang, release) |
| var = { |
| 'go': 'GRPC_GO_ROOT', |
| 'java': 'GRPC_JAVA_ROOT', |
| 'node': 'GRPC_NODE_ROOT' |
| }.get(lang, 'GRPC_ROOT') |
| env[var] = stack_base |
| |
| for runtime in client_matrix.LANG_RUNTIME_MATRIX[lang]: |
| job = build_image_jobspec(runtime, env, release, stack_base) |
| docker_images.append(job.tag) |
| build_jobs.append(job) |
| |
| jobset.message('START', 'Building interop docker images.', do_newline=True) |
| print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs)) |
| |
| num_failures, _ = jobset.run( |
| build_jobs, |
| newline_on_success=True, |
| maxjobs=multiprocessing.cpu_count()) |
| if num_failures: |
| jobset.message( |
| 'FAILED', 'Failed to build interop docker images.', do_newline=True) |
| docker_images_cleanup.extend(docker_images) |
| sys.exit(1) |
| |
| jobset.message( |
| 'SUCCESS', |
| 'All docker images built for %s at %s.' % (lang, release), |
| do_newline=True) |
| |
| if release != 'master': |
| commit_log = os.path.join(stack_base, 'commit_log') |
| if os.path.exists(commit_log): |
| for image in docker_images: |
| add_files_to_image(image, [commit_log], 'release=%s' % release) |
| return docker_images |
| |
| |
| def cleanup(): |
| if not args.keep: |
| for image in docker_images_cleanup: |
| dockerjob.remove_image(image, skip_nonexistent=True) |
| |
| |
| docker_images_cleanup = [] |
| atexit.register(cleanup) |
| |
| |
| def maybe_apply_patches_on_git_tag(stack_base, lang, release): |
| files_to_patch = [] |
| for release_info in client_matrix.LANG_RELEASE_MATRIX[lang]: |
| if client_matrix.get_release_tag_name(release_info) == release: |
| if release_info[release] is not None: |
| files_to_patch = release_info[release].get('patch') |
| break |
| if not files_to_patch: |
| return |
| patch_file_relative_path = 'patches/%s_%s/git_repo.patch' % (lang, release) |
| patch_file = os.path.abspath( |
| os.path.join(os.path.dirname(__file__), patch_file_relative_path)) |
| if not os.path.exists(patch_file): |
| jobset.message('FAILED', |
| 'expected patch file |%s| to exist' % patch_file) |
| sys.exit(1) |
| subprocess.check_output( |
| ['git', 'apply', patch_file], cwd=stack_base, stderr=subprocess.STDOUT) |
| |
| # TODO(jtattermusch): this really would need simplification and refactoring |
| # - "git add" and "git commit" can easily be done in a single command |
| # - it looks like the only reason for the existence of the "files_to_patch" |
| # entry is to perform "git add" - which is clumsy and fragile. |
| # - we only allow a single patch with name "git_repo.patch". A better design |
| # would be to allow multiple patches that can have more descriptive names. |
| for repo_relative_path in files_to_patch: |
| subprocess.check_output( |
| ['git', 'add', repo_relative_path], |
| cwd=stack_base, |
| stderr=subprocess.STDOUT) |
| subprocess.check_output( |
| [ |
| 'git', 'commit', '-m', |
| ('Hack performed on top of %s git ' |
| 'tag in order to build and run the %s ' |
| 'interop tests on that tag.' % (lang, release)) |
| ], |
| cwd=stack_base, |
| stderr=subprocess.STDOUT) |
| |
| |
| def checkout_grpc_stack(lang, release): |
| """Invokes 'git check' for the lang/release and returns directory created.""" |
| assert args.git_checkout and args.git_checkout_root |
| |
| if not os.path.exists(args.git_checkout_root): |
| os.makedirs(args.git_checkout_root) |
| |
| repo = client_matrix.get_github_repo(lang) |
| # Get the subdir name part of repo |
| # For example, 'git@github.com:grpc/grpc-go.git' should use 'grpc-go'. |
| repo_dir = os.path.splitext(os.path.basename(repo))[0] |
| stack_base = os.path.join(args.git_checkout_root, repo_dir) |
| |
| # Clean up leftover repo dir if necessary. |
| if not args.reuse_git_root and os.path.exists(stack_base): |
| jobset.message('START', 'Removing git checkout root.', do_newline=True) |
| shutil.rmtree(stack_base) |
| |
| if not os.path.exists(stack_base): |
| subprocess.check_call( |
| ['git', 'clone', '--recursive', repo], |
| cwd=os.path.dirname(stack_base)) |
| |
| # git checkout. |
| jobset.message( |
| 'START', |
| 'git checkout %s from %s' % (release, stack_base), |
| do_newline=True) |
| # We should NEVER do checkout on current tree !!! |
| assert not os.path.dirname(__file__).startswith(stack_base) |
| output = subprocess.check_output( |
| ['git', 'checkout', release], cwd=stack_base, stderr=subprocess.STDOUT) |
| maybe_apply_patches_on_git_tag(stack_base, lang, release) |
| commit_log = subprocess.check_output(['git', 'log', '-1'], cwd=stack_base) |
| jobset.message( |
| 'SUCCESS', |
| 'git checkout', |
| '%s: %s' % (str(output), commit_log), |
| do_newline=True) |
| |
| # Write git log to commit_log so it can be packaged with the docker image. |
| with open(os.path.join(stack_base, 'commit_log'), 'w') as f: |
| f.write(commit_log) |
| return stack_base |
| |
| |
| languages = args.language if args.language != ['all'] else _LANGUAGES |
| for lang in languages: |
| docker_images = build_all_images_for_lang(lang) |
| for image in docker_images: |
| if args.upload_images: |
| jobset.message('START', 'Uploading %s' % image, do_newline=True) |
| # docker image name must be in the format <gcr_path>/<image>:<gcr_tag> |
| assert image.startswith(args.gcr_path) and image.find(':') != -1 |
| subprocess.call(['gcloud', 'docker', '--', 'push', image]) |
| else: |
| # Uploading (and overwriting images) by default can easily break things. |
| print('Not uploading image %s, run with --upload_images to upload.' |
| % image) |
