[RELAND] [RELAND] .circleci: Improve docker image build workflow (#38335)
Summary:
This reverts commit 6e66e8562f276e2015af8ff76437a3f0277c4bcc.
Two things learned from the previous reland:
* `cirlceci-agent step halt` doesn't actually halt the step in place, you must explicitly exit the step after the `step halt` is called
* Even though `circleci` uses `git` to checkout repositories inside of docker images, that does not mean `git` is available after the fact.
<details>
<summary> Changes from previous reland </summary>
```patch
commit cc99a12c9029472bd73325876bc0e9dbb1746b05
Author: Eli Uriegas <eliuriegas@fb.com>
Date: Tue May 12 10:58:18 2020 -0700
.cirlceci: Install git for gc, exit step explicitly
Signed-off-by: Eli Uriegas <eliuriegas@fb.com>
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 481d7889da..856a0fb10a 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -2018,13 +2018,15 @@ jobs:
export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
eval $(aws ecr get-login --no-include-email --region us-east-1)
set -x
+ PREVIOUS_DOCKER_TAG=$(git rev-parse "$(git merge-base HEAD << pipeline.git.base_revision >>):.circleci/docker")
# Check if image already exists, if it does then skip building it
if docker manifest inspect "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/${IMAGE_NAME}:${DOCKER_TAG}"; then
circleci-agent step halt
+ # circleci-agent step halt doesn't actually halt the step so we need to
+ # explicitly exit the step here ourselves before it causes too much trouble
+ exit 0
fi
- PREVIOUS_DOCKER_TAG=$(git rev-parse "$(git merge-base HEAD << pipeline.git.base_revision >>):.circleci/docker")
# If no image exists but the hash is the same as the previous hash then we should error out here
- # no stampeding herd effect plz.
if [[ ${PREVIOUS_DOCKER_TAG} = ${DOCKER_TAG} ]]; then
echo "ERROR: Something has gone wrong and the previous image isn't available for the merge-base of your branch"
echo " contact the PyTorch team to restore the original images"
diff --git a/.circleci/ecr_gc_docker/Dockerfile b/.circleci/ecr_gc_docker/Dockerfile
index d0198acb86..36347d5e6d 100644
--- a/.circleci/ecr_gc_docker/Dockerfile
+++ b/.circleci/ecr_gc_docker/Dockerfile
@@ -1,6 +1,6 @@
FROM ubuntu:16.04
-RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log
+RUN apt-get update && apt-get install -y git python-pip && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log
ADD requirements.txt /requirements.txt
diff --git a/.circleci/verbatim-sources/docker_jobs.yml b/.circleci/verbatim-sources/docker_jobs.yml
index e04d11c5cd..3918cc04ae 100644
--- a/.circleci/verbatim-sources/docker_jobs.yml
+++ b/.circleci/verbatim-sources/docker_jobs.yml
@@ -35,13 +35,15 @@
export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
eval $(aws ecr get-login --no-include-email --region us-east-1)
set -x
+ PREVIOUS_DOCKER_TAG=$(git rev-parse "$(git merge-base HEAD << pipeline.git.base_revision >>):.circleci/docker")
# Check if image already exists, if it does then skip building it
if docker manifest inspect "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/${IMAGE_NAME}:${DOCKER_TAG}"; then
circleci-agent step halt
+ # circleci-agent step halt doesn't actually halt the step so we need to
+ # explicitly exit the step here ourselves before it causes too much trouble
+ exit 0
fi
- PREVIOUS_DOCKER_TAG=$(git rev-parse "$(git merge-base HEAD << pipeline.git.base_revision >>):.circleci/docker")
# If no image exists but the hash is the same as the previous hash then we should error out here
- # no stampeding herd effect plz.
if [[ ${PREVIOUS_DOCKER_TAG} = ${DOCKER_TAG} ]]; then
echo "ERROR: Something has gone wrong and the previous image isn't available for the merge-base of your branch"
echo " contact the PyTorch team to restore the original images"
```
</details>
Pull Request resolved: https://github.com/pytorch/pytorch/pull/38335
Differential Revision: D21536269
Pulled By: seemethere
fbshipit-source-id: 5577f84fa49dd6e1e88fce461646fd68be3d417d
diff --git a/.circleci/cimodel/data/pytorch_build_definitions.py b/.circleci/cimodel/data/pytorch_build_definitions.py
index 6ef69b6..b560aa3 100644
--- a/.circleci/cimodel/data/pytorch_build_definitions.py
+++ b/.circleci/cimodel/data/pytorch_build_definitions.py
@@ -11,11 +11,6 @@
DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/"
-# ARE YOU EDITING THIS NUMBER? MAKE SURE YOU READ THE GUIDANCE AT THE
-# TOP OF .circleci/config.yml
-DOCKER_IMAGE_VERSION = "9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
-
-
@dataclass
class Conf:
distro: str
@@ -60,11 +55,14 @@
return result
def gen_docker_image_path(self):
+ return miniutils.quote(
+ DOCKER_IMAGE_PATH_BASE + self.gen_docker_image_name()
+ )
+ def gen_docker_image_name(self):
parms_source = self.parent_build or self
base_build_env_name = "-".join(parms_source.get_parms(True))
-
- return miniutils.quote(DOCKER_IMAGE_PATH_BASE + base_build_env_name + ":" + str(DOCKER_IMAGE_VERSION))
+ return base_build_env_name
def get_build_job_name_pieces(self, build_or_test):
return self.get_parms(False) + [build_or_test]
@@ -97,6 +95,9 @@
job_def["name"] = self.gen_build_name(phase)
job_def["requires"] = ["setup"]
+ if phase == "build":
+ job_def["requires"].append(miniutils.quote("docker-" + self.gen_docker_image_name()))
+
if phase == "test":
# TODO When merging the caffe2 and pytorch jobs, it might be convenient for a while to make a
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 37e9fd4..d7c079e 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -39,6 +39,21 @@
name: Attaching workspace
at: ~/workspace
+ # Must be run after attaching workspace from previous steps
+ load_shared_env:
+ description: "Loads .circleci/shared/env_file into ${BASH_ENV}"
+ parameters:
+ # For some weird reason we decide to reattach our workspace to ~/workspace so
+ # in the vein of making it simple let's assume our share env_file is here
+ root:
+ type: string
+ default: "~/workspace"
+ steps:
+ - run:
+ name: "Load .circleci/shared/env_file into ${BASH_ENV}"
+ command: |
+ cat << parameters.root >>/.circleci/shared/env_file >> ${BASH_ENV}
+
# This system setup script is meant to run before the CI-related scripts, e.g.,
# installing Git client, checking out code, setting up CI env, and
# building/testing.
@@ -401,6 +416,7 @@
steps:
# See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
- attach_scripts
+ - load_shared_env
- setup_linux_system_environment
- checkout
- optional_merge_target_branch
@@ -411,9 +427,9 @@
command: |
set -e
# Pull Docker image and run build
- echo "DOCKER_IMAGE: "${DOCKER_IMAGE}
- time docker pull ${DOCKER_IMAGE} >/dev/null
- export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
+ echo "DOCKER_IMAGE: "${DOCKER_IMAGE}:${DOCKER_TAG}
+ time docker pull ${DOCKER_IMAGE}:${DOCKER_TAG} >/dev/null
+ export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE}:${DOCKER_TAG})
git submodule sync && git submodule update -q --init --recursive
@@ -436,7 +452,7 @@
# The xla build uses the same docker image as
# pytorch-linux-trusty-py3.6-gcc5.4-build. In the push step, we have to
# distinguish between them so the test can pick up the correct image.
- output_image=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+ output_image=${DOCKER_IMAGE}:${DOCKER_TAG}-${CIRCLE_SHA1}
if [[ ${BUILD_ENVIRONMENT} == *"xla"* ]]; then
export COMMIT_DOCKER_IMAGE=$output_image-xla
elif [[ ${BUILD_ENVIRONMENT} == *"libtorch"* ]]; then
@@ -463,6 +479,7 @@
steps:
# See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
- attach_scripts
+ - load_shared_env
- setup_linux_system_environment
- setup_ci_environment
- run:
@@ -471,7 +488,7 @@
command: |
set -e
# See Note [Special build images]
- output_image=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+ output_image=${DOCKER_IMAGE}:${DOCKER_TAG}-${CIRCLE_SHA1}
if [[ ${BUILD_ENVIRONMENT} == *"xla"* ]]; then
export COMMIT_DOCKER_IMAGE=$output_image-xla
elif [[ ${BUILD_ENVIRONMENT} == *"libtorch"* ]]; then
@@ -1279,13 +1296,14 @@
environment:
BUILD_ENVIRONMENT: pytorch-python-doc-push
# TODO: stop hardcoding this
- DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
resource_class: large
machine:
image: ubuntu-1604:201903-01
steps:
# See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
- attach_scripts
+ - load_shared_env
- setup_linux_system_environment
- setup_ci_environment
- run:
@@ -1293,7 +1311,7 @@
no_output_timeout: "1h"
command: |
set -ex
- export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+ export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}:${DOCKER_TAG}-${CIRCLE_SHA1}
echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
@@ -1330,13 +1348,14 @@
pytorch_cpp_doc_push:
environment:
BUILD_ENVIRONMENT: pytorch-cpp-doc-push
- DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
resource_class: large
machine:
image: ubuntu-1604:201903-01
steps:
# See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
- attach_scripts
+ - load_shared_env
- setup_linux_system_environment
- setup_ci_environment
- run:
@@ -1344,7 +1363,7 @@
no_output_timeout: "1h"
command: |
set -ex
- export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+ export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}:${DOCKER_TAG}-${CIRCLE_SHA1}
echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
@@ -1440,13 +1459,14 @@
pytorch_android_gradle_build:
environment:
BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build
- DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
PYTHON_VERSION: "3.6"
resource_class: large
machine:
image: ubuntu-1604:201903-01
steps:
- attach_scripts
+ - load_shared_env
- setup_linux_system_environment
- checkout
- setup_ci_environment
@@ -1455,7 +1475,7 @@
no_output_timeout: "1h"
command: |
set -eux
- docker_image_commit=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+ docker_image_commit=${DOCKER_IMAGE}:${DOCKER_TAG}-${CIRCLE_SHA1}
docker_image_libtorch_android_x86_32=${docker_image_commit}-android-x86_32
docker_image_libtorch_android_x86_64=${docker_image_commit}-android-x86_64
@@ -1526,13 +1546,14 @@
pytorch_android_publish_snapshot:
environment:
BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-publish-snapshot
- DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
PYTHON_VERSION: "3.6"
resource_class: large
machine:
image: ubuntu-1604:201903-01
steps:
- attach_scripts
+ - load_shared_env
- setup_linux_system_environment
- checkout
- setup_ci_environment
@@ -1541,7 +1562,7 @@
no_output_timeout: "1h"
command: |
set -eux
- docker_image_commit=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+ docker_image_commit=${DOCKER_IMAGE}:${DOCKER_TAG}-${CIRCLE_SHA1}
docker_image_libtorch_android_x86_32_gradle=${docker_image_commit}-android-x86_32-gradle
@@ -1562,13 +1583,14 @@
pytorch_android_gradle_build-x86_32:
environment:
BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build-only-x86_32
- DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
PYTHON_VERSION: "3.6"
resource_class: large
machine:
image: ubuntu-1604:201903-01
steps:
- attach_scripts
+ - load_shared_env
- run:
name: filter out not PR runs
no_output_timeout: "5m"
@@ -1585,7 +1607,7 @@
no_output_timeout: "1h"
command: |
set -e
- docker_image_libtorch_android_x86_32=${DOCKER_IMAGE}-${CIRCLE_SHA1}-android-x86_32
+ docker_image_libtorch_android_x86_32=${DOCKER_IMAGE}:${DOCKER_TAG}-${CIRCLE_SHA1}-android-x86_32
echo "docker_image_libtorch_android_x86_32: "${docker_image_libtorch_android_x86_32}
# x86
@@ -1730,6 +1752,7 @@
steps:
# See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
- attach_scripts
+ - load_shared_env
- setup_linux_system_environment
- checkout
- setup_ci_environment
@@ -1739,9 +1762,9 @@
command: |
set -e
# Pull Docker image and run build
- echo "DOCKER_IMAGE: "${DOCKER_IMAGE}
- time docker pull ${DOCKER_IMAGE} >/dev/null
- export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
+ echo "DOCKER_IMAGE: "${DOCKER_IMAGE}:${DOCKER_TAG}
+ time docker pull ${DOCKER_IMAGE}:${DOCKER_TAG} >/dev/null
+ export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE}:${DOCKER_TAG})
echo "Do NOT merge master branch into $CIRCLE_BRANCH in environment $BUILD_ENVIRONMENT"
@@ -1756,7 +1779,7 @@
# Push intermediate Docker image for next phase to use
if [ -z "${BUILD_ONLY}" ]; then
# Augment our output image name with bazel to avoid collisions
- output_image=${DOCKER_IMAGE}-bazel-${CIRCLE_SHA1}
+ output_image=${DOCKER_IMAGE}:${DOCKER_TAG}-bazel-${CIRCLE_SHA1}
export COMMIT_DOCKER_IMAGE=$output_image
docker commit "$id" ${COMMIT_DOCKER_IMAGE}
time docker push ${COMMIT_DOCKER_IMAGE}
@@ -1769,6 +1792,7 @@
steps:
# See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
- attach_scripts
+ - load_shared_env
- setup_linux_system_environment
- setup_ci_environment
- run:
@@ -1776,7 +1800,7 @@
no_output_timeout: "90m"
command: |
set -e
- output_image=${DOCKER_IMAGE}-bazel-${CIRCLE_SHA1}
+ output_image=${DOCKER_IMAGE}:${DOCKER_TAG}-bazel-${CIRCLE_SHA1}
export COMMIT_DOCKER_IMAGE=$output_image
echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
@@ -1809,14 +1833,14 @@
pytorch_doc_test:
environment:
BUILD_ENVIRONMENT: pytorch-doc-test
- # TODO: stop hardcoding this
- DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
resource_class: medium
machine:
image: ubuntu-1604:201903-01
steps:
# See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
- attach_scripts
+ - load_shared_env
- setup_linux_system_environment
- setup_ci_environment
- run:
@@ -1824,7 +1848,7 @@
no_output_timeout: "30m"
command: |
set -ex
- export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+ export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}:${DOCKER_TAG}-${CIRCLE_SHA1}
echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
@@ -1968,9 +1992,48 @@
resource_class: large
environment:
IMAGE_NAME: << parameters.image_name >>
+ # Enable 'docker manifest'
+ DOCKER_CLI_EXPERIMENTAL: "enabled"
+ DOCKER_BUILDKIT: 1
steps:
- checkout
- run:
+ name: Calculate docker tag
+ command: |
+ set -x
+ mkdir .circleci/shared
+ # git keeps a hash of all sub trees
+ echo "export DOCKER_TAG=$(git rev-parse HEAD:.circleci/docker)" >> .circleci/shared/env_file
+ # Saves our calculated docker tag to our workpace for later use
+ - persist_to_workspace:
+ root: .
+ paths:
+ - .circleci/shared/
+ - load_shared_env:
+ root: .
+ - run:
+ name: Check if image should be built
+ command: |
+ set +x
+ export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
+ export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
+ eval $(aws ecr get-login --no-include-email --region us-east-1)
+ set -x
+ PREVIOUS_DOCKER_TAG=$(git rev-parse "$(git merge-base HEAD << pipeline.git.base_revision >>):.circleci/docker")
+ # Check if image already exists, if it does then skip building it
+ if docker manifest inspect "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/${IMAGE_NAME}:${DOCKER_TAG}"; then
+ circleci-agent step halt
+ # circleci-agent step halt doesn't actually halt the step so we need to
+ # explicitly exit the step here ourselves before it causes too much trouble
+ exit 0
+ fi
+ # If no image exists but the hash is the same as the previous hash then we should error out here
+ if [[ ${PREVIOUS_DOCKER_TAG} = ${DOCKER_TAG} ]]; then
+ echo "ERROR: Something has gone wrong and the previous image isn't available for the merge-base of your branch"
+ echo " contact the PyTorch team to restore the original images"
+ exit 1
+ fi
+ - run:
name: build_docker_image_<< parameters.image_name >>
no_output_timeout: "1h"
command: |
@@ -1978,7 +2041,9 @@
export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
set -x
+ source .circleci/shared/env_file
cd .circleci/docker && ./build_docker.sh
+
docker_for_ecr_gc_build_job:
machine:
image: ubuntu-1604:201903-01
@@ -2005,14 +2070,25 @@
type: string
environment:
PROJECT: << parameters.project >>
+ # TODO: Remove legacy image tags once we feel comfortable with new docker image tags
IMAGE_TAG: << parameters.tags_to_keep >>
docker:
- image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/gc/ecr
aws_auth:
aws_access_key_id: ${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
aws_secret_access_key: ${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
-
steps:
+ - checkout
+ - run:
+ # NOTE: see 'docker_build_job' for how these tags actually get built
+ name: dynamically generate tags to keep
+ no_output_timeout: "1h"
+ command: |
+ GENERATED_IMAGE_TAG=$(\
+ git log --oneline --pretty='%H' .circleci/docker \
+ | xargs -I '{}' git rev-parse '{}:.circleci/docker' \
+ | paste -sd "," -)
+ echo "export GENERATED_IMAGE_TAG='${GENERATED_IMAGE_TAG}'" >> ${BASH_ENV}
- run:
name: garbage collecting for ecr images
no_output_timeout: "1h"
@@ -2021,7 +2097,7 @@
export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
set -x
- /usr/bin/gc.py --filter-prefix ${PROJECT} --ignore-tags ${IMAGE_TAG}
+ /usr/bin/gc.py --filter-prefix ${PROJECT} --ignore-tags "${IMAGE_TAG},${GENERATED_IMAGE_TAG}"
docker_hub_index_job:
docker:
@@ -2396,6 +2472,7 @@
name: pytorch_linux_xenial_pynightly_build
requires:
- setup
+ - "docker-pytorch-linux-xenial-pynightly"
filters:
branches:
only:
@@ -2403,7 +2480,7 @@
- /ci-all\/.*/
- /release\/.*/
build_environment: "pytorch-linux-xenial-pynightly-build"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-pynightly:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-pynightly"
- pytorch_linux_test:
name: pytorch_linux_xenial_pynightly_test
requires:
@@ -2416,21 +2493,22 @@
- /ci-all\/.*/
- /release\/.*/
build_environment: "pytorch-linux-xenial-pynightly-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-pynightly:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-pynightly"
resource_class: large
- pytorch_linux_build:
name: pytorch_linux_xenial_py3_6_gcc5_4_build
requires:
- setup
+ - "docker-pytorch-linux-xenial-py3.6-gcc5.4"
build_environment: "pytorch-linux-xenial-py3.6-gcc5.4-build"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
- pytorch_linux_test:
name: pytorch_linux_xenial_py3_6_gcc5_4_test
requires:
- setup
- pytorch_linux_xenial_py3_6_gcc5_4_build
build_environment: "pytorch-linux-xenial-py3.6-gcc5.4-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
resource_class: large
- pytorch_python_doc_push:
requires:
@@ -2447,12 +2525,13 @@
- setup
- pytorch_linux_xenial_py3_6_gcc5_4_build
build_environment: "pytorch-linux-backward-compatibility-check-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
resource_class: large
- pytorch_linux_build:
name: pytorch_paralleltbb_linux_xenial_py3_6_gcc5_4_build
requires:
- setup
+ - "docker-pytorch-linux-xenial-py3.6-gcc5.4"
filters:
branches:
only:
@@ -2460,7 +2539,7 @@
- /ci-all\/.*/
- /release\/.*/
build_environment: "pytorch-paralleltbb-linux-xenial-py3.6-gcc5.4-build"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
- pytorch_linux_test:
name: pytorch_paralleltbb_linux_xenial_py3_6_gcc5_4_test
requires:
@@ -2473,12 +2552,13 @@
- /ci-all\/.*/
- /release\/.*/
build_environment: "pytorch-paralleltbb-linux-xenial-py3.6-gcc5.4-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
resource_class: large
- pytorch_linux_build:
name: pytorch_parallelnative_linux_xenial_py3_6_gcc5_4_build
requires:
- setup
+ - "docker-pytorch-linux-xenial-py3.6-gcc5.4"
filters:
branches:
only:
@@ -2486,7 +2566,7 @@
- /ci-all\/.*/
- /release\/.*/
build_environment: "pytorch-parallelnative-linux-xenial-py3.6-gcc5.4-build"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
- pytorch_linux_test:
name: pytorch_parallelnative_linux_xenial_py3_6_gcc5_4_test
requires:
@@ -2499,12 +2579,13 @@
- /ci-all\/.*/
- /release\/.*/
build_environment: "pytorch-parallelnative-linux-xenial-py3.6-gcc5.4-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
resource_class: large
- pytorch_linux_build:
name: pytorch_linux_xenial_py3_6_gcc7_build
requires:
- setup
+ - "docker-pytorch-linux-xenial-py3.6-gcc7"
filters:
branches:
only:
@@ -2512,7 +2593,7 @@
- /ci-all\/.*/
- /release\/.*/
build_environment: "pytorch-linux-xenial-py3.6-gcc7-build"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc7:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc7"
- pytorch_linux_test:
name: pytorch_linux_xenial_py3_6_gcc7_test
requires:
@@ -2525,26 +2606,28 @@
- /ci-all\/.*/
- /release\/.*/
build_environment: "pytorch-linux-xenial-py3.6-gcc7-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc7:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc7"
resource_class: large
- pytorch_linux_build:
name: pytorch_linux_xenial_py3_clang5_asan_build
requires:
- setup
+ - "docker-pytorch-linux-xenial-py3-clang5-asan"
build_environment: "pytorch-linux-xenial-py3-clang5-asan-build"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-asan:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-asan"
- pytorch_linux_test:
name: pytorch_linux_xenial_py3_clang5_asan_test
requires:
- setup
- pytorch_linux_xenial_py3_clang5_asan_build
build_environment: "pytorch-linux-xenial-py3-clang5-asan-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-asan:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-asan"
resource_class: large
- pytorch_linux_build:
name: pytorch_linux_xenial_cuda9_2_cudnn7_py3_gcc7_build
requires:
- setup
+ - "docker-pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc7"
filters:
branches:
only:
@@ -2552,7 +2635,7 @@
- /ci-all\/.*/
- /release\/.*/
build_environment: "pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc7-build"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc7:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc7"
- pytorch_linux_test:
name: pytorch_linux_xenial_cuda9_2_cudnn7_py3_gcc7_test
requires:
@@ -2565,13 +2648,14 @@
- /ci-all\/.*/
- /release\/.*/
build_environment: "pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc7-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc7:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc7"
use_cuda_docker_runtime: "1"
resource_class: gpu.medium
- pytorch_linux_build:
name: pytorch_linux_xenial_cuda10_1_cudnn7_py3_gcc7_build
requires:
- setup
+ - "docker-pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7"
filters:
branches:
only:
@@ -2579,7 +2663,7 @@
- /ci-all\/.*/
- /release\/.*/
build_environment: "pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7-build"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7"
- pytorch_linux_test:
name: pytorch_linux_xenial_cuda10_1_cudnn7_py3_gcc7_test
requires:
@@ -2592,7 +2676,7 @@
- /ci-all\/.*/
- /release\/.*/
build_environment: "pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7"
use_cuda_docker_runtime: "1"
resource_class: gpu.medium
- pytorch_linux_test:
@@ -2607,7 +2691,7 @@
- /ci-all\/.*/
- /release\/.*/
build_environment: "pytorch-linux-xenial-cuda10.1-cudnn7-py3-multigpu-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7"
use_cuda_docker_runtime: "1"
resource_class: gpu.large
- pytorch_linux_test:
@@ -2622,7 +2706,7 @@
- /ci-all\/.*/
- /release\/.*/
build_environment: "pytorch-linux-xenial-cuda10.1-cudnn7-py3-NO_AVX2-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7"
use_cuda_docker_runtime: "1"
resource_class: gpu.medium
- pytorch_linux_test:
@@ -2637,7 +2721,7 @@
- /ci-all\/.*/
- /release\/.*/
build_environment: "pytorch-linux-xenial-cuda10.1-cudnn7-py3-NO_AVX-NO_AVX2-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7"
use_cuda_docker_runtime: "1"
resource_class: gpu.medium
- pytorch_linux_test:
@@ -2652,7 +2736,7 @@
- /ci-all\/.*/
- /release\/.*/
build_environment: "pytorch-linux-xenial-cuda10.1-cudnn7-py3-slow-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7"
use_cuda_docker_runtime: "1"
resource_class: gpu.medium
- pytorch_linux_test:
@@ -2667,48 +2751,52 @@
- /ci-all\/.*/
- /release\/.*/
build_environment: "pytorch-linux-xenial-cuda10.1-cudnn7-py3-nogpu-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7"
resource_class: large
- pytorch_linux_build:
name: pytorch_linux_xenial_cuda10_2_cudnn7_py3_gcc7_build
requires:
- setup
+ - "docker-pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
build_environment: "pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7-build"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
- pytorch_linux_test:
name: pytorch_linux_xenial_cuda10_2_cudnn7_py3_gcc7_test
requires:
- setup
- pytorch_linux_xenial_cuda10_2_cudnn7_py3_gcc7_build
build_environment: "pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
use_cuda_docker_runtime: "1"
resource_class: gpu.medium
- pytorch_linux_build:
name: pytorch_libtorch_linux_xenial_cuda10_2_cudnn7_py3_gcc7_build
requires:
- setup
+ - "docker-pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
build_environment: "pytorch-libtorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7-build"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
- pytorch_linux_test:
name: pytorch_libtorch_linux_xenial_cuda10_2_cudnn7_py3_gcc7_test
requires:
- setup
- pytorch_libtorch_linux_xenial_cuda10_2_cudnn7_py3_gcc7_build
build_environment: "pytorch-libtorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
use_cuda_docker_runtime: "1"
resource_class: gpu.medium
- pytorch_linux_build:
name: pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_build
requires:
- setup
+ - "docker-pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_32-build"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
- pytorch_linux_build:
name: pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_64_build
requires:
- setup
+ - "docker-pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
filters:
branches:
only:
@@ -2716,11 +2804,12 @@
- /ci-all\/.*/
- /release\/.*/
build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_64-build"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
- pytorch_linux_build:
name: pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v7a_build
requires:
- setup
+ - "docker-pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
filters:
branches:
only:
@@ -2728,11 +2817,12 @@
- /ci-all\/.*/
- /release\/.*/
build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-arm-v7a-build"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
- pytorch_linux_build:
name: pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v8a_build
requires:
- setup
+ - "docker-pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
filters:
branches:
only:
@@ -2740,34 +2830,36 @@
- /ci-all\/.*/
- /release\/.*/
build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-arm-v8a-build"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
- pytorch_linux_build:
name: pytorch_linux_bionic_py3_6_clang9_build
requires:
- setup
+ - "docker-pytorch-linux-bionic-py3.6-clang9"
build_environment: "pytorch-linux-bionic-py3.6-clang9-build"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-py3.6-clang9:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-py3.6-clang9"
- pytorch_linux_test:
name: pytorch_linux_bionic_py3_6_clang9_test
requires:
- setup
- pytorch_linux_bionic_py3_6_clang9_build
build_environment: "pytorch-linux-bionic-py3.6-clang9-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-py3.6-clang9:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-py3.6-clang9"
resource_class: large
- pytorch_linux_build:
name: pytorch_xla_linux_bionic_py3_6_clang9_build
requires:
- setup
+ - "docker-pytorch-linux-bionic-py3.6-clang9"
build_environment: "pytorch-xla-linux-bionic-py3.6-clang9-build"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-py3.6-clang9:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-py3.6-clang9"
- pytorch_linux_test:
name: pytorch_xla_linux_bionic_py3_6_clang9_test
requires:
- setup
- pytorch_xla_linux_bionic_py3_6_clang9_build
build_environment: "pytorch-xla-linux-bionic-py3.6-clang9-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-py3.6-clang9:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-py3.6-clang9"
resource_class: large
# Warning: indentation here matters!
@@ -2834,28 +2926,32 @@
name: pytorch_linux_xenial_py3_clang5_mobile_build
requires:
- setup
+ - docker-pytorch-linux-xenial-py3-clang5-asan
build_environment: "pytorch-linux-xenial-py3-clang5-mobile-build"
build_only: "1"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-asan:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-asan"
- pytorch_linux_build:
name: pytorch_linux_xenial_py3_clang5_mobile_custom_build_static
requires:
- setup
+ - docker-pytorch-linux-xenial-py3-clang5-asan
build_environment: "pytorch-linux-xenial-py3-clang5-mobile-custom-build-static"
build_only: "1"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-asan:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-asan"
- pytorch_linux_build:
name: pytorch_linux_xenial_py3_clang5_mobile_custom_build_dynamic
requires:
- setup
+ - docker-pytorch-linux-xenial-py3-clang5-android-ndk-r19c
build_environment: "pytorch-linux-xenial-py3-clang5-mobile-custom-build-dynamic"
build_only: "1"
# Use LLVM-DEV toolchain in android-ndk-r19c docker image
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
- pytorch_linux_build:
name: pytorch_linux_xenial_py3_clang5_mobile_code_analysis
requires:
- setup
+ - docker-pytorch-linux-xenial-py3-clang5-android-ndk-r19c
# Most of this CI is already covered by "mobile-custom-build-dynamic" job
filters:
branches:
@@ -2866,14 +2962,14 @@
build_environment: "pytorch-linux-xenial-py3-clang5-mobile-code-analysis"
build_only: "1"
# Use LLVM-DEV toolchain in android-ndk-r19c docker image
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
- pytorch_linux_test:
name: pytorch_linux_xenial_py3_6_gcc5_4_ge_config_legacy_test
requires:
- setup
- pytorch_linux_xenial_py3_6_gcc5_4_build
build_environment: "pytorch-linux-xenial-py3.6-gcc5.4-ge_config_legacy-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
resource_class: large
- pytorch_linux_test:
name: pytorch_linux_xenial_py3_6_gcc5_4_ge_config_profiling_test
@@ -2881,7 +2977,7 @@
- setup
- pytorch_linux_xenial_py3_6_gcc5_4_build
build_environment: "pytorch-linux-xenial-py3.6-gcc5.4-ge_config_profiling-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
resource_class: large
- pytorch_linux_test:
name: pytorch_linux_xenial_cuda10_2_cudnn7_py3_ge_config_legacy_test
@@ -2889,7 +2985,7 @@
- setup
- pytorch_linux_xenial_cuda10_2_cudnn7_py3_gcc7_build
build_environment: "pytorch-linux-xenial-cuda10.1-cudnn7-ge_config_legacy-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
use_cuda_docker_runtime: "1"
resource_class: gpu.medium
- pytorch_linux_test:
@@ -2898,15 +2994,16 @@
- setup
- pytorch_linux_xenial_cuda10_2_cudnn7_py3_gcc7_build
build_environment: "pytorch-linux-xenial-cuda10.1-cudnn7-ge_config_profiling-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
use_cuda_docker_runtime: "1"
resource_class: gpu.medium
- pytorch_linux_bazel_build:
name: pytorch_bazel_build
requires:
- setup
+ - docker-pytorch-linux-xenial-py3.6-gcc7
build_environment: "pytorch-linux-xenial-py3.6-gcc7-bazel-build"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc7:f990c76a-a798-42bb-852f-5be5006f8026"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc7"
resource_class: large
- pytorch_linux_bazel_test:
name: pytorch_bazel_test
@@ -2914,7 +3011,7 @@
- setup
- pytorch_bazel_build
build_environment: "pytorch-linux-xenial-py3.6-gcc7-bazel-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc7:f990c76a-a798-42bb-852f-5be5006f8026"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc7"
- caffe2_linux_build:
name: caffe2_onnx_main_py3_6_clang7_ubuntu16_04_build
requires:
@@ -4941,7 +5038,8 @@
build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_32"
requires:
- setup
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ - docker-pytorch-linux-xenial-py3-clang5-android-ndk-r19c
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
filters:
branches:
only: nightly
@@ -4950,7 +5048,8 @@
build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_64"
requires:
- setup
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ - docker-pytorch-linux-xenial-py3-clang5-android-ndk-r19c
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
filters:
branches:
only: nightly
@@ -4959,7 +5058,8 @@
build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-arm-v7a"
requires:
- setup
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ - docker-pytorch-linux-xenial-py3-clang5-android-ndk-r19c
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
filters:
branches:
only: nightly
@@ -4968,7 +5068,8 @@
build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-arm-v8a"
requires:
- setup
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ - docker-pytorch-linux-xenial-py3-clang5-android-ndk-r19c
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
filters:
branches:
only: nightly
@@ -6985,63 +7086,54 @@
branches:
only: postnightly
- docker_build:
- triggers:
- - schedule:
- cron: "0 15 * * 0"
- filters:
- branches:
- only:
- - master
- jobs:
- - docker_for_ecr_gc_build_job
+ # - docker_for_ecr_gc_build_job
- docker_build_job:
- name: "pytorch-linux-bionic-py3.6-clang9"
+ name: "docker-pytorch-linux-bionic-py3.6-clang9"
image_name: "pytorch-linux-bionic-py3.6-clang9"
- docker_build_job:
- name: "pytorch-linux-xenial-cuda10-cudnn7-py3-gcc7"
+ name: "docker-pytorch-linux-xenial-cuda10-cudnn7-py3-gcc7"
image_name: "pytorch-linux-xenial-cuda10-cudnn7-py3-gcc7"
- docker_build_job:
- name: "pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7"
+ name: "docker-pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7"
image_name: "pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7"
- docker_build_job:
- name: "pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
+ name: "docker-pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
image_name: "pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
- docker_build_job:
- name: "pytorch-linux-xenial-cuda9-cudnn7-py3"
+ name: "docker-pytorch-linux-xenial-cuda9-cudnn7-py3"
image_name: "pytorch-linux-xenial-cuda9-cudnn7-py3"
- docker_build_job:
- name: "pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc5.4"
+ name: "docker-pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc5.4"
image_name: "pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc5.4"
- docker_build_job:
- name: "pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc7"
+ name: "docker-pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc7"
image_name: "pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc7"
- docker_build_job:
- name: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
+ name: "docker-pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
image_name: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
- docker_build_job:
- name: "pytorch-linux-xenial-py3-clang5-asan"
+ name: "docker-pytorch-linux-xenial-py3-clang5-asan"
image_name: "pytorch-linux-xenial-py3-clang5-asan"
- docker_build_job:
- name: "pytorch-linux-xenial-py3.8"
+ name: "docker-pytorch-linux-xenial-py3.8"
image_name: "pytorch-linux-xenial-py3.8"
- docker_build_job:
- name: "pytorch-linux-xenial-py3.6-clang7"
+ name: "docker-pytorch-linux-xenial-py3.6-clang7"
image_name: "pytorch-linux-xenial-py3.6-clang7"
- docker_build_job:
- name: "pytorch-linux-xenial-py3.6-gcc4.8"
+ name: "docker-pytorch-linux-xenial-py3.6-gcc4.8"
image_name: "pytorch-linux-xenial-py3.6-gcc4.8"
- docker_build_job:
- name: "pytorch-linux-xenial-py3.6-gcc5.4"
+ name: "docker-pytorch-linux-xenial-py3.6-gcc5.4"
image_name: "pytorch-linux-xenial-py3.6-gcc5.4"
- docker_build_job:
- name: "pytorch-linux-xenial-py3.6-gcc7.2"
+ name: "docker-pytorch-linux-xenial-py3.6-gcc7.2"
image_name: "pytorch-linux-xenial-py3.6-gcc7.2"
- docker_build_job:
- name: "pytorch-linux-xenial-py3.6-gcc7"
+ name: "docker-pytorch-linux-xenial-py3.6-gcc7"
image_name: "pytorch-linux-xenial-py3.6-gcc7"
- docker_build_job:
- name: "pytorch-linux-xenial-pynightly"
+ name: "docker-pytorch-linux-xenial-pynightly"
image_name: "pytorch-linux-xenial-pynightly"
ecr_gc:
triggers:
diff --git a/.circleci/docker/build.sh b/.circleci/docker/build.sh
index 6dc249f..d6fddfa 100755
--- a/.circleci/docker/build.sh
+++ b/.circleci/docker/build.sh
@@ -188,6 +188,7 @@
# it's no longer needed.
docker build \
--no-cache \
+ --progress=plain \
--build-arg "TRAVIS_DL_URL_PREFIX=${TRAVIS_DL_URL_PREFIX}" \
--build-arg "BUILD_ENVIRONMENT=${image}" \
--build-arg "PROTOBUF=${PROTOBUF:-}" \
diff --git a/.circleci/docker/build_docker.sh b/.circleci/docker/build_docker.sh
index eeb4e70..1e86950 100755
--- a/.circleci/docker/build_docker.sh
+++ b/.circleci/docker/build_docker.sh
@@ -6,14 +6,8 @@
$* || (sleep 1 && $*) || (sleep 2 && $*)
}
-# If UPSTREAM_BUILD_ID is set (see trigger job), then we can
-# use it to tag this build with the same ID used to tag all other
-# base image builds. Also, we can try and pull the previous
-# image first, to avoid rebuilding layers that haven't changed.
-
-#until we find a way to reliably reuse previous build, this last_tag is not in use
-# last_tag="$(( CIRCLE_BUILD_NUM - 1 ))"
-tag="${CIRCLE_WORKFLOW_ID}"
+# Setup from previous step, typically located in .circleci/artifacts/env_file
+tag="${DOCKER_TAG}"
registry="308535385114.dkr.ecr.us-east-1.amazonaws.com"
diff --git a/.circleci/ecr_gc_docker/Dockerfile b/.circleci/ecr_gc_docker/Dockerfile
index d0198ac..36347d5 100644
--- a/.circleci/ecr_gc_docker/Dockerfile
+++ b/.circleci/ecr_gc_docker/Dockerfile
@@ -1,6 +1,6 @@
FROM ubuntu:16.04
-RUN apt-get update && apt-get install -y python-pip && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log
+RUN apt-get update && apt-get install -y git python-pip && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log
ADD requirements.txt /requirements.txt
diff --git a/.circleci/validate-docker-version.py b/.circleci/validate-docker-version.py
deleted file mode 100755
index 9410b88..0000000
--- a/.circleci/validate-docker-version.py
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/usr/bin/env python3
-import cimodel.data.caffe2_build_definitions as caffe2_build_definitions
-import cimodel.data.pytorch_build_definitions as pytorch_build_definitions
-from yaml import load
-
-try:
- from yaml import CLoader as Loader
-except ImportError:
- from yaml import Loader
-
-
-def load_config(filename=".circleci/config.yml"):
- with open(filename, "r") as fh:
- return load("".join(fh.readlines()), Loader)
-
-
-def load_tags_for_projects(workflow_config):
- return {
- v["ecr_gc_job"]["project"]: v["ecr_gc_job"]["tags_to_keep"]
- for v in workflow_config["workflows"]["ecr_gc"]["jobs"]
- if isinstance(v, dict) and "ecr_gc_job" in v
- }
-
-
-def check_version(job, tags, expected_version):
- valid_versions = tags[job].split(",")
- if expected_version not in valid_versions:
- raise RuntimeError(
- "We configured {} to use Docker version {}; but this "
- "version is not configured in job ecr_gc_job_for_{}. Non-deployed versions will be "
- "garbage collected two weeks after they are created. DO NOT LAND "
- "THIS TO MASTER without also updating ossci-job-dsl with this version."
- "\n\nDeployed versions: {}".format(job, expected_version, job, tags[job])
- )
-
-
-def validate_docker_version():
- tags = load_tags_for_projects(load_config())
- check_version("pytorch", tags, pytorch_build_definitions.DOCKER_IMAGE_VERSION)
- check_version("caffe2", tags, caffe2_build_definitions.DOCKER_IMAGE_VERSION)
-
-
-if __name__ == "__main__":
- validate_docker_version()
diff --git a/.circleci/verbatim-sources/commands.yml b/.circleci/verbatim-sources/commands.yml
index a4ec6d8..683a0b7 100644
--- a/.circleci/verbatim-sources/commands.yml
+++ b/.circleci/verbatim-sources/commands.yml
@@ -10,6 +10,21 @@
name: Attaching workspace
at: ~/workspace
+ # Must be run after attaching workspace from previous steps
+ load_shared_env:
+ description: "Loads .circleci/shared/env_file into ${BASH_ENV}"
+ parameters:
+ # For some weird reason we decide to reattach our workspace to ~/workspace so
+ # in the vein of making it simple let's assume our share env_file is here
+ root:
+ type: string
+ default: "~/workspace"
+ steps:
+ - run:
+ name: "Load .circleci/shared/env_file into ${BASH_ENV}"
+ command: |
+ cat << parameters.root >>/.circleci/shared/env_file >> ${BASH_ENV}
+
# This system setup script is meant to run before the CI-related scripts, e.g.,
# installing Git client, checking out code, setting up CI env, and
# building/testing.
diff --git a/.circleci/verbatim-sources/docker_jobs.yml b/.circleci/verbatim-sources/docker_jobs.yml
index 6ce3440..3918cc0 100644
--- a/.circleci/verbatim-sources/docker_jobs.yml
+++ b/.circleci/verbatim-sources/docker_jobs.yml
@@ -8,9 +8,48 @@
resource_class: large
environment:
IMAGE_NAME: << parameters.image_name >>
+ # Enable 'docker manifest'
+ DOCKER_CLI_EXPERIMENTAL: "enabled"
+ DOCKER_BUILDKIT: 1
steps:
- checkout
- run:
+ name: Calculate docker tag
+ command: |
+ set -x
+ mkdir .circleci/shared
+ # git keeps a hash of all sub trees
+ echo "export DOCKER_TAG=$(git rev-parse HEAD:.circleci/docker)" >> .circleci/shared/env_file
+ # Saves our calculated docker tag to our workpace for later use
+ - persist_to_workspace:
+ root: .
+ paths:
+ - .circleci/shared/
+ - load_shared_env:
+ root: .
+ - run:
+ name: Check if image should be built
+ command: |
+ set +x
+ export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
+ export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
+ eval $(aws ecr get-login --no-include-email --region us-east-1)
+ set -x
+ PREVIOUS_DOCKER_TAG=$(git rev-parse "$(git merge-base HEAD << pipeline.git.base_revision >>):.circleci/docker")
+ # Check if image already exists, if it does then skip building it
+ if docker manifest inspect "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/${IMAGE_NAME}:${DOCKER_TAG}"; then
+ circleci-agent step halt
+ # circleci-agent step halt doesn't actually halt the step so we need to
+ # explicitly exit the step here ourselves before it causes too much trouble
+ exit 0
+ fi
+ # If no image exists but the hash is the same as the previous hash then we should error out here
+ if [[ ${PREVIOUS_DOCKER_TAG} = ${DOCKER_TAG} ]]; then
+ echo "ERROR: Something has gone wrong and the previous image isn't available for the merge-base of your branch"
+ echo " contact the PyTorch team to restore the original images"
+ exit 1
+ fi
+ - run:
name: build_docker_image_<< parameters.image_name >>
no_output_timeout: "1h"
command: |
@@ -18,7 +57,9 @@
export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
set -x
+ source .circleci/shared/env_file
cd .circleci/docker && ./build_docker.sh
+
docker_for_ecr_gc_build_job:
machine:
image: ubuntu-1604:201903-01
@@ -45,14 +86,25 @@
type: string
environment:
PROJECT: << parameters.project >>
+ # TODO: Remove legacy image tags once we feel comfortable with new docker image tags
IMAGE_TAG: << parameters.tags_to_keep >>
docker:
- image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/gc/ecr
aws_auth:
aws_access_key_id: ${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
aws_secret_access_key: ${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
-
steps:
+ - checkout
+ - run:
+ # NOTE: see 'docker_build_job' for how these tags actually get built
+ name: dynamically generate tags to keep
+ no_output_timeout: "1h"
+ command: |
+ GENERATED_IMAGE_TAG=$(\
+ git log --oneline --pretty='%H' .circleci/docker \
+ | xargs -I '{}' git rev-parse '{}:.circleci/docker' \
+ | paste -sd "," -)
+ echo "export GENERATED_IMAGE_TAG='${GENERATED_IMAGE_TAG}'" >> ${BASH_ENV}
- run:
name: garbage collecting for ecr images
no_output_timeout: "1h"
@@ -61,7 +113,7 @@
export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
set -x
- /usr/bin/gc.py --filter-prefix ${PROJECT} --ignore-tags ${IMAGE_TAG}
+ /usr/bin/gc.py --filter-prefix ${PROJECT} --ignore-tags "${IMAGE_TAG},${GENERATED_IMAGE_TAG}"
docker_hub_index_job:
docker:
diff --git a/.circleci/verbatim-sources/job-specs-custom.yml b/.circleci/verbatim-sources/job-specs-custom.yml
index aa22f22..2af23eb 100644
--- a/.circleci/verbatim-sources/job-specs-custom.yml
+++ b/.circleci/verbatim-sources/job-specs-custom.yml
@@ -2,13 +2,14 @@
environment:
BUILD_ENVIRONMENT: pytorch-python-doc-push
# TODO: stop hardcoding this
- DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
resource_class: large
machine:
image: ubuntu-1604:201903-01
steps:
# See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
- attach_scripts
+ - load_shared_env
- setup_linux_system_environment
- setup_ci_environment
- run:
@@ -16,7 +17,7 @@
no_output_timeout: "1h"
command: |
set -ex
- export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+ export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}:${DOCKER_TAG}-${CIRCLE_SHA1}
echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
@@ -53,13 +54,14 @@
pytorch_cpp_doc_push:
environment:
BUILD_ENVIRONMENT: pytorch-cpp-doc-push
- DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
resource_class: large
machine:
image: ubuntu-1604:201903-01
steps:
# See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
- attach_scripts
+ - load_shared_env
- setup_linux_system_environment
- setup_ci_environment
- run:
@@ -67,7 +69,7 @@
no_output_timeout: "1h"
command: |
set -ex
- export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+ export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}:${DOCKER_TAG}-${CIRCLE_SHA1}
echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
@@ -163,13 +165,14 @@
pytorch_android_gradle_build:
environment:
BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build
- DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
PYTHON_VERSION: "3.6"
resource_class: large
machine:
image: ubuntu-1604:201903-01
steps:
- attach_scripts
+ - load_shared_env
- setup_linux_system_environment
- checkout
- setup_ci_environment
@@ -178,7 +181,7 @@
no_output_timeout: "1h"
command: |
set -eux
- docker_image_commit=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+ docker_image_commit=${DOCKER_IMAGE}:${DOCKER_TAG}-${CIRCLE_SHA1}
docker_image_libtorch_android_x86_32=${docker_image_commit}-android-x86_32
docker_image_libtorch_android_x86_64=${docker_image_commit}-android-x86_64
@@ -249,13 +252,14 @@
pytorch_android_publish_snapshot:
environment:
BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-publish-snapshot
- DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
PYTHON_VERSION: "3.6"
resource_class: large
machine:
image: ubuntu-1604:201903-01
steps:
- attach_scripts
+ - load_shared_env
- setup_linux_system_environment
- checkout
- setup_ci_environment
@@ -264,7 +268,7 @@
no_output_timeout: "1h"
command: |
set -eux
- docker_image_commit=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+ docker_image_commit=${DOCKER_IMAGE}:${DOCKER_TAG}-${CIRCLE_SHA1}
docker_image_libtorch_android_x86_32_gradle=${docker_image_commit}-android-x86_32-gradle
@@ -285,13 +289,14 @@
pytorch_android_gradle_build-x86_32:
environment:
BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build-only-x86_32
- DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
PYTHON_VERSION: "3.6"
resource_class: large
machine:
image: ubuntu-1604:201903-01
steps:
- attach_scripts
+ - load_shared_env
- run:
name: filter out not PR runs
no_output_timeout: "5m"
@@ -308,7 +313,7 @@
no_output_timeout: "1h"
command: |
set -e
- docker_image_libtorch_android_x86_32=${DOCKER_IMAGE}-${CIRCLE_SHA1}-android-x86_32
+ docker_image_libtorch_android_x86_32=${DOCKER_IMAGE}:${DOCKER_TAG}-${CIRCLE_SHA1}-android-x86_32
echo "docker_image_libtorch_android_x86_32: "${docker_image_libtorch_android_x86_32}
# x86
@@ -453,6 +458,7 @@
steps:
# See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
- attach_scripts
+ - load_shared_env
- setup_linux_system_environment
- checkout
- setup_ci_environment
@@ -462,9 +468,9 @@
command: |
set -e
# Pull Docker image and run build
- echo "DOCKER_IMAGE: "${DOCKER_IMAGE}
- time docker pull ${DOCKER_IMAGE} >/dev/null
- export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
+ echo "DOCKER_IMAGE: "${DOCKER_IMAGE}:${DOCKER_TAG}
+ time docker pull ${DOCKER_IMAGE}:${DOCKER_TAG} >/dev/null
+ export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE}:${DOCKER_TAG})
echo "Do NOT merge master branch into $CIRCLE_BRANCH in environment $BUILD_ENVIRONMENT"
@@ -479,7 +485,7 @@
# Push intermediate Docker image for next phase to use
if [ -z "${BUILD_ONLY}" ]; then
# Augment our output image name with bazel to avoid collisions
- output_image=${DOCKER_IMAGE}-bazel-${CIRCLE_SHA1}
+ output_image=${DOCKER_IMAGE}:${DOCKER_TAG}-bazel-${CIRCLE_SHA1}
export COMMIT_DOCKER_IMAGE=$output_image
docker commit "$id" ${COMMIT_DOCKER_IMAGE}
time docker push ${COMMIT_DOCKER_IMAGE}
@@ -492,6 +498,7 @@
steps:
# See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
- attach_scripts
+ - load_shared_env
- setup_linux_system_environment
- setup_ci_environment
- run:
@@ -499,7 +506,7 @@
no_output_timeout: "90m"
command: |
set -e
- output_image=${DOCKER_IMAGE}-bazel-${CIRCLE_SHA1}
+ output_image=${DOCKER_IMAGE}:${DOCKER_TAG}-bazel-${CIRCLE_SHA1}
export COMMIT_DOCKER_IMAGE=$output_image
echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
@@ -532,14 +539,14 @@
pytorch_doc_test:
environment:
BUILD_ENVIRONMENT: pytorch-doc-test
- # TODO: stop hardcoding this
- DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
resource_class: medium
machine:
image: ubuntu-1604:201903-01
steps:
# See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
- attach_scripts
+ - load_shared_env
- setup_linux_system_environment
- setup_ci_environment
- run:
@@ -547,7 +554,7 @@
no_output_timeout: "30m"
command: |
set -ex
- export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+ export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}:${DOCKER_TAG}-${CIRCLE_SHA1}
echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
diff --git a/.circleci/verbatim-sources/pytorch-job-specs.yml b/.circleci/verbatim-sources/pytorch-job-specs.yml
index b2dd4ee..2c417c1 100644
--- a/.circleci/verbatim-sources/pytorch-job-specs.yml
+++ b/.circleci/verbatim-sources/pytorch-job-specs.yml
@@ -6,6 +6,7 @@
steps:
# See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
- attach_scripts
+ - load_shared_env
- setup_linux_system_environment
- checkout
- optional_merge_target_branch
@@ -16,9 +17,9 @@
command: |
set -e
# Pull Docker image and run build
- echo "DOCKER_IMAGE: "${DOCKER_IMAGE}
- time docker pull ${DOCKER_IMAGE} >/dev/null
- export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
+ echo "DOCKER_IMAGE: "${DOCKER_IMAGE}:${DOCKER_TAG}
+ time docker pull ${DOCKER_IMAGE}:${DOCKER_TAG} >/dev/null
+ export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE}:${DOCKER_TAG})
git submodule sync && git submodule update -q --init --recursive
@@ -41,7 +42,7 @@
# The xla build uses the same docker image as
# pytorch-linux-trusty-py3.6-gcc5.4-build. In the push step, we have to
# distinguish between them so the test can pick up the correct image.
- output_image=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+ output_image=${DOCKER_IMAGE}:${DOCKER_TAG}-${CIRCLE_SHA1}
if [[ ${BUILD_ENVIRONMENT} == *"xla"* ]]; then
export COMMIT_DOCKER_IMAGE=$output_image-xla
elif [[ ${BUILD_ENVIRONMENT} == *"libtorch"* ]]; then
@@ -68,6 +69,7 @@
steps:
# See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
- attach_scripts
+ - load_shared_env
- setup_linux_system_environment
- setup_ci_environment
- run:
@@ -76,7 +78,7 @@
command: |
set -e
# See Note [Special build images]
- output_image=${DOCKER_IMAGE}-${CIRCLE_SHA1}
+ output_image=${DOCKER_IMAGE}:${DOCKER_TAG}-${CIRCLE_SHA1}
if [[ ${BUILD_ENVIRONMENT} == *"xla"* ]]; then
export COMMIT_DOCKER_IMAGE=$output_image-xla
elif [[ ${BUILD_ENVIRONMENT} == *"libtorch"* ]]; then
diff --git a/.circleci/verbatim-sources/workflows-docker-builder.yml b/.circleci/verbatim-sources/workflows-docker-builder.yml
index 3ef03d5..db6205c 100644
--- a/.circleci/verbatim-sources/workflows-docker-builder.yml
+++ b/.circleci/verbatim-sources/workflows-docker-builder.yml
@@ -1,58 +1,49 @@
- docker_build:
- triggers:
- - schedule:
- cron: "0 15 * * 0"
- filters:
- branches:
- only:
- - master
- jobs:
- - docker_for_ecr_gc_build_job
+ # - docker_for_ecr_gc_build_job
- docker_build_job:
- name: "pytorch-linux-bionic-py3.6-clang9"
+ name: "docker-pytorch-linux-bionic-py3.6-clang9"
image_name: "pytorch-linux-bionic-py3.6-clang9"
- docker_build_job:
- name: "pytorch-linux-xenial-cuda10-cudnn7-py3-gcc7"
+ name: "docker-pytorch-linux-xenial-cuda10-cudnn7-py3-gcc7"
image_name: "pytorch-linux-xenial-cuda10-cudnn7-py3-gcc7"
- docker_build_job:
- name: "pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7"
+ name: "docker-pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7"
image_name: "pytorch-linux-xenial-cuda10.1-cudnn7-py3-gcc7"
- docker_build_job:
- name: "pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
+ name: "docker-pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
image_name: "pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
- docker_build_job:
- name: "pytorch-linux-xenial-cuda9-cudnn7-py3"
+ name: "docker-pytorch-linux-xenial-cuda9-cudnn7-py3"
image_name: "pytorch-linux-xenial-cuda9-cudnn7-py3"
- docker_build_job:
- name: "pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc5.4"
+ name: "docker-pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc5.4"
image_name: "pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc5.4"
- docker_build_job:
- name: "pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc7"
+ name: "docker-pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc7"
image_name: "pytorch-linux-xenial-cuda9.2-cudnn7-py3-gcc7"
- docker_build_job:
- name: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
+ name: "docker-pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
image_name: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
- docker_build_job:
- name: "pytorch-linux-xenial-py3-clang5-asan"
+ name: "docker-pytorch-linux-xenial-py3-clang5-asan"
image_name: "pytorch-linux-xenial-py3-clang5-asan"
- docker_build_job:
- name: "pytorch-linux-xenial-py3.8"
+ name: "docker-pytorch-linux-xenial-py3.8"
image_name: "pytorch-linux-xenial-py3.8"
- docker_build_job:
- name: "pytorch-linux-xenial-py3.6-clang7"
+ name: "docker-pytorch-linux-xenial-py3.6-clang7"
image_name: "pytorch-linux-xenial-py3.6-clang7"
- docker_build_job:
- name: "pytorch-linux-xenial-py3.6-gcc4.8"
+ name: "docker-pytorch-linux-xenial-py3.6-gcc4.8"
image_name: "pytorch-linux-xenial-py3.6-gcc4.8"
- docker_build_job:
- name: "pytorch-linux-xenial-py3.6-gcc5.4"
+ name: "docker-pytorch-linux-xenial-py3.6-gcc5.4"
image_name: "pytorch-linux-xenial-py3.6-gcc5.4"
- docker_build_job:
- name: "pytorch-linux-xenial-py3.6-gcc7.2"
+ name: "docker-pytorch-linux-xenial-py3.6-gcc7.2"
image_name: "pytorch-linux-xenial-py3.6-gcc7.2"
- docker_build_job:
- name: "pytorch-linux-xenial-py3.6-gcc7"
+ name: "docker-pytorch-linux-xenial-py3.6-gcc7"
image_name: "pytorch-linux-xenial-py3.6-gcc7"
- docker_build_job:
- name: "pytorch-linux-xenial-pynightly"
+ name: "docker-pytorch-linux-xenial-pynightly"
image_name: "pytorch-linux-xenial-pynightly"
diff --git a/.circleci/verbatim-sources/workflows-nightly-android-binary-builds.yml b/.circleci/verbatim-sources/workflows-nightly-android-binary-builds.yml
index 3b92f53..2c0cb6d 100644
--- a/.circleci/verbatim-sources/workflows-nightly-android-binary-builds.yml
+++ b/.circleci/verbatim-sources/workflows-nightly-android-binary-builds.yml
@@ -3,7 +3,8 @@
build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_32"
requires:
- setup
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ - docker-pytorch-linux-xenial-py3-clang5-android-ndk-r19c
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
filters:
branches:
only: nightly
@@ -12,7 +13,8 @@
build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_64"
requires:
- setup
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ - docker-pytorch-linux-xenial-py3-clang5-android-ndk-r19c
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
filters:
branches:
only: nightly
@@ -21,7 +23,8 @@
build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-arm-v7a"
requires:
- setup
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ - docker-pytorch-linux-xenial-py3-clang5-android-ndk-r19c
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
filters:
branches:
only: nightly
@@ -30,7 +33,8 @@
build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-arm-v8a"
requires:
- setup
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ - docker-pytorch-linux-xenial-py3-clang5-android-ndk-r19c
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
filters:
branches:
only: nightly
diff --git a/.circleci/verbatim-sources/workflows-pytorch-bazel-builds.yml b/.circleci/verbatim-sources/workflows-pytorch-bazel-builds.yml
index 825ea50..1f6ed3e 100644
--- a/.circleci/verbatim-sources/workflows-pytorch-bazel-builds.yml
+++ b/.circleci/verbatim-sources/workflows-pytorch-bazel-builds.yml
@@ -2,8 +2,9 @@
name: pytorch_bazel_build
requires:
- setup
+ - docker-pytorch-linux-xenial-py3.6-gcc7
build_environment: "pytorch-linux-xenial-py3.6-gcc7-bazel-build"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc7:f990c76a-a798-42bb-852f-5be5006f8026"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc7"
resource_class: large
- pytorch_linux_bazel_test:
name: pytorch_bazel_test
@@ -11,4 +12,4 @@
- setup
- pytorch_bazel_build
build_environment: "pytorch-linux-xenial-py3.6-gcc7-bazel-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc7:f990c76a-a798-42bb-852f-5be5006f8026"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc7"
diff --git a/.circleci/verbatim-sources/workflows-pytorch-ge-config-tests.yml b/.circleci/verbatim-sources/workflows-pytorch-ge-config-tests.yml
index 5800490..767e201 100644
--- a/.circleci/verbatim-sources/workflows-pytorch-ge-config-tests.yml
+++ b/.circleci/verbatim-sources/workflows-pytorch-ge-config-tests.yml
@@ -4,7 +4,7 @@
- setup
- pytorch_linux_xenial_py3_6_gcc5_4_build
build_environment: "pytorch-linux-xenial-py3.6-gcc5.4-ge_config_legacy-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
resource_class: large
- pytorch_linux_test:
name: pytorch_linux_xenial_py3_6_gcc5_4_ge_config_profiling_test
@@ -12,7 +12,7 @@
- setup
- pytorch_linux_xenial_py3_6_gcc5_4_build
build_environment: "pytorch-linux-xenial-py3.6-gcc5.4-ge_config_profiling-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
resource_class: large
- pytorch_linux_test:
name: pytorch_linux_xenial_cuda10_2_cudnn7_py3_ge_config_legacy_test
@@ -20,7 +20,7 @@
- setup
- pytorch_linux_xenial_cuda10_2_cudnn7_py3_gcc7_build
build_environment: "pytorch-linux-xenial-cuda10.1-cudnn7-ge_config_legacy-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
use_cuda_docker_runtime: "1"
resource_class: gpu.medium
- pytorch_linux_test:
@@ -29,6 +29,6 @@
- setup
- pytorch_linux_xenial_cuda10_2_cudnn7_py3_gcc7_build
build_environment: "pytorch-linux-xenial-cuda10.1-cudnn7-ge_config_profiling-test"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
use_cuda_docker_runtime: "1"
resource_class: gpu.medium
diff --git a/.circleci/verbatim-sources/workflows-pytorch-mobile-builds.yml b/.circleci/verbatim-sources/workflows-pytorch-mobile-builds.yml
index 2a91b46..6afa28c 100644
--- a/.circleci/verbatim-sources/workflows-pytorch-mobile-builds.yml
+++ b/.circleci/verbatim-sources/workflows-pytorch-mobile-builds.yml
@@ -3,28 +3,32 @@
name: pytorch_linux_xenial_py3_clang5_mobile_build
requires:
- setup
+ - docker-pytorch-linux-xenial-py3-clang5-asan
build_environment: "pytorch-linux-xenial-py3-clang5-mobile-build"
build_only: "1"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-asan:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-asan"
- pytorch_linux_build:
name: pytorch_linux_xenial_py3_clang5_mobile_custom_build_static
requires:
- setup
+ - docker-pytorch-linux-xenial-py3-clang5-asan
build_environment: "pytorch-linux-xenial-py3-clang5-mobile-custom-build-static"
build_only: "1"
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-asan:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-asan"
- pytorch_linux_build:
name: pytorch_linux_xenial_py3_clang5_mobile_custom_build_dynamic
requires:
- setup
+ - docker-pytorch-linux-xenial-py3-clang5-android-ndk-r19c
build_environment: "pytorch-linux-xenial-py3-clang5-mobile-custom-build-dynamic"
build_only: "1"
# Use LLVM-DEV toolchain in android-ndk-r19c docker image
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
- pytorch_linux_build:
name: pytorch_linux_xenial_py3_clang5_mobile_code_analysis
requires:
- setup
+ - docker-pytorch-linux-xenial-py3-clang5-android-ndk-r19c
# Most of this CI is already covered by "mobile-custom-build-dynamic" job
filters:
branches:
@@ -35,4 +39,4 @@
build_environment: "pytorch-linux-xenial-py3-clang5-mobile-code-analysis"
build_only: "1"
# Use LLVM-DEV toolchain in android-ndk-r19c docker image
- docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index f34efcb..9e546e7 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -21,10 +21,6 @@
run: |
pip install -r requirements.txt
cd .circleci && ./ensure-consistency.py
- - name: Ensure Docker version is correctly deployed
- run: |
- pip install pyyaml
- .circleci/validate-docker-version.py
- name: Shellcheck Jenkins scripts
run: |
sudo apt-get install -y shellcheck
