.circleci: Only generate docker configs we need (#65728)
Summary:
Pull Request resolved: https://github.com/pytorch/pytorch/pull/65728
Changes the docker image generation script to only include image build
jobs for images that we actually use within CircleCI
Signed-off-by: Eli Uriegas <eliuriegas@fb.com>
cc ezyang seemethere malfet pytorch/pytorch-dev-infra
Test Plan: Imported from OSS
Reviewed By: janeyx99
Differential Revision: D31224674
Pulled By: seemethere
fbshipit-source-id: 64b14e1a4ef82d345ec7b898c4c89d9a9419e4de
diff --git a/.circleci/cimodel/data/simple/docker_definitions.py b/.circleci/cimodel/data/simple/docker_definitions.py
index fd5ac66..2091770 100644
--- a/.circleci/cimodel/data/simple/docker_definitions.py
+++ b/.circleci/cimodel/data/simple/docker_definitions.py
@@ -6,22 +6,6 @@
# TODO: make this generated from a matrix rather than just a static list
IMAGE_NAMES = [
- "pytorch-linux-bionic-cuda10.2-cudnn7-py3.9-gcc7",
- "pytorch-linux-bionic-py3.6-clang9",
- "pytorch-linux-bionic-cuda10.2-cudnn7-py3.6-clang9",
- "pytorch-linux-bionic-py3.8-gcc9",
- "pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7",
- "pytorch-linux-xenial-cuda11.1-cudnn8-py3-gcc7",
- "pytorch-linux-xenial-cuda11.3-cudnn8-py3-gcc7",
- "pytorch-linux-xenial-py3-clang5-android-ndk-r19c",
- "pytorch-linux-xenial-py3-clang5-asan",
- "pytorch-linux-xenial-py3-clang7-asan",
- "pytorch-linux-xenial-py3-clang7-onnx",
- "pytorch-linux-xenial-py3.8",
- "pytorch-linux-xenial-py3.6-clang7",
- "pytorch-linux-xenial-py3.6-gcc5.4", # this one is used in doc builds
- "pytorch-linux-xenial-py3.6-gcc7.2",
- "pytorch-linux-xenial-py3.6-gcc7",
"pytorch-linux-bionic-rocm4.1-py3.6",
"pytorch-linux-bionic-rocm4.2-py3.6",
"pytorch-linux-bionic-rocm4.3.1-py3.6",
@@ -32,10 +16,12 @@
# pytorch_build_data.py
SLOW_GRADCHECK_IMAGE_NAME = "pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
-def get_workflow_jobs(only_slow_gradcheck=False):
+def get_workflow_jobs(images=IMAGE_NAMES, only_slow_gradcheck=False):
"""Generates a list of docker image build definitions"""
ret = []
- for image_name in IMAGE_NAMES:
+ for image_name in images:
+ if image_name.startswith('docker-'):
+ image_name = image_name.lstrip('docker-')
if only_slow_gradcheck and image_name is not SLOW_GRADCHECK_IMAGE_NAME:
continue
diff --git a/.circleci/config.yml b/.circleci/config.yml
index d3f45dd..54b8085 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -6874,59 +6874,6 @@
build:
jobs:
- docker_build_job:
- name: "docker-pytorch-linux-bionic-cuda10.2-cudnn7-py3.9-gcc7"
- image_name: "pytorch-linux-bionic-cuda10.2-cudnn7-py3.9-gcc7"
- - docker_build_job:
- name: "docker-pytorch-linux-bionic-py3.6-clang9"
- image_name: "pytorch-linux-bionic-py3.6-clang9"
- - docker_build_job:
- name: "docker-pytorch-linux-bionic-cuda10.2-cudnn7-py3.6-clang9"
- image_name: "pytorch-linux-bionic-cuda10.2-cudnn7-py3.6-clang9"
- - docker_build_job:
- name: "docker-pytorch-linux-bionic-py3.8-gcc9"
- image_name: "pytorch-linux-bionic-py3.8-gcc9"
- - docker_build_job:
- name: "docker-pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
- image_name: "pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
- - docker_build_job:
- name: "docker-pytorch-linux-xenial-cuda11.1-cudnn8-py3-gcc7"
- image_name: "pytorch-linux-xenial-cuda11.1-cudnn8-py3-gcc7"
- - docker_build_job:
- name: "docker-pytorch-linux-xenial-cuda11.3-cudnn8-py3-gcc7"
- image_name: "pytorch-linux-xenial-cuda11.3-cudnn8-py3-gcc7"
- - docker_build_job:
- name: "docker-pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
- image_name: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
- - docker_build_job:
- name: "docker-pytorch-linux-xenial-py3-clang5-asan"
- image_name: "pytorch-linux-xenial-py3-clang5-asan"
- - docker_build_job:
- name: "docker-pytorch-linux-xenial-py3-clang7-asan"
- image_name: "pytorch-linux-xenial-py3-clang7-asan"
- - docker_build_job:
- name: "docker-pytorch-linux-xenial-py3-clang7-onnx"
- image_name: "pytorch-linux-xenial-py3-clang7-onnx"
- - docker_build_job:
- name: "docker-pytorch-linux-xenial-py3.8"
- image_name: "pytorch-linux-xenial-py3.8"
- - docker_build_job:
- name: "docker-pytorch-linux-xenial-py3.6-clang7"
- image_name: "pytorch-linux-xenial-py3.6-clang7"
- - docker_build_job:
- name: "docker-pytorch-linux-xenial-py3.6-gcc5.4"
- image_name: "pytorch-linux-xenial-py3.6-gcc5.4"
- filters:
- branches:
- only: /.*/
- tags:
- only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
- - docker_build_job:
- name: "docker-pytorch-linux-xenial-py3.6-gcc7.2"
- image_name: "pytorch-linux-xenial-py3.6-gcc7.2"
- - docker_build_job:
- name: "docker-pytorch-linux-xenial-py3.6-gcc7"
- image_name: "pytorch-linux-xenial-py3.6-gcc7"
- - docker_build_job:
name: "docker-pytorch-linux-bionic-rocm4.1-py3.6"
image_name: "pytorch-linux-bionic-rocm4.1-py3.6"
- docker_build_job:
@@ -8938,9 +8885,9 @@
only:
- postnightly
executor: windows-with-nvidia-gpu
- when: << pipeline.parameters.run_build >>
- master_build:
- jobs:
+ - docker_build_job:
+ name: "docker-pytorch-linux-bionic-py3.6-clang9"
+ image_name: "pytorch-linux-bionic-py3.6-clang9"
- docker_build_job:
name: "docker-pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
image_name: "pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
@@ -8948,11 +8895,28 @@
name: "docker-pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
image_name: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
- docker_build_job:
+ name: "docker-pytorch-linux-xenial-py3-clang5-asan"
+ image_name: "pytorch-linux-xenial-py3-clang5-asan"
+ - docker_build_job:
+ name: "docker-pytorch-linux-xenial-py3-clang7-asan"
+ image_name: "pytorch-linux-xenial-py3-clang7-asan"
+ - docker_build_job:
+ name: "docker-pytorch-linux-xenial-py3-clang7-onnx"
+ image_name: "pytorch-linux-xenial-py3-clang7-onnx"
+ - docker_build_job:
name: "docker-pytorch-linux-xenial-py3.6-gcc5.4"
image_name: "pytorch-linux-xenial-py3.6-gcc5.4"
+ filters:
+ branches:
+ only: /.*/
+ tags:
+ only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
- docker_build_job:
name: "docker-pytorch-linux-xenial-py3.6-gcc7"
image_name: "pytorch-linux-xenial-py3.6-gcc7"
+ when: << pipeline.parameters.run_build >>
+ master_build:
+ jobs:
- pytorch_linux_build:
name: pytorch_linux_xenial_py3_6_gcc5_4_build
requires:
@@ -9085,6 +9049,18 @@
name: binary_linux_libtorch_3_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_test
requires:
- binary_linux_libtorch_3_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_build
+ - docker_build_job:
+ name: "docker-pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
+ image_name: "pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
+ - docker_build_job:
+ name: "docker-pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
+ image_name: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
+ - docker_build_job:
+ name: "docker-pytorch-linux-xenial-py3.6-gcc5.4"
+ image_name: "pytorch-linux-xenial-py3.6-gcc5.4"
+ - docker_build_job:
+ name: "docker-pytorch-linux-xenial-py3.6-gcc7"
+ image_name: "pytorch-linux-xenial-py3.6-gcc7"
when: << pipeline.parameters.run_master_build >>
slow_gradcheck_build:
jobs:
diff --git a/.circleci/generate_config_yml.py b/.circleci/generate_config_yml.py
index 6513d2e..41f0487 100755
--- a/.circleci/generate_config_yml.py
+++ b/.circleci/generate_config_yml.py
@@ -77,15 +77,15 @@
for line in filter(None, lines):
output_filehandle.write(line + "\n")
-def filter_master_only_jobs(items):
- def _for_all_items(items, functor) -> None:
- if isinstance(items, list):
- for item in items:
- _for_all_items(item, functor)
- if isinstance(items, dict) and len(items) == 1:
- item_type, item = next(iter(items.items()))
- functor(item_type, item)
+def _for_all_items(items, functor) -> None:
+ if isinstance(items, list):
+ for item in items:
+ _for_all_items(item, functor)
+ if isinstance(items, dict) and len(items) == 1:
+ item_type, item = next(iter(items.items()))
+ functor(item_type, item)
+def filter_master_only_jobs(items):
def _is_master_item(item):
filters = item.get('filters', None)
branches = filters.get('branches', None) if filters is not None else None
@@ -123,9 +123,24 @@
_for_all_items(items, _save_requires_if_master)
return _do_filtering(items)
+def generate_required_docker_images(items):
+ required_docker_images = set()
+
+ def _requires_docker_image(item_type, item):
+ requires = item.get('requires', None)
+ if not isinstance(requires, list):
+ return
+ for requirement in requires:
+ requirement = requirement.replace('"', '')
+ if requirement.startswith('docker-'):
+ required_docker_images.add(requirement)
+
+ _for_all_items(items, _requires_docker_image)
+ return required_docker_images
def gen_build_workflows_tree():
build_workflows_functions = [
+ # For rocm images, which don't have a circleci job equivalent
cimodel.data.simple.docker_definitions.get_workflow_jobs,
pytorch_build_definitions.get_workflow_jobs,
cimodel.data.simple.macos_definitions.get_workflow_jobs,
@@ -140,6 +155,12 @@
binary_build_definitions.get_binary_smoke_test_jobs,
]
build_jobs = [f() for f in build_workflows_functions]
+ build_jobs.extend(
+ cimodel.data.simple.docker_definitions.get_workflow_jobs(
+ # sort for consistency
+ sorted(generate_required_docker_images(build_jobs))
+ )
+ )
master_build_jobs = filter_master_only_jobs(build_jobs)
binary_build_functions = [
@@ -148,10 +169,16 @@
binary_build_definitions.get_nightly_uploads,
]
- slow_gradcheck_jobs = [
- pytorch_build_definitions.get_workflow_jobs,
- cimodel.data.simple.docker_definitions.get_workflow_jobs,
+ slow_gradcheck_functions = [
+ pytorch_build_definitions.get_workflow_jobs
]
+ slow_gradcheck_jobs = [f(only_slow_gradcheck=True) for f in slow_gradcheck_functions]
+ slow_gradcheck_jobs.extend(
+ cimodel.data.simple.docker_definitions.get_workflow_jobs(
+ # sort for consistency
+ sorted(generate_required_docker_images(slow_gradcheck_jobs))
+ )
+ )
return {
"workflows": {
@@ -169,7 +196,7 @@
},
"slow_gradcheck_build": {
"when": r"<< pipeline.parameters.run_slow_gradcheck_build >>",
- "jobs": [f(only_slow_gradcheck=True) for f in slow_gradcheck_jobs],
+ "jobs": slow_gradcheck_jobs,
},
}
}
