[BE] Move upload logic to shared template (#72426)
Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/72426
Test Plan: Imported from OSS
Reviewed By: mruberry
Differential Revision: D34050483
Pulled By: malfet
fbshipit-source-id: c8ab8505433a8eab3da10a1d2f990496e9e6300c
(cherry picked from commit 81346d7c8b8610aa9906a52a35e1b2d38d89442a)
diff --git a/.github/templates/linux_binary_build_workflow.yml.j2 b/.github/templates/linux_binary_build_workflow.yml.j2
index 6e24392..86144ff 100644
--- a/.github/templates/linux_binary_build_workflow.yml.j2
+++ b/.github/templates/linux_binary_build_workflow.yml.j2
@@ -1,4 +1,5 @@
{% import 'common.yml.j2' as common %}
+{% import 'upload.yml.j2' as upload %}
{%- block name -%}
# Template is at: .github/templates/linux_binary_build_workflow.yml.j2
@@ -6,26 +7,6 @@
name: !{{ build_environment }}
{%- endblock %}
-{%- macro binary_env(config) -%}
- env:
- PACKAGE_TYPE: !{{ config["package_type"] }}
- # TODO: This is a legacy variable that we eventually want to get rid of in
- # favor of GPU_ARCH_VERSION
- DESIRED_CUDA: !{{ config["desired_cuda"] }}
-{%- if config["gpu_arch_version"] %}
- GPU_ARCH_VERSION: !{{ config["gpu_arch_version"] }}
-{%- endif %}
- GPU_ARCH_TYPE: !{{ config["gpu_arch_type"] }}
- DOCKER_IMAGE: !{{ config["container_image"] }}
- SKIP_ALL_TESTS: 1
-{%- if config["package_type"] == "libtorch" %}
- LIBTORCH_VARIANT: !{{ config["libtorch_variant"] }}
- DESIRED_DEVTOOLSET: !{{ config["devtoolset"] }}
-{%- else %}
- DESIRED_PYTHON: "!{{ config["python_version"] }}"
-{%- endif %}
-{%- endmacro %}
-
on:
push:
# NOTE: Meta Employees can trigger new nightlies using: https://fburl.com/trigger_pytorch_nightly_build
@@ -68,7 +49,7 @@
if: ${{ github.repository_owner == 'pytorch' }}
runs-on: linux.4xlarge
timeout-minutes: !{{ common.timeout_minutes }}
- !{{ binary_env(config) }}
+ !{{ upload.binary_env(config) }}
steps:
!{{ common.setup_ec2_linux() }}
!{{ common.checkout(deep_clone=False, directory="pytorch") }}
@@ -130,7 +111,7 @@
runs-on: linux.4xlarge
{%- endif %}
timeout-minutes: !{{ common.timeout_minutes }}
- !{{ binary_env(config) }}
+ !{{ upload.binary_env(config) }}
steps:
!{{ common.setup_ec2_linux() }}
- uses: seemethere/download-artifact-s3@0504774707cbc8603d7dca922e8026eb8bf3b47b
@@ -193,53 +174,5 @@
docker exec -t -w "${PYTORCH_ROOT}" -e OUTPUT_SCRIPT="/run.sh" "${container_name}" bash -c "bash .circleci/scripts/binary_linux_test.sh"
docker exec -t "${container_name}" bash -c "source ${BINARY_ENV_FILE} && bash -x /run.sh"
!{{ common.teardown_ec2_linux("pytorch/") }}
- !{{ config["build_name"] }}-upload: # Uploading
- runs-on: linux.2xlarge # self hosted runner to download ec2 artifacts
- if: ${{ github.repository_owner == 'pytorch' }}
- needs: !{{ config["build_name"] }}-test
- !{{ binary_env(config) }}
- steps:
- !{{ common.setup_ec2_linux() }}
- - name: Clone pytorch/pytorch
- uses: actions/checkout@v2
- - uses: seemethere/download-artifact-s3@0504774707cbc8603d7dca922e8026eb8bf3b47b
- name: Download Build Artifacts
- with:
- name: !{{ config["build_name"] }}
- path: "${{ runner.temp }}/artifacts/"
- - name: Set DRY_RUN (only for tagged pushes)
- if: ${{ github.event_name == 'push' && (github.event.ref == 'refs/heads/nightly' || startsWith(github.event.ref, 'refs/tags/'))}}
- run: |
- echo "DRY_RUN=disabled" >> "$GITHUB_ENV"
- - name: Set UPLOAD_CHANNEL (only for tagged pushes)
- if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/')}}
- run: |
- # reference ends with an RC suffix
- if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then
- echo "UPLOAD_CHANNEL=test" >> "$GITHUB_ENV"
- fi
- - name: Upload binaries
- env:
- PKG_DIR: "${{ runner.temp }}/artifacts"
- UPLOAD_SUBFOLDER: "${{ env.DESIRED_CUDA }}"
- # When running these on pull_request events these should be blank
- AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_ACCESS_KEY_ID }}
- AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_SECRET_KEY }}
- ANACONDA_API_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }}
- run: |
- docker run --rm -i \
- -e ANACONDA_API_TOKEN \
- -e AWS_ACCESS_KEY_ID \
- -e AWS_SECRET_ACCESS_KEY \
- -e DRY_RUN \
- -e PACKAGE_TYPE \
- -e PKG_DIR=/artifacts \
- -e UPLOAD_CHANNEL \
- -e UPLOAD_SUBFOLDER \
- -v "${RUNNER_TEMP}/artifacts:/artifacts" \
- -v "${GITHUB_WORKSPACE}:/v" \
- -w /v \
- 308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/miniconda3:4.10.3 \
- bash -c '.circleci/scripts/binary_upload.sh'
- !{{ common.teardown_ec2_linux() }}
+ !{{ upload.upload_binaries(config) }}
{%- endfor %}
diff --git a/.github/templates/upload.yml.j2 b/.github/templates/upload.yml.j2
new file mode 100644
index 0000000..27be069
--- /dev/null
+++ b/.github/templates/upload.yml.j2
@@ -0,0 +1,85 @@
+{% import 'common.yml.j2' as common %}
+
+{%- macro binary_env(config, is_windows=False) -%}
+ env:
+{%- if is_windows %}
+ PYTORCH_ROOT: ${{ github.workspace }}/pytorch
+ BUILDER_ROOT: ${{ github.workspace }}/builder
+{%- endif %}
+ PACKAGE_TYPE: !{{ config["package_type"] }}
+ # TODO: This is a legacy variable that we eventually want to get rid of in
+ # favor of GPU_ARCH_VERSION
+ DESIRED_CUDA: !{{ config["desired_cuda"] }}
+{%- if config["gpu_arch_version"] %}
+ GPU_ARCH_VERSION: !{{ config["gpu_arch_version"] }}
+{%- endif %}
+ GPU_ARCH_TYPE: !{{ config["gpu_arch_type"] }}
+{%- if not is_windows %}
+ DOCKER_IMAGE: !{{ config["container_image"] }}
+{%- endif %}
+ SKIP_ALL_TESTS: 1
+{%- if config["package_type"] == "libtorch" %}
+ LIBTORCH_VARIANT: !{{ config["libtorch_variant"] }}
+ DESIRED_DEVTOOLSET: !{{ config["devtoolset"] }}
+{%- if is_windows %}
+ # This is a dummy value for libtorch to work correctly with our batch scripts
+ # without this value pip does not get installed for some reason
+ DESIRED_PYTHON: "3.7"
+{%- endif %}
+{%- else %}
+ DESIRED_PYTHON: "!{{ config["python_version"] }}"
+{%- endif %}
+{%- endmacro %}
+
+
+{%- macro upload_binaries(config, is_windows=False) -%}
+!{{ config["build_name"] }}-upload: # Uploading
+ runs-on: linux.2xlarge # self hosted runner to download ec2 artifacts
+ if: ${{ github.repository_owner == 'pytorch' }}
+ needs: !{{ config["build_name"] }}-test
+ !{{ binary_env(config, is_windows) }}
+ steps:
+ !{{ common.setup_ec2_linux() }}
+ - name: Clone pytorch/pytorch
+ uses: actions/checkout@v2
+ - uses: seemethere/download-artifact-s3@0504774707cbc8603d7dca922e8026eb8bf3b47b
+ name: Download Build Artifacts
+ with:
+ name: !{{ config["build_name"] }}
+ path: "${{ runner.temp }}/artifacts/"
+ - name: Set DRY_RUN (only for tagged pushes)
+ if: ${{ github.event_name == 'push' && (github.event.ref == 'refs/heads/nightly' || startsWith(github.event.ref, 'refs/tags/'))}}
+ run: |
+ echo "DRY_RUN=disabled" >> "$GITHUB_ENV"
+ - name: Set UPLOAD_CHANNEL (only for tagged pushes)
+ if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/')}}
+ run: |
+ # reference ends with an RC suffix
+ if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then
+ echo "UPLOAD_CHANNEL=test" >> "$GITHUB_ENV"
+ fi
+ - name: Upload binaries
+ env:
+ PKG_DIR: "${{ runner.temp }}/artifacts"
+ UPLOAD_SUBFOLDER: "${{ env.DESIRED_CUDA }}"
+ # When running these on pull_request events these should be blank
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_SECRET_KEY }}
+ ANACONDA_API_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }}
+ run: |
+ docker run --rm -i \
+ -e ANACONDA_API_TOKEN \
+ -e AWS_ACCESS_KEY_ID \
+ -e AWS_SECRET_ACCESS_KEY \
+ -e DRY_RUN \
+ -e PACKAGE_TYPE \
+ -e PKG_DIR=/artifacts \
+ -e UPLOAD_CHANNEL \
+ -e UPLOAD_SUBFOLDER \
+ -v "${RUNNER_TEMP}/artifacts:/artifacts" \
+ -v "${GITHUB_WORKSPACE}:/v" \
+ -w /v \
+ 308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/miniconda3:4.10.3 \
+ bash -c '.circleci/scripts/binary_upload.sh'
+ !{{ common.teardown_ec2_linux() }}
+{%- endmacro -%}
diff --git a/.github/templates/windows_binary_build_workflow.yml.j2 b/.github/templates/windows_binary_build_workflow.yml.j2
index 01cd315..5f49176 100644
--- a/.github/templates/windows_binary_build_workflow.yml.j2
+++ b/.github/templates/windows_binary_build_workflow.yml.j2
@@ -1,4 +1,5 @@
{% import 'common.yml.j2' as common %}
+{% import 'upload.yml.j2' as upload %}
{%- block name -%}
# Template is at: .github/templates/windows_binary_build_workflow.yml.j2
@@ -6,30 +7,6 @@
name: !{{ build_environment }}
{%- endblock %}
-{%- macro binary_env(config) -%}
- env:
- PYTORCH_ROOT: ${{ github.workspace }}/pytorch
- BUILDER_ROOT: ${{ github.workspace }}/builder
- PACKAGE_TYPE: !{{ config["package_type"] }}
- # TODO: This is a legacy variable that we eventually want to get rid of in
- # favor of GPU_ARCH_VERSION
- DESIRED_CUDA: !{{ config["desired_cuda"] }}
-{%- if config["gpu_arch_version"] %}
- GPU_ARCH_VERSION: !{{ config["gpu_arch_version"] }}
-{%- endif %}
- GPU_ARCH_TYPE: !{{ config["gpu_arch_type"] }}
- SKIP_ALL_TESTS: 1
-{%- if config["package_type"] == "libtorch" %}
- LIBTORCH_VARIANT: !{{ config["libtorch_variant"] }}
- DESIRED_DEVTOOLSET: !{{ config["devtoolset"] }}
- # This is a dummy value for libtorch to work correctly with our batch scripts
- # without this value pip does not get installed for some reason
- DESIRED_PYTHON: "3.7"
-{%- else %}
- DESIRED_PYTHON: "!{{ config["python_version"] }}"
-{%- endif %}
-{%- endmacro %}
-
{%- macro set_runner_specific_vars() -%}
# NOTE: These environment variables are put here so that they can be applied on every job equally
# They are also here because setting them at a workflow level doesn't give us access to the
@@ -79,7 +56,7 @@
!{{ config["build_name"] }}-build:
runs-on: windows.4xlarge
timeout-minutes: !{{ common.timeout_minutes }}
- !{{ binary_env(config) }}
+ !{{ upload.binary_env(config, True) }}
steps:
!{{ common.setup_ec2_windows() }}
!{{ set_runner_specific_vars() }}
@@ -118,7 +95,7 @@
runs-on: windows.4xlarge
{%- endif %}
timeout-minutes: !{{ common.timeout_minutes }}
- !{{ binary_env(config) }}
+ !{{ upload.binary_env(config, True) }}
steps:
!{{ common.setup_ec2_windows() }}
!{{ set_runner_specific_vars() }}
@@ -146,53 +123,5 @@
run: |
"${PYTORCH_ROOT}/.circleci/scripts/binary_windows_test.sh"
!{{ common.wait_and_kill_ssh_windows('pytorch') }}
- !{{ config["build_name"] }}-upload: # Uploading
- runs-on: linux.2xlarge # self hosted runner to download ec2 artifacts
- if: ${{ github.repository_owner == 'pytorch' }}
- needs: !{{ config["build_name"] }}-test
- !{{ binary_env(config) }}
- steps:
- !{{ common.setup_ec2_linux() }}
- - name: Clone pytorch/pytorch
- uses: actions/checkout@v2
- - uses: seemethere/download-artifact-s3@0504774707cbc8603d7dca922e8026eb8bf3b47b
- name: Download Build Artifacts
- with:
- name: !{{ config["build_name"] }}
- path: "${{ runner.temp }}/artifacts/"
- - name: Set DRY_RUN (only for tagged pushes)
- if: ${{ github.event_name == 'push' && (github.event.ref == 'refs/heads/nightly' || startsWith(github.event.ref, 'refs/tags/'))}}
- run: |
- echo "DRY_RUN=disabled" >> "$GITHUB_ENV"
- - name: Set UPLOAD_CHANNEL (only for tagged pushes)
- if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/')}}
- run: |
- # reference ends with an RC suffix
- if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then
- echo "UPLOAD_CHANNEL=test" >> "$GITHUB_ENV"
- fi
- - name: Upload binaries
- env:
- PKG_DIR: "${{ runner.temp }}/artifacts"
- UPLOAD_SUBFOLDER: "${{ env.DESIRED_CUDA }}"
- # When running these on pull_request events these should be blank
- AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_ACCESS_KEY_ID }}
- AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_SECRET_KEY }}
- ANACONDA_API_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }}
- run: |
- docker run --rm -i \
- -e ANACONDA_API_TOKEN \
- -e AWS_ACCESS_KEY_ID \
- -e AWS_SECRET_ACCESS_KEY \
- -e DRY_RUN \
- -e PACKAGE_TYPE \
- -e PKG_DIR=/artifacts \
- -e UPLOAD_CHANNEL \
- -e UPLOAD_SUBFOLDER \
- -v "${RUNNER_TEMP}/artifacts:/artifacts" \
- -v "${GITHUB_WORKSPACE}:/v" \
- -w /v \
- 308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/miniconda3:4.10.3 \
- bash -c '.circleci/scripts/binary_upload.sh'
- !{{ common.teardown_ec2_linux() }}
+ !{{ upload.upload_binaries(config, True) }}
{%- endfor %}
diff --git a/.github/workflows/generated-linux-binary-conda.yml b/.github/workflows/generated-linux-binary-conda.yml
index 541720f..6c1628d 100644
--- a/.github/workflows/generated-linux-binary-conda.yml
+++ b/.github/workflows/generated-linux-binary-conda.yml
@@ -1,4 +1,5 @@
# @generated DO NOT EDIT MANUALLY
+
# Template is at: .github/templates/linux_binary_build_workflow.yml.j2
# Generation script: .github/scripts/generate_ci_workflows.py
name: linux-binary-conda
diff --git a/.github/workflows/generated-linux-binary-libtorch-cxx11-abi.yml b/.github/workflows/generated-linux-binary-libtorch-cxx11-abi.yml
index b83f171..5946042 100644
--- a/.github/workflows/generated-linux-binary-libtorch-cxx11-abi.yml
+++ b/.github/workflows/generated-linux-binary-libtorch-cxx11-abi.yml
@@ -1,4 +1,5 @@
# @generated DO NOT EDIT MANUALLY
+
# Template is at: .github/templates/linux_binary_build_workflow.yml.j2
# Generation script: .github/scripts/generate_ci_workflows.py
name: linux-binary-libtorch-cxx11-abi
diff --git a/.github/workflows/generated-linux-binary-libtorch-pre-cxx11.yml b/.github/workflows/generated-linux-binary-libtorch-pre-cxx11.yml
index d119640..1bb3f03 100644
--- a/.github/workflows/generated-linux-binary-libtorch-pre-cxx11.yml
+++ b/.github/workflows/generated-linux-binary-libtorch-pre-cxx11.yml
@@ -1,4 +1,5 @@
# @generated DO NOT EDIT MANUALLY
+
# Template is at: .github/templates/linux_binary_build_workflow.yml.j2
# Generation script: .github/scripts/generate_ci_workflows.py
name: linux-binary-libtorch-pre-cxx11
diff --git a/.github/workflows/generated-linux-binary-manywheel.yml b/.github/workflows/generated-linux-binary-manywheel.yml
index b20cf57..562669c 100644
--- a/.github/workflows/generated-linux-binary-manywheel.yml
+++ b/.github/workflows/generated-linux-binary-manywheel.yml
@@ -1,4 +1,5 @@
# @generated DO NOT EDIT MANUALLY
+
# Template is at: .github/templates/linux_binary_build_workflow.yml.j2
# Generation script: .github/scripts/generate_ci_workflows.py
name: linux-binary-manywheel
diff --git a/.github/workflows/generated-windows-binary-libtorch-cxx11-abi.yml b/.github/workflows/generated-windows-binary-libtorch-cxx11-abi.yml
index cfa3c60..f4c536c 100644
--- a/.github/workflows/generated-windows-binary-libtorch-cxx11-abi.yml
+++ b/.github/workflows/generated-windows-binary-libtorch-cxx11-abi.yml
@@ -1,4 +1,5 @@
# @generated DO NOT EDIT MANUALLY
+
# Template is at: .github/templates/windows_binary_build_workflow.yml.j2
# Generation script: .github/scripts/generate_ci_workflows.py
name: windows-binary-libtorch-cxx11-abi
diff --git a/.github/workflows/generated-windows-binary-libtorch-pre-cxx11.yml b/.github/workflows/generated-windows-binary-libtorch-pre-cxx11.yml
index bf48abc..b801710 100644
--- a/.github/workflows/generated-windows-binary-libtorch-pre-cxx11.yml
+++ b/.github/workflows/generated-windows-binary-libtorch-pre-cxx11.yml
@@ -1,4 +1,5 @@
# @generated DO NOT EDIT MANUALLY
+
# Template is at: .github/templates/windows_binary_build_workflow.yml.j2
# Generation script: .github/scripts/generate_ci_workflows.py
name: windows-binary-libtorch-pre-cxx11
diff --git a/.github/workflows/generated-windows-binary-wheel.yml b/.github/workflows/generated-windows-binary-wheel.yml
index 9a44e7f..7f4baec 100644
--- a/.github/workflows/generated-windows-binary-wheel.yml
+++ b/.github/workflows/generated-windows-binary-wheel.yml
@@ -1,4 +1,5 @@
# @generated DO NOT EDIT MANUALLY
+
# Template is at: .github/templates/windows_binary_build_workflow.yml.j2
# Generation script: .github/scripts/generate_ci_workflows.py
name: windows-binary-wheel