[BE] Move upload logic to shared template (#72426)

Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/72426

Test Plan: Imported from OSS

Reviewed By: mruberry

Differential Revision: D34050483

Pulled By: malfet

fbshipit-source-id: c8ab8505433a8eab3da10a1d2f990496e9e6300c
(cherry picked from commit 81346d7c8b8610aa9906a52a35e1b2d38d89442a)
diff --git a/.github/templates/linux_binary_build_workflow.yml.j2 b/.github/templates/linux_binary_build_workflow.yml.j2
index 6e24392..86144ff 100644
--- a/.github/templates/linux_binary_build_workflow.yml.j2
+++ b/.github/templates/linux_binary_build_workflow.yml.j2
@@ -1,4 +1,5 @@
 {% import 'common.yml.j2' as common %}
+{% import 'upload.yml.j2' as upload %}
 
 {%- block name -%}
 # Template is at:    .github/templates/linux_binary_build_workflow.yml.j2
@@ -6,26 +7,6 @@
 name: !{{ build_environment }}
 {%- endblock %}
 
-{%- macro binary_env(config) -%}
-    env:
-      PACKAGE_TYPE: !{{ config["package_type"] }}
-      # TODO: This is a legacy variable that we eventually want to get rid of in
-      #       favor of GPU_ARCH_VERSION
-      DESIRED_CUDA: !{{ config["desired_cuda"] }}
-{%- if config["gpu_arch_version"] %}
-      GPU_ARCH_VERSION: !{{ config["gpu_arch_version"] }}
-{%- endif %}
-      GPU_ARCH_TYPE: !{{ config["gpu_arch_type"] }}
-      DOCKER_IMAGE: !{{ config["container_image"] }}
-      SKIP_ALL_TESTS: 1
-{%- if config["package_type"] == "libtorch" %}
-      LIBTORCH_VARIANT: !{{ config["libtorch_variant"] }}
-      DESIRED_DEVTOOLSET: !{{ config["devtoolset"] }}
-{%- else %}
-      DESIRED_PYTHON: "!{{ config["python_version"] }}"
-{%- endif %}
-{%- endmacro %}
-
 on:
   push:
     # NOTE: Meta Employees can trigger new nightlies using: https://fburl.com/trigger_pytorch_nightly_build
@@ -68,7 +49,7 @@
     if: ${{ github.repository_owner == 'pytorch' }}
     runs-on: linux.4xlarge
     timeout-minutes: !{{ common.timeout_minutes }}
-    !{{ binary_env(config) }}
+    !{{ upload.binary_env(config) }}
     steps:
       !{{ common.setup_ec2_linux() }}
       !{{ common.checkout(deep_clone=False, directory="pytorch") }}
@@ -130,7 +111,7 @@
     runs-on: linux.4xlarge
 {%- endif %}
     timeout-minutes: !{{ common.timeout_minutes }}
-    !{{ binary_env(config) }}
+    !{{ upload.binary_env(config) }}
     steps:
       !{{ common.setup_ec2_linux() }}
       - uses: seemethere/download-artifact-s3@0504774707cbc8603d7dca922e8026eb8bf3b47b
@@ -193,53 +174,5 @@
           docker exec -t -w "${PYTORCH_ROOT}" -e OUTPUT_SCRIPT="/run.sh" "${container_name}" bash -c "bash .circleci/scripts/binary_linux_test.sh"
           docker exec -t "${container_name}" bash -c "source ${BINARY_ENV_FILE} && bash -x /run.sh"
       !{{ common.teardown_ec2_linux("pytorch/") }}
-  !{{ config["build_name"] }}-upload:  # Uploading
-    runs-on: linux.2xlarge  # self hosted runner to download ec2 artifacts
-    if: ${{ github.repository_owner == 'pytorch' }}
-    needs: !{{ config["build_name"] }}-test
-    !{{ binary_env(config) }}
-    steps:
-      !{{ common.setup_ec2_linux() }}
-      - name: Clone pytorch/pytorch
-        uses: actions/checkout@v2
-      - uses: seemethere/download-artifact-s3@0504774707cbc8603d7dca922e8026eb8bf3b47b
-        name: Download Build Artifacts
-        with:
-          name: !{{ config["build_name"] }}
-          path: "${{ runner.temp }}/artifacts/"
-      - name: Set DRY_RUN (only for tagged pushes)
-        if: ${{ github.event_name == 'push' && (github.event.ref == 'refs/heads/nightly' || startsWith(github.event.ref, 'refs/tags/'))}}
-        run: |
-          echo "DRY_RUN=disabled" >> "$GITHUB_ENV"
-      - name: Set UPLOAD_CHANNEL (only for tagged pushes)
-        if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/')}}
-        run: |
-          # reference ends with an RC suffix
-          if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then
-            echo "UPLOAD_CHANNEL=test" >> "$GITHUB_ENV"
-          fi
-      - name: Upload binaries
-        env:
-          PKG_DIR: "${{ runner.temp }}/artifacts"
-          UPLOAD_SUBFOLDER: "${{ env.DESIRED_CUDA }}"
-          # When running these on pull_request events these should be blank
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_SECRET_KEY }}
-          ANACONDA_API_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }}
-        run: |
-          docker run --rm -i \
-            -e ANACONDA_API_TOKEN \
-            -e AWS_ACCESS_KEY_ID \
-            -e AWS_SECRET_ACCESS_KEY \
-            -e DRY_RUN \
-            -e PACKAGE_TYPE \
-            -e PKG_DIR=/artifacts \
-            -e UPLOAD_CHANNEL \
-            -e UPLOAD_SUBFOLDER \
-            -v "${RUNNER_TEMP}/artifacts:/artifacts" \
-            -v "${GITHUB_WORKSPACE}:/v" \
-            -w /v \
-            308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/miniconda3:4.10.3 \
-            bash -c '.circleci/scripts/binary_upload.sh'
-      !{{ common.teardown_ec2_linux() }}
+  !{{ upload.upload_binaries(config) }}
 {%- endfor %}
diff --git a/.github/templates/upload.yml.j2 b/.github/templates/upload.yml.j2
new file mode 100644
index 0000000..27be069
--- /dev/null
+++ b/.github/templates/upload.yml.j2
@@ -0,0 +1,85 @@
+{% import 'common.yml.j2' as common %}
+
+{%- macro binary_env(config, is_windows=False) -%}
+    env:
+{%- if is_windows %}
+      PYTORCH_ROOT: ${{ github.workspace }}/pytorch
+      BUILDER_ROOT: ${{ github.workspace }}/builder
+{%- endif %}
+      PACKAGE_TYPE: !{{ config["package_type"] }}
+      # TODO: This is a legacy variable that we eventually want to get rid of in
+      #       favor of GPU_ARCH_VERSION
+      DESIRED_CUDA: !{{ config["desired_cuda"] }}
+{%- if config["gpu_arch_version"] %}
+      GPU_ARCH_VERSION: !{{ config["gpu_arch_version"] }}
+{%- endif %}
+      GPU_ARCH_TYPE: !{{ config["gpu_arch_type"] }}
+{%- if not is_windows %}
+      DOCKER_IMAGE: !{{ config["container_image"] }}
+{%- endif %}
+      SKIP_ALL_TESTS: 1
+{%- if config["package_type"] == "libtorch" %}
+      LIBTORCH_VARIANT: !{{ config["libtorch_variant"] }}
+      DESIRED_DEVTOOLSET: !{{ config["devtoolset"] }}
+{%- if is_windows %}
+      # This is a dummy value for libtorch to work correctly with our batch scripts
+      # without this value pip does not get installed for some reason
+      DESIRED_PYTHON: "3.7"
+{%- endif %}
+{%- else %}
+      DESIRED_PYTHON: "!{{ config["python_version"] }}"
+{%- endif %}
+{%- endmacro %}
+
+
+{%- macro upload_binaries(config, is_windows=False) -%}
+!{{ config["build_name"] }}-upload:  # Uploading
+    runs-on: linux.2xlarge  # self hosted runner to download ec2 artifacts
+    if: ${{ github.repository_owner == 'pytorch' }}
+    needs: !{{ config["build_name"] }}-test
+    !{{ binary_env(config, is_windows) }}
+    steps:
+      !{{ common.setup_ec2_linux() }}
+      - name: Clone pytorch/pytorch
+        uses: actions/checkout@v2
+      - uses: seemethere/download-artifact-s3@0504774707cbc8603d7dca922e8026eb8bf3b47b
+        name: Download Build Artifacts
+        with:
+          name: !{{ config["build_name"] }}
+          path: "${{ runner.temp }}/artifacts/"
+      - name: Set DRY_RUN (only for tagged pushes)
+        if: ${{ github.event_name == 'push' && (github.event.ref == 'refs/heads/nightly' || startsWith(github.event.ref, 'refs/tags/'))}}
+        run: |
+          echo "DRY_RUN=disabled" >> "$GITHUB_ENV"
+      - name: Set UPLOAD_CHANNEL (only for tagged pushes)
+        if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/')}}
+        run: |
+          # reference ends with an RC suffix
+          if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then
+            echo "UPLOAD_CHANNEL=test" >> "$GITHUB_ENV"
+          fi
+      - name: Upload binaries
+        env:
+          PKG_DIR: "${{ runner.temp }}/artifacts"
+          UPLOAD_SUBFOLDER: "${{ env.DESIRED_CUDA }}"
+          # When running these on pull_request events these should be blank
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_SECRET_KEY }}
+          ANACONDA_API_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }}
+        run: |
+          docker run --rm -i \
+            -e ANACONDA_API_TOKEN \
+            -e AWS_ACCESS_KEY_ID \
+            -e AWS_SECRET_ACCESS_KEY \
+            -e DRY_RUN \
+            -e PACKAGE_TYPE \
+            -e PKG_DIR=/artifacts \
+            -e UPLOAD_CHANNEL \
+            -e UPLOAD_SUBFOLDER \
+            -v "${RUNNER_TEMP}/artifacts:/artifacts" \
+            -v "${GITHUB_WORKSPACE}:/v" \
+            -w /v \
+            308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/miniconda3:4.10.3 \
+            bash -c '.circleci/scripts/binary_upload.sh'
+      !{{ common.teardown_ec2_linux() }}
+{%- endmacro -%}
diff --git a/.github/templates/windows_binary_build_workflow.yml.j2 b/.github/templates/windows_binary_build_workflow.yml.j2
index 01cd315..5f49176 100644
--- a/.github/templates/windows_binary_build_workflow.yml.j2
+++ b/.github/templates/windows_binary_build_workflow.yml.j2
@@ -1,4 +1,5 @@
 {% import 'common.yml.j2' as common %}
+{% import 'upload.yml.j2' as upload %}
 
 {%- block name -%}
 # Template is at:    .github/templates/windows_binary_build_workflow.yml.j2
@@ -6,30 +7,6 @@
 name: !{{ build_environment }}
 {%- endblock %}
 
-{%- macro binary_env(config) -%}
-    env:
-      PYTORCH_ROOT: ${{ github.workspace }}/pytorch
-      BUILDER_ROOT: ${{ github.workspace }}/builder
-      PACKAGE_TYPE: !{{ config["package_type"] }}
-      # TODO: This is a legacy variable that we eventually want to get rid of in
-      #       favor of GPU_ARCH_VERSION
-      DESIRED_CUDA: !{{ config["desired_cuda"] }}
-{%- if config["gpu_arch_version"] %}
-      GPU_ARCH_VERSION: !{{ config["gpu_arch_version"] }}
-{%- endif %}
-      GPU_ARCH_TYPE: !{{ config["gpu_arch_type"] }}
-      SKIP_ALL_TESTS: 1
-{%- if config["package_type"] == "libtorch" %}
-      LIBTORCH_VARIANT: !{{ config["libtorch_variant"] }}
-      DESIRED_DEVTOOLSET: !{{ config["devtoolset"] }}
-      # This is a dummy value for libtorch to work correctly with our batch scripts
-      # without this value pip does not get installed for some reason
-      DESIRED_PYTHON: "3.7"
-{%- else %}
-      DESIRED_PYTHON: "!{{ config["python_version"] }}"
-{%- endif %}
-{%- endmacro %}
-
 {%- macro set_runner_specific_vars() -%}
       # NOTE: These environment variables are put here so that they can be applied on every job equally
       #       They are also here because setting them at a workflow level doesn't give us access to the
@@ -79,7 +56,7 @@
   !{{ config["build_name"] }}-build:
     runs-on: windows.4xlarge
     timeout-minutes: !{{ common.timeout_minutes }}
-    !{{ binary_env(config) }}
+    !{{ upload.binary_env(config, True) }}
     steps:
       !{{ common.setup_ec2_windows() }}
       !{{ set_runner_specific_vars() }}
@@ -118,7 +95,7 @@
     runs-on: windows.4xlarge
 {%- endif %}
     timeout-minutes: !{{ common.timeout_minutes }}
-    !{{ binary_env(config) }}
+    !{{ upload.binary_env(config, True) }}
     steps:
       !{{ common.setup_ec2_windows() }}
       !{{ set_runner_specific_vars() }}
@@ -146,53 +123,5 @@
         run: |
           "${PYTORCH_ROOT}/.circleci/scripts/binary_windows_test.sh"
       !{{ common.wait_and_kill_ssh_windows('pytorch') }}
-  !{{ config["build_name"] }}-upload:  # Uploading
-    runs-on: linux.2xlarge  # self hosted runner to download ec2 artifacts
-    if: ${{ github.repository_owner == 'pytorch' }}
-    needs: !{{ config["build_name"] }}-test
-    !{{ binary_env(config) }}
-    steps:
-      !{{ common.setup_ec2_linux() }}
-      - name: Clone pytorch/pytorch
-        uses: actions/checkout@v2
-      - uses: seemethere/download-artifact-s3@0504774707cbc8603d7dca922e8026eb8bf3b47b
-        name: Download Build Artifacts
-        with:
-          name: !{{ config["build_name"] }}
-          path: "${{ runner.temp }}/artifacts/"
-      - name: Set DRY_RUN (only for tagged pushes)
-        if: ${{ github.event_name == 'push' && (github.event.ref == 'refs/heads/nightly' || startsWith(github.event.ref, 'refs/tags/'))}}
-        run: |
-          echo "DRY_RUN=disabled" >> "$GITHUB_ENV"
-      - name: Set UPLOAD_CHANNEL (only for tagged pushes)
-        if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/')}}
-        run: |
-          # reference ends with an RC suffix
-          if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then
-            echo "UPLOAD_CHANNEL=test" >> "$GITHUB_ENV"
-          fi
-      - name: Upload binaries
-        env:
-          PKG_DIR: "${{ runner.temp }}/artifacts"
-          UPLOAD_SUBFOLDER: "${{ env.DESIRED_CUDA }}"
-          # When running these on pull_request events these should be blank
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_SECRET_KEY }}
-          ANACONDA_API_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }}
-        run: |
-          docker run --rm -i \
-            -e ANACONDA_API_TOKEN \
-            -e AWS_ACCESS_KEY_ID \
-            -e AWS_SECRET_ACCESS_KEY \
-            -e DRY_RUN \
-            -e PACKAGE_TYPE \
-            -e PKG_DIR=/artifacts \
-            -e UPLOAD_CHANNEL \
-            -e UPLOAD_SUBFOLDER \
-            -v "${RUNNER_TEMP}/artifacts:/artifacts" \
-            -v "${GITHUB_WORKSPACE}:/v" \
-            -w /v \
-            308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/miniconda3:4.10.3 \
-            bash -c '.circleci/scripts/binary_upload.sh'
-      !{{ common.teardown_ec2_linux() }}
+  !{{ upload.upload_binaries(config, True) }}
 {%- endfor %}
diff --git a/.github/workflows/generated-linux-binary-conda.yml b/.github/workflows/generated-linux-binary-conda.yml
index 541720f..6c1628d 100644
--- a/.github/workflows/generated-linux-binary-conda.yml
+++ b/.github/workflows/generated-linux-binary-conda.yml
@@ -1,4 +1,5 @@
 # @generated DO NOT EDIT MANUALLY
+
 # Template is at:    .github/templates/linux_binary_build_workflow.yml.j2
 # Generation script: .github/scripts/generate_ci_workflows.py
 name: linux-binary-conda
diff --git a/.github/workflows/generated-linux-binary-libtorch-cxx11-abi.yml b/.github/workflows/generated-linux-binary-libtorch-cxx11-abi.yml
index b83f171..5946042 100644
--- a/.github/workflows/generated-linux-binary-libtorch-cxx11-abi.yml
+++ b/.github/workflows/generated-linux-binary-libtorch-cxx11-abi.yml
@@ -1,4 +1,5 @@
 # @generated DO NOT EDIT MANUALLY
+
 # Template is at:    .github/templates/linux_binary_build_workflow.yml.j2
 # Generation script: .github/scripts/generate_ci_workflows.py
 name: linux-binary-libtorch-cxx11-abi
diff --git a/.github/workflows/generated-linux-binary-libtorch-pre-cxx11.yml b/.github/workflows/generated-linux-binary-libtorch-pre-cxx11.yml
index d119640..1bb3f03 100644
--- a/.github/workflows/generated-linux-binary-libtorch-pre-cxx11.yml
+++ b/.github/workflows/generated-linux-binary-libtorch-pre-cxx11.yml
@@ -1,4 +1,5 @@
 # @generated DO NOT EDIT MANUALLY
+
 # Template is at:    .github/templates/linux_binary_build_workflow.yml.j2
 # Generation script: .github/scripts/generate_ci_workflows.py
 name: linux-binary-libtorch-pre-cxx11
diff --git a/.github/workflows/generated-linux-binary-manywheel.yml b/.github/workflows/generated-linux-binary-manywheel.yml
index b20cf57..562669c 100644
--- a/.github/workflows/generated-linux-binary-manywheel.yml
+++ b/.github/workflows/generated-linux-binary-manywheel.yml
@@ -1,4 +1,5 @@
 # @generated DO NOT EDIT MANUALLY
+
 # Template is at:    .github/templates/linux_binary_build_workflow.yml.j2
 # Generation script: .github/scripts/generate_ci_workflows.py
 name: linux-binary-manywheel
diff --git a/.github/workflows/generated-windows-binary-libtorch-cxx11-abi.yml b/.github/workflows/generated-windows-binary-libtorch-cxx11-abi.yml
index cfa3c60..f4c536c 100644
--- a/.github/workflows/generated-windows-binary-libtorch-cxx11-abi.yml
+++ b/.github/workflows/generated-windows-binary-libtorch-cxx11-abi.yml
@@ -1,4 +1,5 @@
 # @generated DO NOT EDIT MANUALLY
+
 # Template is at:    .github/templates/windows_binary_build_workflow.yml.j2
 # Generation script: .github/scripts/generate_ci_workflows.py
 name: windows-binary-libtorch-cxx11-abi
diff --git a/.github/workflows/generated-windows-binary-libtorch-pre-cxx11.yml b/.github/workflows/generated-windows-binary-libtorch-pre-cxx11.yml
index bf48abc..b801710 100644
--- a/.github/workflows/generated-windows-binary-libtorch-pre-cxx11.yml
+++ b/.github/workflows/generated-windows-binary-libtorch-pre-cxx11.yml
@@ -1,4 +1,5 @@
 # @generated DO NOT EDIT MANUALLY
+
 # Template is at:    .github/templates/windows_binary_build_workflow.yml.j2
 # Generation script: .github/scripts/generate_ci_workflows.py
 name: windows-binary-libtorch-pre-cxx11
diff --git a/.github/workflows/generated-windows-binary-wheel.yml b/.github/workflows/generated-windows-binary-wheel.yml
index 9a44e7f..7f4baec 100644
--- a/.github/workflows/generated-windows-binary-wheel.yml
+++ b/.github/workflows/generated-windows-binary-wheel.yml
@@ -1,4 +1,5 @@
 # @generated DO NOT EDIT MANUALLY
+
 # Template is at:    .github/templates/windows_binary_build_workflow.yml.j2
 # Generation script: .github/scripts/generate_ci_workflows.py
 name: windows-binary-wheel