Enable bionic-cuda10.2-cudnn7-py3.9-gcc7 in GHA (#60204)
Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/60204
Test Plan: Imported from OSS
Reviewed By: malfet
Differential Revision: D29430679
Pulled By: samestep
fbshipit-source-id: 9380f5535cd370ec7aabf609a6170c8cb4df505d
diff --git a/.github/scripts/generate_ci_workflows.py b/.github/scripts/generate_ci_workflows.py
index d782d22..2bd1a7f 100755
--- a/.github/scripts/generate_ci_workflows.py
+++ b/.github/scripts/generate_ci_workflows.py
@@ -138,6 +138,11 @@
# test_runner_type=LINUX_CPU_TEST_RUNNER,
# ),
PyTorchLinuxWorkflow(
+ build_environment="pytorch-linux-bionic-cuda10.2-cudnn7-py3.9-gcc7",
+ docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-bionic-cuda10.2-cudnn7-py3.9-gcc7",
+ test_runner_type=LINUX_CUDA_TEST_RUNNER,
+ ),
+ PyTorchLinuxWorkflow(
build_environment="pytorch-linux-xenial-cuda10.2-cudnn7-py3.6-gcc7",
docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7",
test_runner_type=LINUX_CUDA_TEST_RUNNER,
diff --git a/.github/workflows/pytorch-linux-bionic-cuda10.2-cudnn7-py3.9-gcc7.yml b/.github/workflows/pytorch-linux-bionic-cuda10.2-cudnn7-py3.9-gcc7.yml
new file mode 100644
index 0000000..80448b8
--- /dev/null
+++ b/.github/workflows/pytorch-linux-bionic-cuda10.2-cudnn7-py3.9-gcc7.yml
@@ -0,0 +1,384 @@
+# @generated DO NOT EDIT MANUALLY
+# Template is at: .github/templates/linux_ci_workflow.yml.j2
+# Generation script: .github/scripts/generate_ci_workflows.py
+name: Linux CI (pytorch-linux-bionic-cuda10.2-cudnn7-py3.9-gcc7)
+
+on:
+ # TODO: Enable pull_request builds when we can verify capacity can be met by auto-scalers
+ push:
+ branches:
+ - master
+ - release/*
+ workflow_dispatch:
+
+env:
+ BUILD_ENVIRONMENT: pytorch-linux-bionic-cuda10.2-cudnn7-py3.9-gcc7
+ DOCKER_IMAGE_BASE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-cuda10.2-cudnn7-py3.9-gcc7
+ SCCACHE_BUCKET: ossci-compiler-cache-circleci-v2
+ TORCH_CUDA_ARCH_LIST: 5.2
+ IN_CI: 1
+ # Used for custom_opertor, jit_hooks, custom_backend, see .jenkins/pytorch/build.sh
+ CUSTOM_TEST_ARTIFACT_BUILD_DIR: build/custom_test_artifacts
+ ALPINE_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/alpine"
+
+concurrency:
+ group: pytorch-linux-bionic-cuda10.2-cudnn7-py3.9-gcc7-${{ github.event.pull_request.number || github.sha }}
+ cancel-in-progress: true
+
+jobs:
+ calculate-docker-image:
+ runs-on: linux.2xlarge
+ env:
+ DOCKER_BUILDKIT: 1
+ timeout-minutes: 90
+ outputs:
+ docker_image: ${{ steps.calculate-tag.outputs.docker_image }}
+ steps:
+ - name: Log in to ECR
+ run: |
+ aws ecr get-login --no-include-email --region us-east-1 > /tmp/ecr-login.sh
+ bash /tmp/ecr-login.sh
+ rm /tmp/ecr-login.sh
+ - name: Chown workspace
+ run: |
+ # Ensure the working directory gets chowned back to the current user
+ docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" .
+ - name: Checkout PyTorch
+ uses: actions/checkout@v2
+ with:
+ # deep clone, to allow use of git merge-base
+ fetch-depth: 0
+ - name: Calculate docker image tag
+ id: calculate-tag
+ run: |
+ DOCKER_TAG=$(git rev-parse HEAD:.circleci/docker)
+ echo "::set-output name=docker_tag::${DOCKER_TAG}"
+ echo "::set-output name=docker_image::${DOCKER_IMAGE_BASE}:${DOCKER_TAG}"
+ - name: Check if image should be built
+ id: check
+ env:
+ DOCKER_TAG: ${{ steps.calculate-tag.outputs.docker_tag }}
+ BASE_REVISION: ${{ github.event.pull_request.base.sha || github.sha }}
+ run: |
+ set -x
+ # Check if image already exists, if it does then skip building it
+ if docker manifest inspect "${DOCKER_IMAGE_BASE}:${DOCKER_TAG}"; then
+ exit 0
+ fi
+ if [[ "$BASE_REVISION" = "$(git rev-parse HEAD)" ]]; then
+ # if we're on the base branch then use the parent commit
+ MERGE_BASE=$(git rev-parse HEAD~)
+ else
+ # otherwise we're on a PR, so use the most recent base commit
+ MERGE_BASE=$(git merge-base HEAD "$BASE_REVISION")
+ fi
+ # Covers the case where a previous tag doesn't exist for the tree
+ # this is only really applicable on trees that don't have `.circleci/docker` at its merge base, i.e. nightly
+ if ! git rev-parse "$MERGE_BASE:.circleci/docker"; then
+ echo "Directory '.circleci/docker' not found in commit $MERGE_BASE, you should probably rebase onto a more recent commit"
+ exit 1
+ fi
+ PREVIOUS_DOCKER_TAG=$(git rev-parse "$MERGE_BASE:.circleci/docker")
+ # If no image exists but the hash is the same as the previous hash then we should error out here
+ if [[ "${PREVIOUS_DOCKER_TAG}" = "${DOCKER_TAG}" ]]; then
+ echo "ERROR: Something has gone wrong and the previous image isn't available for the merge-base of your branch"
+ echo " contact the PyTorch team to restore the original images"
+ exit 1
+ fi
+ echo ::set-output name=rebuild::yes
+ - name: Build and push docker image
+ if: steps.check.outputs.rebuild
+ env:
+ DOCKER_TAG: ${{ steps.calculate-tag.outputs.docker_tag }}
+ DOCKER_SKIP_S3_UPLOAD: 1
+ run: |
+ export IMAGE_NAME=${DOCKER_IMAGE_BASE#308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/}
+ cd .circleci/docker && ./build_docker.sh
+
+ build:
+ runs-on: linux.2xlarge
+ needs: calculate-docker-image
+ env:
+ DOCKER_IMAGE: ${{ needs.calculate-docker-image.outputs.docker_image }}
+ JOB_BASE_NAME: pytorch-linux-bionic-cuda10.2-cudnn7-py3.9-gcc7-build
+ steps:
+ - name: Log in to ECR
+ run: |
+ aws ecr get-login --no-include-email --region us-east-1 > /tmp/ecr-login.sh
+ bash /tmp/ecr-login.sh
+ rm /tmp/ecr-login.sh
+ - name: Chown workspace
+ run: |
+ # Ensure the working directory gets chowned back to the current user
+ docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" .
+ - name: Checkout PyTorch
+ uses: actions/checkout@v2
+ with:
+ fetch-depth: 0 # deep clone, to allow sharding to use git rev-list
+ submodules: recursive
+ - name: Pull docker image
+ run: |
+ docker pull "${DOCKER_IMAGE}"
+ - name: Preserve github env variables for use in docker
+ run: |
+ env | grep '^GITHUB' > "/tmp/github_env_${GITHUB_RUN_ID}"
+ - name: Build PyTorch
+ run: |
+ docker run \
+ -e BUILD_ENVIRONMENT \
+ -e JOB_BASE_NAME \
+ -e MAX_JOBS="$(nproc --ignore=2)" \
+ -e SCCACHE_BUCKET \
+ -e CUSTOM_TEST_ARTIFACT_BUILD_DIR \
+ -e SKIP_SCCACHE_INITIALIZATION=1 \
+ -e TORCH_CUDA_ARCH_LIST \
+ --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \
+ --security-opt seccomp=unconfined \
+ --cap-add=SYS_PTRACE \
+ --tty \
+ --user jenkins \
+ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \
+ -w /var/lib/jenkins/workspace \
+ "${DOCKER_IMAGE}" \
+ sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh'
+ - name: Display and upload binary build size statistics (Click Me)
+ # temporary hack: set CIRCLE_* vars, until we update
+ # tools/stats/print_test_stats.py to natively support GitHub Actions
+ env:
+ SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
+ CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
+ CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
+ CIRCLE_SHA1: ${{ github.event.pull_request.head.sha || github.sha }}
+ CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
+ CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
+ run: |
+ export PYTHONPATH=$PWD
+ COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
+ export COMMIT_TIME
+ pip3 install requests
+ python3 .circleci/scripts/upload_binary_size_to_scuba.py || exit 0
+ - name: Chown workspace
+ run: |
+ # Ensure the working directory gets chowned back to the current user
+ docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" .
+ - name: Archive artifacts into zip
+ run: |
+ zip -r artifacts.zip dist/ build/ .pytorch-test-times.json
+ # Upload to github so that people can click and download artifacts
+ - uses: actions/upload-artifact@v2
+ # Don't fail on upload to GH since it's only for user convenience
+ continue-on-error: true
+ name: Store PyTorch Build Artifacts on Github
+ with:
+ name: ${{ env.BUILD_ENVIRONMENT }}
+ retention-days: 14
+ if-no-files-found: error
+ path:
+ artifacts.zip
+ - uses: seemethere/upload-artifact-s3@9d7ceb0ab39c2c88d93ef7792b27425b27d59162
+ name: Store PyTorch Build Artifacts on S3
+ with:
+ name: ${{ env.BUILD_ENVIRONMENT }}
+ retention-days: 14
+ if-no-files-found: error
+ path:
+ artifacts.zip
+ - name: Clean up docker images
+ if: always()
+ run: |
+ # Prune all of the docker images
+ docker system prune -af
+
+ generate-test-matrix:
+ if: ${{ github.repository_owner == 'pytorch' }}
+ runs-on: ubuntu-18.04
+ env:
+ NUM_TEST_SHARDS: 1
+ outputs:
+ matrix: ${{ steps.set-matrix.outputs.matrix }}
+ container:
+ image: python:3.9
+ steps:
+ - name: Clone pytorch/pytorch
+ uses: actions/checkout@v2
+ - name: Generating test matrix
+ id: set-matrix
+ run: |
+ # outputting for debugging purposes
+ MATRIX=$(python .github/scripts/generate_pytorch_test_matrix.py)
+ echo "${MATRIX}"
+ echo "::set-output name=matrix::${MATRIX}"
+
+ test:
+ runs-on: linux.8xlarge.nvidia.gpu
+ needs:
+ - calculate-docker-image
+ - build
+ - generate-test-matrix
+ strategy:
+ matrix: ${{ fromJson(needs.generate-test-matrix.outputs.matrix) }}
+ fail-fast: false
+ env:
+ DOCKER_IMAGE: ${{ needs.calculate-docker-image.outputs.docker_image }}
+ JOB_BASE_NAME: pytorch-linux-bionic-cuda10.2-cudnn7-py3.9-gcc7-test
+ NUM_TEST_SHARDS: 1
+ TEST_CONFIG: ${{ matrix.test_config }}
+ steps:
+ - name: Log in to ECR
+ run: |
+ aws ecr get-login --no-include-email --region us-east-1 > /tmp/ecr-login.sh
+ bash /tmp/ecr-login.sh
+ rm /tmp/ecr-login.sh
+ - name: Chown workspace
+ run: |
+ # Ensure the working directory gets chowned back to the current user
+ docker run --rm -v "$(pwd)/../":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" .
+ - name: Checkout PyTorch
+ uses: actions/checkout@v2
+ with:
+ submodules: recursive
+ - name: Pull docker image
+ run: |
+ docker pull "${DOCKER_IMAGE}"
+ - name: Install nvidia driver, nvidia-docker runtime, set GPU_FLAG
+ if: ${{ contains(env.BUILD_ENVIRONMENT, 'cuda') }}
+ run: |
+ bash .github/scripts/install_nvidia_utils_linux.sh
+ echo "GPU_FLAG=--gpus all" >> "${GITHUB_ENV}"
+ - name: Determine shm-size
+ run: |
+ shm_size="1g"
+ case "${BUILD_ENVIRONMENT}" in
+ *cuda*)
+ shm_size="2g"
+ ;;
+ *rocm*)
+ shm_size="8g"
+ ;;
+ esac
+ echo "SHM_SIZE=${shm_size}" >> "${GITHUB_ENV}"
+ - uses: seemethere/download-artifact-s3@0504774707cbc8603d7dca922e8026eb8bf3b47b
+ name: Download PyTorch Build Artifacts
+ with:
+ name: ${{ env.BUILD_ENVIRONMENT }}
+ - name: Unzip artifacts
+ run: |
+ unzip -o artifacts.zip
+ - name: Output disk space left
+ run: |
+ sudo df -H
+ - name: Preserve github env variables for use in docker
+ run: |
+ env | grep '^GITHUB' > "/tmp/github_env_${GITHUB_RUN_ID}"
+ - name: Test PyTorch
+ run: |
+ if [[ $NUM_TEST_SHARDS -eq 2 ]]; then
+ export SHARD_NUMBER=$TEST_CONFIG
+ else
+ export SHARD_NUMBER=0
+ fi
+ # TODO: Stop building test binaries as part of the build phase
+ # Used for GPU_FLAG since that doesn't play nice
+ # shellcheck disable=SC2086
+ docker run \
+ ${GPU_FLAG:-} \
+ -e BUILD_ENVIRONMENT \
+ -e CUSTOM_TEST_ARTIFACT_BUILD_DIR \
+ -e GITHUB_ACTIONS \
+ -e IN_CI \
+ -e SHARD_NUMBER \
+ -e JOB_BASE_NAME \
+ -e MAX_JOBS="$(nproc --ignore=2)" \
+ --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \
+ --security-opt seccomp=unconfined \
+ --cap-add=SYS_PTRACE \
+ --shm-size="${SHM_SIZE}" \
+ --tty \
+ --user jenkins \
+ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \
+ -w /var/lib/jenkins/workspace \
+ "${DOCKER_IMAGE}" \
+ sh -c 'sudo chown -R jenkins . && pip install dist/*.whl && .jenkins/pytorch/test.sh'
+ - name: Chown workspace
+ if: always()
+ run: |
+ # Ensure the working directory gets chowned back to the current user
+ docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" .
+ - name: Zip test reports for upload
+ if: always()
+ run: |
+ # Remove any previous test reports if they exist
+ rm -f test-reports-*.zip
+ zip -r "test-reports-${TEST_CONFIG}.zip" test -i '*.xml'
+ - uses: actions/upload-artifact@v2
+ name: Store PyTorch Test Reports
+ if: always()
+ with:
+ name: test-reports
+ retention-days: 14
+ if-no-files-found: error
+ path:
+ test-reports-*.zip
+ - name: Clean up docker images
+ if: always()
+ run: |
+ # Ensure the working directory gets chowned back to the current user
+ docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" .
+ # Prune all of the docker images
+ docker system prune -af
+
+ # this is a separate step from test because the log files from test are too
+ # long: basically, GitHub tries to render all of the log files when you click
+ # through an action causing extreme slowdown on actions that contain too many
+ # logs (like test); we can always move it back to the other one, but it
+ # doesn't create the best experience
+ render_test_results:
+ if: always()
+ needs:
+ - test
+ runs-on: ubuntu-18.04
+ steps:
+ - name: Checkout PyTorch
+ uses: actions/checkout@v2
+ with:
+ # deep clone, to allow tools/stats/print_test_stats.py to use Git commands
+ fetch-depth: 0
+ - uses: actions/download-artifact@v2
+ name: Download PyTorch Test Reports
+ with:
+ name: test-reports
+ path: .
+ - name: Unzip test reports
+ run: |
+ # Should preserve paths so reports should still be in test/test-reports
+ unzip -o 'test-reports-*.zip'
+ - uses: actions/setup-python@v2
+ with:
+ python-version: 3.9
+ - name: Install dependencies
+ # boto3 version copied from .circleci/docker/common/install_conda.sh
+ run: |
+ pip install -r requirements.txt
+ pip install boto3==1.16.34 junitparser rich
+ - name: Output Test Results (Click Me)
+ run: |
+ python tools/render_junit.py test
+ - name: Parse ref
+ id: parse-ref
+ run: .github/scripts/parse_ref.py
+ - name: Display and upload test statistics (Click Me)
+ # temporary hack: set CIRCLE_* vars, until we update
+ # tools/stats/print_test_stats.py to natively support GitHub Actions
+ env:
+ SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
+ CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
+ JOB_BASE_NAME: pytorch-linux-bionic-cuda10.2-cudnn7-py3.9-gcc7-test
+ CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
+ CIRCLE_SHA1: ${{ github.event.pull_request.head.sha || github.sha }}
+ CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
+ CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
+ run: |
+ export PYTHONPATH=$PWD
+ python tools/stats/print_test_stats.py --upload-to-s3 --compare-with-s3 test
diff --git a/.jenkins/pytorch/test.sh b/.jenkins/pytorch/test.sh
index 307687d..79ebdc9 100755
--- a/.jenkins/pytorch/test.sh
+++ b/.jenkins/pytorch/test.sh
@@ -265,7 +265,9 @@
build/bin/TCPStoreTest --gtest_output=xml:$TEST_REPORTS_DIR/TCPStoreTest.xml
MPIEXEC=$(command -v mpiexec)
- if [[ -n "$MPIEXEC" ]]; then
+ # TODO: this is disabled on GitHub Actions until this issue is resolved
+ # https://github.com/pytorch/pytorch/issues/60756
+ if [[ -n "$MPIEXEC" ]] && [[ -z "$GITHUB_ACTIONS" ]]; then
MPICMD="${MPIEXEC} -np 2 build/bin/ProcessGroupMPITest"
eval "$MPICMD"
fi
