| # @generated DO NOT EDIT MANUALLY | |
| # Template is at: .github/templates/linux_ci_workflow.yml.j2 | |
| # Generation script: .github/scripts/generate_ci_workflows.py | |
| name: linux-docs-push | |
| on: | |
| pull_request: | |
| types: [opened, synchronize, reopened, unassigned] | |
| schedule: | |
| - cron: 0 0 * * * | |
| workflow_dispatch: | |
| env: | |
| BUILD_ENVIRONMENT: linux-docs-push | |
| DOCKER_IMAGE_BASE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.7-gcc5.4 | |
| SCCACHE_BUCKET: ossci-compiler-cache-circleci-v2 | |
| XLA_CLANG_CACHE_S3_BUCKET_NAME: ossci-compiler-clang-cache-circleci-xla | |
| TORCH_CUDA_ARCH_LIST: 5.2 | |
| IN_CI: 1 | |
| IS_GHA: 1 | |
| # This is used for the phase of adding wheel tests only, will be removed once completed | |
| IN_WHEEL_TEST: 1 | |
| # Used for custom_opertor, jit_hooks, custom_backend, see .jenkins/pytorch/build.sh | |
| CUSTOM_TEST_ARTIFACT_BUILD_DIR: build/custom_test_artifacts | |
| ALPINE_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/alpine" | |
| PR_LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| AWS_DEFAULT_REGION: us-east-1 | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| SHA1: ${{ github.event.pull_request.head.sha || github.sha }} | |
| PYTORCH_RETRY_TEST_CASES: 1 | |
| concurrency: | |
| group: linux-docs-push-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }} | |
| cancel-in-progress: true | |
| jobs: | |
| build: | |
| runs-on: linux.2xlarge | |
| timeout-minutes: 240 | |
| if: ${{ (github.repository == 'pytorch/pytorch') && ( | |
| (github.event_name == 'push') || | |
| (github.event_name == 'schedule') || | |
| (contains(github.event.pull_request.labels.*.name, 'ciflow/all') || contains(github.event.pull_request.labels.*.name, 'ciflow/cpu') || contains(github.event.pull_request.labels.*.name, 'ciflow/linux') || contains(github.event.pull_request.labels.*.name, 'ciflow/scheduled')) || | |
| (false)) | |
| }} | |
| env: | |
| JOB_BASE_NAME: linux-docs-push-build | |
| IS_PROBOT_TRIGGER_EVENT: ${{ (github.event.action == 'unassigned') && (github.event.assigneed.login == 'pytorchbot') }} | |
| LABEL_CONDITIONS: ${{ contains(github.event.pull_request.labels.*.name, 'ciflow/all') || contains(github.event.pull_request.labels.*.name, 'ciflow/cpu') || contains(github.event.pull_request.labels.*.name, 'ciflow/linux') || contains(github.event.pull_request.labels.*.name, 'ciflow/scheduled') }} | |
| outputs: | |
| docker_image: ${{ steps.calculate-tag.outputs.docker_image }} | |
| steps: | |
| - name: print labels | |
| run: echo "${PR_LABELS}" | |
| - name: Display EC2 information | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| function get_ec2_metadata() { | |
| # Pulled from instance metadata endpoint for EC2 | |
| # see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html | |
| category=$1 | |
| curl -fsSL "http://169.254.169.254/latest/meta-data/${category}" | |
| } | |
| echo "ami-id: $(get_ec2_metadata ami-id)" | |
| echo "instance-id: $(get_ec2_metadata instance-id)" | |
| echo "instance-type: $(get_ec2_metadata instance-type)" | |
| - name: Log in to ECR | |
| env: | |
| AWS_RETRY_MODE: standard | |
| AWS_MAX_ATTEMPTS: 5 | |
| run: | | |
| AWS_ACCOUNT_ID=$(aws sts get-caller-identity|grep Account|cut -f4 -d\") | |
| aws ecr get-login-password --region "$AWS_DEFAULT_REGION" | docker login --username AWS \ | |
| --password-stdin "$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com" | |
| - name: Chown workspace | |
| run: | | |
| retry () { | |
| "$@" || (sleep 1 && "$@") || (sleep 2 && "$@") | |
| } | |
| retry docker pull "${ALPINE_IMAGE}" | |
| # Ensure the working directory gets chowned back to the current user | |
| docker run --pull=never --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" . | |
| - name: Clean workspace | |
| run: | | |
| rm -rf "${GITHUB_WORKSPACE:?}/*" | |
| rm -f ~/.ssh/authorized_keys | |
| - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)" | |
| uses: seemethere/add-github-ssh-key@v1 | |
| with: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Preserve github env variables for use in docker | |
| run: | | |
| env | grep '^GITHUB' > "/tmp/github_env_${GITHUB_RUN_ID}" | |
| - name: Checkout PyTorch | |
| uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9 | |
| with: | |
| # deep clone, to allow use of git merge-base | |
| fetch-depth: 0 | |
| submodules: recursive | |
| - name: Clean PyTorch checkout | |
| run: | | |
| # Remove any artifacts from the previous checkouts | |
| git clean -fxd | |
| - name: Calculate docker image tag | |
| id: calculate-tag | |
| run: | | |
| DOCKER_TAG=$(git rev-parse HEAD:.circleci/docker) | |
| echo "DOCKER_TAG=${DOCKER_TAG}" >> "${GITHUB_ENV}" | |
| echo "DOCKER_IMAGE=${DOCKER_IMAGE_BASE}:${DOCKER_TAG}" >> "${GITHUB_ENV}" | |
| echo "::set-output name=docker_tag::${DOCKER_TAG}" | |
| echo "::set-output name=docker_image::${DOCKER_IMAGE_BASE}:${DOCKER_TAG}" | |
| - name: Check if image should be built | |
| id: check | |
| env: | |
| BASE_REVISION: ${{ github.event.pull_request.base.sha || github.sha }} | |
| run: | | |
| set -x | |
| # Check if image already exists, if it does then skip building it | |
| if docker manifest inspect "${DOCKER_IMAGE_BASE}:${DOCKER_TAG}"; then | |
| exit 0 | |
| fi | |
| if [[ "$BASE_REVISION" = "$(git rev-parse HEAD)" ]]; then | |
| # if we're on the base branch then use the parent commit | |
| MERGE_BASE=$(git rev-parse HEAD~) | |
| else | |
| # otherwise we're on a PR, so use the most recent base commit | |
| MERGE_BASE=$(git merge-base HEAD "$BASE_REVISION") | |
| fi | |
| # Covers the case where a previous tag doesn't exist for the tree | |
| # this is only really applicable on trees that don't have `.circleci/docker` at its merge base, i.e. nightly | |
| if ! git rev-parse "$MERGE_BASE:.circleci/docker"; then | |
| echo "Directory '.circleci/docker' not found in commit $MERGE_BASE, you should probably rebase onto a more recent commit" | |
| exit 1 | |
| fi | |
| PREVIOUS_DOCKER_TAG=$(git rev-parse "$MERGE_BASE:.circleci/docker") | |
| # If no image exists but the hash is the same as the previous hash then we should error out here | |
| if [[ "${PREVIOUS_DOCKER_TAG}" = "${DOCKER_TAG}" ]]; then | |
| echo "ERROR: Something has gone wrong and the previous image isn't available for the merge-base of your branch" | |
| echo " contact the PyTorch team to restore the original images" | |
| exit 1 | |
| fi | |
| echo ::set-output name=rebuild::yes | |
| - name: Build and push docker image | |
| if: ${{ steps.check.outputs.rebuild }} | |
| env: | |
| DOCKER_SKIP_S3_UPLOAD: 1 | |
| working-directory: .circleci/docker | |
| run: | | |
| export IMAGE_NAME=${DOCKER_IMAGE_BASE#308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/} | |
| ./build_docker.sh | |
| - name: Pull Docker image | |
| run: | | |
| retry () { | |
| "$@" || (sleep 1 && "$@") || (sleep 2 && "$@") | |
| } | |
| retry docker pull "${DOCKER_IMAGE}" | |
| - name: Parse ref | |
| id: parse-ref | |
| run: .github/scripts/parse_ref.py | |
| - name: Build | |
| env: | |
| BRANCH: ${{ steps.parse-ref.outputs.branch }} | |
| run: | | |
| # detached container should get cleaned up by teardown_ec2_linux | |
| container_name=$(docker run \ | |
| -e BUILD_ENVIRONMENT \ | |
| -e JOB_BASE_NAME \ | |
| -e MAX_JOBS="$(nproc --ignore=2)" \ | |
| -e AWS_DEFAULT_REGION \ | |
| -e IS_GHA \ | |
| -e PR_NUMBER \ | |
| -e SHA1 \ | |
| -e BRANCH \ | |
| -e GITHUB_RUN_ID \ | |
| -e SCCACHE_BUCKET \ | |
| -e XLA_CLANG_CACHE_S3_BUCKET_NAME \ | |
| -e CUSTOM_TEST_ARTIFACT_BUILD_DIR \ | |
| -e SKIP_SCCACHE_INITIALIZATION=1 \ | |
| -e TORCH_CUDA_ARCH_LIST \ | |
| -e PR_LABELS \ | |
| -e http_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e https_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e no_proxy="localhost,127.0.0.1,github.com,amazonaws.com,s3.amazonaws.com,169.254.169.254,169.254.170.2,/var/run/docker.sock" \ | |
| --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \ | |
| --security-opt seccomp=unconfined \ | |
| --cap-add=SYS_PTRACE \ | |
| --tty \ | |
| --detach \ | |
| --user jenkins \ | |
| -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ | |
| -w /var/lib/jenkins/workspace \ | |
| "${DOCKER_IMAGE}" | |
| ) | |
| docker exec -t "${container_name}" sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' | |
| - name: Display and upload binary build size statistics (Click Me) | |
| # temporary hack: set CIRCLE_* vars, until we update | |
| # tools/stats/print_test_stats.py to natively support GitHub Actions | |
| env: | |
| SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} | |
| BRANCH: ${{ steps.parse-ref.outputs.branch }} | |
| TAG: ${{ steps.parse-ref.outputs.tag }} | |
| WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}' | |
| run: | | |
| COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) | |
| export COMMIT_TIME | |
| pip3 install requests==2.26 boto3==1.16.34 | |
| python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 | |
| - name: Chown workspace | |
| run: | | |
| # Ensure the working directory gets chowned back to the current user | |
| docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" . | |
| - name: Archive artifacts into zip | |
| run: | | |
| zip -1 -r artifacts.zip dist/ build/custom_test_artifacts build/lib build/bin .pytorch-test-times.json | |
| - uses: seemethere/upload-artifact-s3@v3 | |
| name: Store PyTorch Build Artifacts on S3 | |
| with: | |
| name: ${{ env.BUILD_ENVIRONMENT }} | |
| retention-days: 14 | |
| if-no-files-found: error | |
| path: | |
| artifacts.zip | |
| - name: Hold runner for 2 hours or until ssh sessions have drained | |
| # Always hold for active ssh sessions | |
| if: always() | |
| run: .github/scripts/wait_for_ssh_to_drain.sh | |
| - name: Chown workspace | |
| if: always() | |
| run: | | |
| # Ensure the working directory gets chowned back to the current user | |
| docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" . | |
| - name: Kill containers, clean up images | |
| if: always() | |
| run: | | |
| # ignore expansion of "docker ps -q" since it could be empty | |
| # shellcheck disable=SC2046 | |
| docker stop $(docker ps -q) || true | |
| # Prune all of the docker images | |
| docker system prune -af | |
| - name: Hold runner for 2 hours or until ssh sessions have drained | |
| # Always hold for active ssh sessions | |
| if: always() | |
| run: .github/scripts/wait_for_ssh_to_drain.sh | |
| - name: Clean up docker images | |
| if: always() | |
| run: | | |
| # Prune all of the docker images | |
| docker system prune -af | |
| build-docs: | |
| runs-on: linux.2xlarge | |
| timeout-minutes: 240 | |
| strategy: | |
| matrix: | |
| docs_type: [cpp, python] | |
| needs: [build] | |
| env: | |
| DOCKER_IMAGE: ${{ needs.build.outputs.docker_image }} | |
| DOCS_TYPE: ${{ matrix.docs_type }} | |
| WITH_PUSH: ${{ github.event_name == 'schedule' }} | |
| steps: | |
| - name: Display EC2 information | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| function get_ec2_metadata() { | |
| # Pulled from instance metadata endpoint for EC2 | |
| # see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html | |
| category=$1 | |
| curl -fsSL "http://169.254.169.254/latest/meta-data/${category}" | |
| } | |
| echo "ami-id: $(get_ec2_metadata ami-id)" | |
| echo "instance-id: $(get_ec2_metadata instance-id)" | |
| echo "instance-type: $(get_ec2_metadata instance-type)" | |
| - name: Log in to ECR | |
| env: | |
| AWS_RETRY_MODE: standard | |
| AWS_MAX_ATTEMPTS: 5 | |
| run: | | |
| AWS_ACCOUNT_ID=$(aws sts get-caller-identity|grep Account|cut -f4 -d\") | |
| aws ecr get-login-password --region "$AWS_DEFAULT_REGION" | docker login --username AWS \ | |
| --password-stdin "$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com" | |
| - name: Chown workspace | |
| run: | | |
| retry () { | |
| "$@" || (sleep 1 && "$@") || (sleep 2 && "$@") | |
| } | |
| retry docker pull "${ALPINE_IMAGE}" | |
| # Ensure the working directory gets chowned back to the current user | |
| docker run --pull=never --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" . | |
| - name: Clean workspace | |
| run: | | |
| rm -rf "${GITHUB_WORKSPACE:?}/*" | |
| rm -f ~/.ssh/authorized_keys | |
| - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)" | |
| uses: seemethere/add-github-ssh-key@v1 | |
| with: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Preserve github env variables for use in docker | |
| run: | | |
| env | grep '^GITHUB' > "/tmp/github_env_${GITHUB_RUN_ID}" | |
| - name: Checkout PyTorch | |
| uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9 | |
| with: | |
| # deep clone, to allow use of git merge-base | |
| fetch-depth: 0 | |
| submodules: recursive | |
| - name: Clean PyTorch checkout | |
| run: | | |
| # Remove any artifacts from the previous checkouts | |
| git clean -fxd | |
| - name: Pull Docker image | |
| run: | | |
| retry () { | |
| "$@" || (sleep 1 && "$@") || (sleep 2 && "$@") | |
| } | |
| retry docker pull "${DOCKER_IMAGE}" | |
| - uses: seemethere/download-artifact-s3@0504774707cbc8603d7dca922e8026eb8bf3b47b | |
| name: Download PyTorch Build Artifacts | |
| with: | |
| name: ${{ env.BUILD_ENVIRONMENT }} | |
| - name: Unzip artifacts | |
| run: | | |
| unzip -o artifacts.zip | |
| - name: Generate netrc (only for docs-push) | |
| if: ${{ github.event_name == 'schedule' }} | |
| env: | |
| GITHUB_PYTORCHBOT_TOKEN: ${{ secrets.GH_PYTORCHBOT_TOKEN }} | |
| run: | | |
| # set credentials for https pushing | |
| echo "machine github.com" > "${RUNNER_TEMP}/.netrc" | |
| echo "login pytorchbot" >> "${RUNNER_TEMP}/.netrc" | |
| echo "password ${GITHUB_PYTORCHBOT_TOKEN}" >> "${RUNNER_TEMP}/.netrc" | |
| - name: Build ${{ matrix.docs_type }} docs | |
| run: | | |
| set -ex | |
| time docker pull "${DOCKER_IMAGE}" > /dev/null | |
| echo "${GITHUB_REF}" | |
| # TODO: Set it correctly when workflows are scheduled on tags | |
| target="master" | |
| # detached container should get cleaned up by teardown_ec2_linux | |
| container_name=$(docker run \ | |
| -e BUILD_ENVIRONMENT \ | |
| -e CUSTOM_TEST_ARTIFACT_BUILD_DIR \ | |
| -e IN_CI \ | |
| -e MAX_JOBS="$(nproc --ignore=2)" \ | |
| -e SHA1="$GITHUB_SHA" \ | |
| -e DOCS_VERSION="${target}" \ | |
| -e DOCS_TYPE \ | |
| -e PR_LABELS \ | |
| -e WITH_PUSH \ | |
| --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \ | |
| --security-opt seccomp=unconfined \ | |
| --cap-add=SYS_PTRACE \ | |
| --tty \ | |
| --detach \ | |
| --user jenkins \ | |
| -v "${RUNNER_TEMP}/.netrc":/var/lib/jenkins/.netrc \ | |
| -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ | |
| -w /var/lib/jenkins/workspace \ | |
| "${DOCKER_IMAGE}" | |
| ) | |
| docker exec -t "${container_name}" bash -c "sudo chown -R jenkins . && pip install dist/*.whl && ./.circleci/scripts/${DOCS_TYPE}_doc_push_script.sh" | |
| - name: Chown workspace | |
| run: | | |
| # Ensure the working directory gets chowned back to the current user | |
| docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" . | |
| - uses: seemethere/upload-artifact-s3@v3 | |
| name: Upload Python Docs Preview | |
| if: ${{ github.event_name == 'pull_request' && matrix.docs_type == 'python' }} | |
| with: | |
| retention-days: 14 | |
| s3-bucket: doc-previews | |
| if-no-files-found: error | |
| path: pytorch.github.io/docs/master/ | |
| s3-prefix: pytorch/${{ github.event.pull_request.number }} | |
| - uses: seemethere/upload-artifact-s3@v3 | |
| name: Upload C++ Docs Preview | |
| if: ${{ github.event_name == 'pull_request' && matrix.docs_type == 'cpp' }} | |
| with: | |
| retention-days: 14 | |
| if-no-files-found: error | |
| s3-bucket: doc-previews | |
| path: cppdocs/ | |
| s3-prefix: pytorch/${{ github.event.pull_request.number }}/cppdocs |