.github/templates/bazel_ci_workflow.yml.j2

{%- extends "linux_ci_workflow.yml.j2" -%}
{%- set exclude_test = true -%}
{% block name -%}
# Template is at:    .github/templates/bazel_ci_workflow.yml.j2
# Generation script: .github/scripts/generate_ci_workflows.py
name: !{{ build_environment }}
{%- endblock %}

on:
{%- if on_pull_request %}
  pull_request:
  {%- if ciflow_config.enabled %}
    {%- if ciflow_config.trigger_action_only %}
    types: [!{{ ciflow_config.trigger_action }}]
    {%- else %}
    types: [opened, synchronize, reopened, !{{ ciflow_config.trigger_action }}]
    {%- endif %}
  {%- endif %}
{%- else %}
  # TODO: Enable pull_request builds when we can verify capacity can be met by auto-scalers
{%- endif %}

{% block build +%}
  # building and testing in a single job since bazel runs only small subset of tests
  build-and-test:
    runs-on: !{{ test_runner_type }}
    needs: [calculate-docker-image, !{{ ciflow_config.root_job_name }}]
    env:
      DOCKER_IMAGE: ${{ needs.calculate-docker-image.outputs.docker_image }}
      JOB_BASE_NAME: !{{ build_environment }}-build-and-test
      NUM_TEST_SHARDS: !{{ num_test_shards }}
      CONTINUE_THROUGH_ERROR: ${{ github.repository_owner == 'pytorch' && (github.event_name == 'push' || github.event_name == 'schedule') }}
    steps:
      !{{ common.setup_ec2_linux() }}
      !{{ common.checkout_pytorch("recursive") }}
      - name: Pull docker image
        run: |
          docker pull "${DOCKER_IMAGE}"
      - name: Determine shm-size
        run: |
          shm_size="1g"
          case "${BUILD_ENVIRONMENT}" in
            *cuda*)
              shm_size="2g"
              ;;
            *rocm*)
              shm_size="8g"
              ;;
          esac
          echo "SHM_SIZE=${shm_size}" >> "${GITHUB_ENV}"
      - name: Output disk space left
        run: |
          sudo df -H
      - name: Preserve github env variables for use in docker
        run: |
          env | grep '^GITHUB' > "/tmp/github_env_${GITHUB_RUN_ID}"
      - name: Build PyTorch
        run: |
          docker run \
            -e BUILD_ENVIRONMENT \
            -e JOB_BASE_NAME \
            -e MAX_JOBS="$(nproc --ignore=2)" \
            -e SCCACHE_BUCKET \
            -e CUSTOM_TEST_ARTIFACT_BUILD_DIR \
            -e PR_LABELS \
            -e SKIP_SCCACHE_INITIALIZATION=1 \
            -e TORCH_CUDA_ARCH_LIST \
            -e http_proxy="!{{ common.squid_proxy }}" -e https_proxy="!{{ common.squid_proxy }}" -e no_proxy="!{{ common.squid_no_proxy }}" \
            --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \
            --security-opt seccomp=unconfined \
            --cap-add=SYS_PTRACE \
            --tty \
            --user jenkins \
            -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \
            -w /var/lib/jenkins/workspace \
            "${DOCKER_IMAGE}" \
            sh -c 'sudo chown -R jenkins . && sudo chown -R jenkins /dev && .jenkins/pytorch/build.sh'
      !{{ common.parse_ref() }}
      - name: Display and upload binary build size statistics (Click Me)
        # temporary hack: set CIRCLE_* vars, until we update
        # tools/stats/print_test_stats.py to natively support GitHub Actions
        env:
          AWS_DEFAULT_REGION: us-east-1
          SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
          CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
          CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
          CIRCLE_SHA1: ${{ github.event.pull_request.head.sha || github.sha }}
          CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
          CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
        run: |
          COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
          export COMMIT_TIME
          pip3 install requests==2.26
          python3 -m tools.stats.upload_binary_size_to_scuba || exit 0
      - name: Test PyTorch
        run: |
          export SHARD_NUMBER=0
          # TODO: Stop building test binaries as part of the build phase
          # Used for GPU_FLAG since that doesn't play nice
          # shellcheck disable=SC2086
          # Make sure we copy test results from bazel-testlogs symlink to
          # a regular directory ./test/test-reports
          docker run \
            ${GPU_FLAG:-} \
            -e BUILD_ENVIRONMENT \
            -e CUSTOM_TEST_ARTIFACT_BUILD_DIR \
            -e GITHUB_ACTIONS \
            -e IN_CI \
            -e SHARD_NUMBER \
            -e JOB_BASE_NAME \
            -e MAX_JOBS="$(nproc --ignore=2)" \
            -e SCCACHE_BUCKET \
            -e CONTINUE_THROUGH_ERROR \
            -e PR_LABELS \
            -e http_proxy="!{{ common.squid_proxy }}" -e https_proxy="!{{ common.squid_proxy }}" -e no_proxy="!{{ common.squid_no_proxy }}" \
            --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \
            --security-opt seccomp=unconfined \
            --cap-add=SYS_PTRACE \
            --shm-size="${SHM_SIZE}" \
            --tty \
            --user jenkins \
            -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \
            -w /var/lib/jenkins/workspace \
            "${DOCKER_IMAGE}" \
            sh -c 'sudo chown -R jenkins . && sudo chown -R jenkins /dev && .jenkins/pytorch/test.sh && cp -Lr ./bazel-testlogs ./test/test-reports'
      - name: Chown workspace
        if: always()
        run: |
          # Ensure the working directory gets chowned back to the current user
          docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" .
      - name: Zip test reports for upload
        if: always()
        env:
          COMMIT_SHA1: ${{ github.event.pull_request.head.sha || github.sha }}
          WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
        run: |
          # Remove any previous test reports if they exist
          rm -f test-reports-*.zip
          zip -r "test-reports-${COMMIT_SHA1}-${WORKFLOW_ID}.zip" test -i '*.xml'
      - uses: actions/upload-artifact@v2
        name: Store PyTorch Test Reports
        if: always()
        with:
          name: test-reports
          retention-days: 14
          if-no-files-found: error
          path:
            test-reports-*.zip
      !{{ common.upload_test_statistics(build_environment) }}
      !{{ common.teardown_ec2_linux() }}
{%- endblock %}