[ci] inline display_ec2 script
This is purely for forward compatibility reasons. For actions and
workflows, we can pin them to a specific hash in our repo. Doing so
makes the migration smoother, because it means that to do CI we don't
need to reference the local checkout at all (and thus older PRs will not
break even if they are based on a commit that doesn't have the new
workflow files).
The only actually new file we reference is `display_ec2_info.sh`. So for
now, inline it. This incurs a little duplication but is worth it.
Pull Request resolved: https://github.com/pytorch/pytorch/pull/74355
Approved by: https://github.com/janeyx99, https://github.com/seemethere
diff --git a/.github/actions/build-android/action.yml b/.github/actions/build-android/action.yml
index e9a4c4d..3fd6f6a 100644
--- a/.github/actions/build-android/action.yml
+++ b/.github/actions/build-android/action.yml
@@ -6,6 +6,12 @@
arch:
description: arch to build
required: true
+ arch-for-build-env:
+ description: |
+ arch to pass to build environment.
+ This is currently different than the arch name we use elswhere, which
+ should be fixed.
+ required: true
github-secret:
description: github token
required: true
@@ -27,7 +33,7 @@
env:
BRANCH: ${{ inputs.branch }}
JOB_BASE_NAME: ${{ inputs.build-environment }}-build-and-test
- BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-${{ inputs.arch }}-build"
+ BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-${{ inputs.arch-for-build-env }}-build"
AWS_DEFAULT_REGION: us-east-1
PR_NUMBER: ${{ github.event.pull_request.number }}
SHA1: ${{ github.event.pull_request.head.sha || github.sha }}
diff --git a/.github/actions/setup-linux/action.yml b/.github/actions/setup-linux/action.yml
index 4a1bffe..27d6139 100644
--- a/.github/actions/setup-linux/action.yml
+++ b/.github/actions/setup-linux/action.yml
@@ -12,7 +12,18 @@
steps:
- name: Display EC2 information
shell: bash
- run: .github/scripts/display_ec2_info.sh
+ run: |
+ set -euo pipefail
+ function get_ec2_metadata() {
+ # Pulled from instance metadata endpoint for EC2
+ # see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html
+ category=$1
+ curl -fsSL "http://169.254.169.254/latest/meta-data/${category}"
+ }
+ echo "ami-id: $(get_ec2_metadata ami-id)"
+ echo "instance-id: $(get_ec2_metadata instance-id)"
+ echo "instance-type: $(get_ec2_metadata instance-type)"
+ echo "system info $(uname -a)"
- name: Start docker if docker deamon is not running
shell: bash
diff --git a/.github/actions/setup-win/action.yml b/.github/actions/setup-win/action.yml
index 763fa11..a69fa5d 100644
--- a/.github/actions/setup-win/action.yml
+++ b/.github/actions/setup-win/action.yml
@@ -15,7 +15,18 @@
steps:
- name: Display EC2 information
shell: bash
- run: .github/scripts/display_ec2_info.sh
+ run: |
+ set -euo pipefail
+ function get_ec2_metadata() {
+ # Pulled from instance metadata endpoint for EC2
+ # see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html
+ category=$1
+ curl -fsSL "http://169.254.169.254/latest/meta-data/${category}"
+ }
+ echo "ami-id: $(get_ec2_metadata ami-id)"
+ echo "instance-id: $(get_ec2_metadata instance-id)"
+ echo "instance-type: $(get_ec2_metadata instance-type)"
+ echo "system info $(uname -a)"
- name: "[FB EMPLOYEES] Enable SSH (Click me for login details)"
uses: seemethere/add-github-ssh-key@v1
diff --git a/.github/actions/upload-test-artifacts/action.yml b/.github/actions/upload-test-artifacts/action.yml
index 32bb6aa..328f302 100644
--- a/.github/actions/upload-test-artifacts/action.yml
+++ b/.github/actions/upload-test-artifacts/action.yml
@@ -38,6 +38,8 @@
- name: Zip JSONs for upload
if: runner.os == 'Windows'
shell: powershell
+ env:
+ FILE_SUFFIX: ${{ inputs.file-suffix }}
run: |
# -ir => recursive include all files in pattern
7z a "test-jsons-$Env:FILE_SUFFIX.zip" -ir'!test\*.json'
@@ -45,6 +47,8 @@
- name: Zip test reports for upload
if: runner.os == 'Windows'
shell: powershell
+ env:
+ FILE_SUFFIX: ${{ inputs.file-suffix }}
run: |
# -ir => recursive include all files in pattern
7z a "test-reports-$Env:FILE_SUFFIX.zip" -ir'!test\*.xml'
diff --git a/.github/scripts/display_ec2_info.sh b/.github/scripts/display_ec2_info.sh
deleted file mode 100755
index 447d967..0000000
--- a/.github/scripts/display_ec2_info.sh
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-function get_ec2_metadata() {
- # Pulled from instance metadata endpoint for EC2
- # see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html
- category=$1
- curl -fsSL "http://169.254.169.254/latest/meta-data/${category}"
-}
-echo "ami-id: $(get_ec2_metadata ami-id)"
-echo "instance-id: $(get_ec2_metadata instance-id)"
-echo "instance-type: $(get_ec2_metadata instance-type)"
-echo "system info $(uname -a)"
diff --git a/.github/workflows/_android-full-build-test.yml b/.github/workflows/_android-full-build-test.yml
index de15494..45246b7 100644
--- a/.github/workflows/_android-full-build-test.yml
+++ b/.github/workflows/_android-full-build-test.yml
@@ -12,6 +12,23 @@
type: string
description: Name of the base docker image to build with.
+ secrets:
+ SONATYPE_NEXUS_USERNAME:
+ description: nexus user
+ required: true
+ SONATYPE_NEXUS_PASSWORD:
+ description: nexus pass
+ required: true
+ ANDROID_SIGN_KEY:
+ description: android key
+ required: true
+ ANDROID_SIGN_PASS:
+ description: android pass
+ required: true
+ SCRIBE_GRAPHQL_ACCESS_TOKEN:
+ description: token for writing to scribe/scuba
+ required: true
+
env:
IN_CI: 1 # TODO delete in favor of GITHUB_ACTIONS
IS_GHA: 1 # TODO delete in favor of GITHUB_ACTIONS
@@ -55,19 +72,21 @@
id: parse-ref
run: .github/scripts/parse_ref.py
- - name: Build arm_v7a
+ - name: Build arm-v7a
uses: ./.github/actions/build-android
with:
arch: arm_v7a
+ arch-for-build-env: arm-v7a
github-secret: ${{ secrets.GITHUB_TOKEN }}
build-environment: ${{ inputs.build-environment }}
docker-image: ${{ steps.calculate-docker-image.outputs.docker-image }}
branch: ${{ steps.parse-ref.outputs.branch }}
- - name: Build arm_v8a
+ - name: Build arm-v8a
uses: ./.github/actions/build-android
with:
arch: arm_v8a
+ arch-for-build-env: arm-v8a
github-secret: ${{ secrets.GITHUB_TOKEN }}
build-environment: ${{ inputs.build-environment }}
docker-image: ${{ steps.calculate-docker-image.outputs.docker-image }}
@@ -77,6 +96,7 @@
uses: ./.github/actions/build-android
with:
arch: x86_32
+ arch-for-build-env: x86_32
github-secret: ${{ secrets.GITHUB_TOKEN }}
build-environment: ${{ inputs.build-environment }}
docker-image: ${{ steps.calculate-docker-image.outputs.docker-image }}
@@ -86,14 +106,14 @@
uses: ./.github/actions/build-android
with:
arch: x86_64
+ arch-for-build-env: x86_64
github-secret: ${{ secrets.GITHUB_TOKEN }}
build-environment: ${{ inputs.build-environment }}
docker-image: ${{ steps.calculate-docker-image.outputs.docker-image }}
branch: ${{ steps.parse-ref.outputs.branch }}
- # TODO: pretty sure this is wrong, since DOCKER_IMAGE is no longer built on every commit
- # https://github.com/pytorch/pytorch/issues/74073
- name: Build final artifact
+ id: build-final
env:
BRANCH: ${{ steps.parse-ref.outputs.branch }}
DOCKER_IMAGE: ${{ steps.calculate-docker-image.outputs.docker-image }}
@@ -135,6 +155,7 @@
mkdir -p "${GITHUB_WORKSPACE}/build_android_artifacts"
docker cp "${id_x86_32}:/var/lib/jenkins/workspace/android/artifacts.tgz" "${GITHUB_WORKSPACE}/build_android_artifacts/"
+ echo "::set-output name=id_x86_32::${id_x86_32}"
- name: Display and upload binary build size statistics (Click Me)
# temporary hack: set CIRCLE_* vars, until we update
@@ -146,14 +167,15 @@
SHA1: ${{ github.event.pull_request.head.sha || github.sha }}
TAG: ${{ steps.parse-ref.outputs.tag }}
WORKFLOW_ID: ${{ github.run_id }}
- ARTIFACTS: ${GITHUB_WORKSPACE}/build_android_artifacts/artifacts.tgz
ANDROID_BUILD_TYPE: prebuilt
+ SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
run: |
# The artifact file is created inside docker container, which contains the result binaries.
# Now unpackage it into the project folder. The subsequent script will scan project folder
# to locate result binaries and report their sizes.
# If artifact file is not provided it assumes that the project folder has been mounted in
# the docker during build and already contains the result binaries, so this step can be skipped.
+ export ARTIFACTS=${GITHUB_WORKSPACE}/build_android_artifacts/artifacts.tgz
if [ -n "${ARTIFACTS}" ]; then
tar xf "${ARTIFACTS}" -C "${GITHUB_WORKSPACE}"
cd "${GITHUB_WORKSPACE}"
@@ -163,6 +185,26 @@
pip3 install requests==2.26 boto3==1.16.34
python3 -m tools.stats.upload_binary_size_to_scuba "android" || exit 0
+ - name: Publish android snapshot
+ if: ${{ github.event_name == 'push' && github.event.ref == 'refs/heads/nightly' }}
+ env:
+ SONATYPE_NEXUS_USERNAME: ${{ secrets.SONATYPE_NEXUS_USERNAME }}
+ SONATYPE_NEXUS_PASSWORD: ${{ secrets.SONATYPE_NEXUS_PASSWORD }}
+ ANDROID_SIGN_KEY: ${{ secrets.ANDROID_SIGN_KEY }}
+ ANDROID_SIGN_PASS: ${{ secrets.ANDROID_SIGN_PASS }}
+ ID_X86_32: ${{ steps.build-final.outputs.id_x86_32 }}
+ run: |
+ set -eux
+ # shellcheck disable=SC2154
+ (echo "cd workspace && ./.circleci/scripts/publish_android_snapshot.sh" | docker exec \
+ -e BUILD_ENVIRONMENT="pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-publish-snapshot" \
+ -e SONATYPE_NEXUS_USERNAME \
+ -e SONATYPE_NEXUS_PASSWORD \
+ -e ANDROID_SIGN_KEY \
+ -e ANDROID_SIGN_PASS \
+ -e http_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e https_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e no_proxy="localhost,127.0.0.1,github.com,amazonaws.com,s3.amazonaws.com,169.254.169.254,169.254.170.2,/var/run/docker.sock" \
+ -u jenkins -i "${ID_X86_32}" bash) 2>&1
+
- name: Store PyTorch Android Build Artifacts on S3
uses: seemethere/upload-artifact-s3@v3
with:
diff --git a/.github/workflows/_bazel-build-test.yml b/.github/workflows/_bazel-build-test.yml
index 6947574..974304e 100644
--- a/.github/workflows/_bazel-build-test.yml
+++ b/.github/workflows/_bazel-build-test.yml
@@ -142,7 +142,7 @@
uses: ./.github/actions/upload-test-artifacts
if: always()
with:
- file-suffix: $bazel-${{ github.job }}
+ file-suffix: bazel-${{ github.job }}
- name: Upload test statistics
if: always()
