.github: Migrate macOS workflows to GHA (#67717)
Summary:
Pull Request resolved: https://github.com/pytorch/pytorch/pull/67717
Signed-off-by: Eli Uriegas <eliuriegas@fb.com>
Test Plan: Imported from OSS
Reviewed By: malfet
Differential Revision: D32287733
Pulled By: seemethere
fbshipit-source-id: 8df6b20aada818ad39895ef87dc280098e09707b
diff --git a/.circleci/config.yml b/.circleci/config.yml
index ba55c33..128a9f0 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -6544,18 +6544,6 @@
build_environment: "pytorch-xla-linux-bionic-py3.6-clang9-test"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-py3.6-clang9"
resource_class: large
- - pytorch_macos_10_15_py3_build:
- name: pytorch_macos_10_15_py3_build
- - pytorch_macos_10_13_py3_build:
- name: pytorch_macos_10_13_py3_build
- - pytorch_macos_10_13_py3_test:
- name: pytorch_macos_10_13_py3_test
- requires:
- - pytorch_macos_10_13_py3_build
- - pytorch_macos_10_13_py3_lite_interpreter_build_test:
- name: pytorch_macos_10_13_py3_lite_interpreter_build_test
- requires:
- - pytorch_macos_10_13_py3_build
- pytorch_linux_build:
build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_32-build"
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
diff --git a/.circleci/generate_config_yml.py b/.circleci/generate_config_yml.py
index 4ef6e17..6e403dc 100755
--- a/.circleci/generate_config_yml.py
+++ b/.circleci/generate_config_yml.py
@@ -15,7 +15,6 @@
import cimodel.data.simple.android_definitions
import cimodel.data.simple.binary_smoketest
import cimodel.data.simple.docker_definitions
-import cimodel.data.simple.macos_definitions
import cimodel.data.simple.mobile_definitions
import cimodel.data.simple.nightly_android
import cimodel.data.simple.nightly_ios
@@ -140,7 +139,6 @@
def gen_build_workflows_tree():
build_workflows_functions = [
pytorch_build_definitions.get_workflow_jobs,
- cimodel.data.simple.macos_definitions.get_workflow_jobs,
cimodel.data.simple.android_definitions.get_workflow_jobs,
cimodel.data.simple.mobile_definitions.get_workflow_jobs,
cimodel.data.simple.binary_smoketest.get_workflow_jobs,
diff --git a/.github/generated-ciflow-ruleset.json b/.github/generated-ciflow-ruleset.json
index e763f90..f7dc7dd 100644
--- a/.github/generated-ciflow-ruleset.json
+++ b/.github/generated-ciflow-ruleset.json
@@ -27,6 +27,9 @@
"linux-xenial-py3.6-gcc5.4",
"linux-xenial-py3.6-gcc7",
"linux-xenial-py3.6-gcc7-bazel-test",
+ "macos-10-15-py3-arm64",
+ "macos-10-15-py3-lite-interpreter-x86-64",
+ "macos-10-15-py3-x86-64",
"parallelnative-linux-xenial-py3.6-gcc5.4",
"periodic-libtorch-linux-xenial-cuda11.1-py3.6-gcc7",
"periodic-linux-xenial-cuda10.2-py3-gcc7-slow-gradcheck",
@@ -133,7 +136,10 @@
"ios-12-5-1-arm64-metal",
"ios-12-5-1-x86-64",
"ios-12-5-1-x86-64-coreml",
- "ios-12-5-1-x86-64-full-jit"
+ "ios-12-5-1-x86-64-full-jit",
+ "macos-10-15-py3-arm64",
+ "macos-10-15-py3-lite-interpreter-x86-64",
+ "macos-10-15-py3-x86-64"
],
"ciflow/mobile": [
"linux-xenial-py3-clang5-mobile-build",
diff --git a/.github/scripts/generate_ci_workflows.py b/.github/scripts/generate_ci_workflows.py
index 152dde5..b5b498d 100755
--- a/.github/scripts/generate_ci_workflows.py
+++ b/.github/scripts/generate_ci_workflows.py
@@ -32,10 +32,10 @@
LINUX_CUDA_TEST_RUNNER,
}
-MACOS_TEST_RUNNER = "macos-10.15"
+MACOS_TEST_RUNNER_10_15 = "macos-10.15"
MACOS_RUNNERS = {
- MACOS_TEST_RUNNER
+ MACOS_TEST_RUNNER_10_15,
}
CUDA_RUNNERS = {
@@ -163,6 +163,7 @@
num_test_shards_on_pull_request: int = -1
distributed_test: bool = True
timeout_after: int = 240
+ xcode_version: str = ''
# The following variables will be set as environment variables,
# so it's easier for both shell and Python scripts to consume it if false is represented as the empty string.
@@ -571,6 +572,7 @@
CIWorkflow(
arch="macos",
build_environment="ios-12-5-1-arm64",
+ test_runner_type=MACOS_TEST_RUNNER_10_15,
exclude_test=True,
ciflow_config=CIFlowConfig(
labels={LABEL_CIFLOW_IOS, LABEL_CIFLOW_MACOS},
@@ -579,6 +581,7 @@
CIWorkflow(
arch="macos",
build_environment="ios-12-5-1-arm64-coreml",
+ test_runner_type=MACOS_TEST_RUNNER_10_15,
exclude_test=True,
ciflow_config=CIFlowConfig(
labels={LABEL_CIFLOW_IOS, LABEL_CIFLOW_MACOS},
@@ -587,6 +590,7 @@
CIWorkflow(
arch="macos",
build_environment="ios-12-5-1-arm64-full-jit",
+ test_runner_type=MACOS_TEST_RUNNER_10_15,
exclude_test=True,
ciflow_config=CIFlowConfig(
labels={LABEL_CIFLOW_IOS, LABEL_CIFLOW_MACOS},
@@ -595,6 +599,7 @@
CIWorkflow(
arch="macos",
build_environment="ios-12-5-1-arm64-custom-ops",
+ test_runner_type=MACOS_TEST_RUNNER_10_15,
exclude_test=True,
ciflow_config=CIFlowConfig(
labels={LABEL_CIFLOW_IOS, LABEL_CIFLOW_MACOS},
@@ -603,6 +608,7 @@
CIWorkflow(
arch="macos",
build_environment="ios-12-5-1-arm64-metal",
+ test_runner_type=MACOS_TEST_RUNNER_10_15,
exclude_test=True,
ciflow_config=CIFlowConfig(
labels={LABEL_CIFLOW_IOS, LABEL_CIFLOW_MACOS},
@@ -611,6 +617,7 @@
CIWorkflow(
arch="macos",
build_environment="ios-12-5-1-x86-64",
+ test_runner_type=MACOS_TEST_RUNNER_10_15,
exclude_test=True,
ciflow_config=CIFlowConfig(
labels={LABEL_CIFLOW_IOS, LABEL_CIFLOW_MACOS},
@@ -619,6 +626,7 @@
CIWorkflow(
arch="macos",
build_environment="ios-12-5-1-x86-64-coreml",
+ test_runner_type=MACOS_TEST_RUNNER_10_15,
exclude_test=True,
ciflow_config=CIFlowConfig(
labels={LABEL_CIFLOW_IOS, LABEL_CIFLOW_MACOS},
@@ -627,6 +635,7 @@
CIWorkflow(
arch="macos",
build_environment="ios-12-5-1-x86-64-full-jit",
+ test_runner_type=MACOS_TEST_RUNNER_10_15,
exclude_test=True,
ciflow_config=CIFlowConfig(
labels={LABEL_CIFLOW_IOS, LABEL_CIFLOW_MACOS},
@@ -634,6 +643,38 @@
),
]
+MACOS_WORKFLOWS = [
+ CIWorkflow(
+ arch="macos",
+ build_environment="macos-10-15-py3-x86-64",
+ xcode_version="12",
+ test_runner_type=MACOS_TEST_RUNNER_10_15,
+ ciflow_config=CIFlowConfig(
+ labels={LABEL_CIFLOW_MACOS},
+ ),
+ ),
+ CIWorkflow(
+ arch="macos",
+ build_environment="macos-10-15-py3-lite-interpreter-x86-64",
+ xcode_version="12",
+ test_runner_type=MACOS_TEST_RUNNER_10_15,
+ exclude_test=True,
+ build_generates_artifacts=False,
+ ciflow_config=CIFlowConfig(
+ labels={LABEL_CIFLOW_MACOS},
+ ),
+ ),
+ CIWorkflow(
+ arch="macos",
+ build_environment="macos-10-15-py3-arm64",
+ test_runner_type=MACOS_TEST_RUNNER_10_15,
+ exclude_test=True,
+ ciflow_config=CIFlowConfig(
+ labels={LABEL_CIFLOW_MACOS},
+ ),
+ ),
+]
+
DOCKER_IMAGES = {
f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-bionic-cuda10.2-cudnn7-py3.6-clang9", # for pytorch/xla
f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-bionic-rocm4.1-py3.6", # for rocm
@@ -667,6 +708,7 @@
(jinja_env.get_template("windows_ci_workflow.yml.j2"), WINDOWS_WORKFLOWS),
(jinja_env.get_template("bazel_ci_workflow.yml.j2"), BAZEL_WORKFLOWS),
(jinja_env.get_template("ios_ci_workflow.yml.j2"), IOS_WORKFLOWS),
+ (jinja_env.get_template("macos_ci_workflow.yml.j2"), MACOS_WORKFLOWS),
(jinja_env.get_template("docker_builds_ci_workflow.yml.j2"), DOCKER_WORKFLOWS),
(jinja_env.get_template("android_ci_workflow.yml.j2"), ANDROID_WORKFLOWS),
]
diff --git a/.github/templates/common.yml.j2 b/.github/templates/common.yml.j2
index f01e8e1..198fc7c 100644
--- a/.github/templates/common.yml.j2
+++ b/.github/templates/common.yml.j2
@@ -119,7 +119,7 @@
{%- endmacro -%}
-{%- macro upload_test_reports(name) -%}
+{%- macro upload_test_reports(name, artifact_name="", use_s3=True) -%}
- name: Zip test reports for upload
if: always()
env:
@@ -139,10 +139,18 @@
rm -f test-reports-*.zip
zip -r "test-reports-${FILE_SUFFIX}.zip" test -i '*.xml'
{%- endif %}
+{%- if use_s3 %}
- uses: !{{ upload_artifact_s3_action }}
name: Store Test Reports on S3
+{%- else %}
+ - uses: actions/upload-artifact@v2
+ name: Store Test Reports on Github
+{%- endif %}
if: always()
with:
+{%- if artifact_name != "" %}
+ name: !{{ artifact_name }}
+{%- endif %}
retention-days: 14
if-no-files-found: error
path:
@@ -232,3 +240,19 @@
export IMAGE_NAME=${DOCKER_IMAGE_BASE#308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/}
./build_docker.sh
{%- endmacro -%}
+
+{%- macro setup_miniconda(python_version) -%}
+ - name: Setup miniconda
+ uses: conda-incubator/setup-miniconda@v2
+ with:
+ auto-update-conda: true
+ python-version: !{{ python_version }}
+ activate-environment: build
+{%- endmacro -%}
+
+{%- macro set_xcode_version(xcode_version) -%}
+{%- if xcode_version != '' %}
+ # Set xcode xcode version to !{{ xcode_version }}
+ DEVELOPER_DIR: /Applications/Xcode_!{{ xcode_version }}.app/Contents/Developer
+{%- endif %}
+{%- endmacro -%}
diff --git a/.github/templates/ios_ci_workflow.yml.j2 b/.github/templates/ios_ci_workflow.yml.j2
index bd65f27..5f7e1b6 100644
--- a/.github/templates/ios_ci_workflow.yml.j2
+++ b/.github/templates/ios_ci_workflow.yml.j2
@@ -29,6 +29,7 @@
BUILD_ENVIRONMENT: !{{ build_environment }}
IN_CI: 1
IS_GHA: 1
+!{{ common.set_xcode_version(xcode_version) }}
jobs:
!{{ common.ciflow_should_run_job(ciflow_config) }}
@@ -43,12 +44,7 @@
IOS_SIGN_KEY_2022: ${{ secrets.IOS_SIGN_KEY_2022 }}
steps:
!{{ common.checkout_pytorch("recursive") }}
- - name: Setup miniconda
- uses: conda-incubator/setup-miniconda@v2
- with:
- auto-update-conda: true
- python-version: 3.8
- activate-environment: build
+ !{{ common.setup_miniconda("3.8") }}
- name: Install ios / conda Dependencies
run: |
# Install dependencies
diff --git a/.github/templates/macos_ci_workflow.yml.j2 b/.github/templates/macos_ci_workflow.yml.j2
new file mode 100644
index 0000000..ee5b00d
--- /dev/null
+++ b/.github/templates/macos_ci_workflow.yml.j2
@@ -0,0 +1,90 @@
+{% import 'common.yml.j2' as common %}
+
+{%- block name -%}
+# Template is at: .github/templates/macos_ci_workflow.yml.j2
+# Generation script: .github/scripts/generate_ci_workflows.py
+name: !{{ build_environment }}
+{%- endblock %}
+
+on:
+ pull_request:
+ types: [opened, synchronize, reopened, !{{ ciflow_config.trigger_action }}]
+
+{%- if is_scheduled %}
+ schedule:
+ - cron: !{{ is_scheduled }}
+{%- else %}
+ push:
+ branches:
+ - master
+ - release/*
+{%- endif %}
+ workflow_dispatch:
+
+# For setup-miniconda, see https://github.com/conda-incubator/setup-miniconda/issues/179
+defaults:
+ run:
+ shell: bash -e -l {0}
+env:
+ BUILD_ENVIRONMENT: !{{ build_environment }}
+ COMPACT_JOB_NAME: !{{ build_environment }}
+ IN_CI: 1
+ IS_GHA: 1
+!{{ common.set_xcode_version(xcode_version) }}
+
+jobs:
+!{{ common.ciflow_should_run_job(ciflow_config) }}
+{% block build_test +%}
+{%- if exclude_test %}
+ build:
+{%- else %}
+ build-test:
+{%- endif %}
+ runs-on: !{{ test_runner_type }}
+ needs: [!{{ ciflow_config.root_job_name }}]
+ env:
+ JOB_BASE_NAME: !{{ build_environment }}
+ # For sccache access (only on non-forked PRs)
+ AWS_ACCESS_KEY_ID: ${{ secrets.MACOS_SCCACHE_S3_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.MACOS_SCCACHE_S3_SECRET_ACCESS_KEY }}
+ steps:
+ !{{ common.checkout_pytorch("recursive") }}
+ !{{ common.setup_miniconda("3.8") }}
+ - name: Install macOS homebrew dependencies
+ run: |
+ # Install dependencies
+ brew install libomp
+ - name: Install sccache (only for non-forked PRs)
+ if: github.event.pull_request.head.repo.full_name == github.repository
+ run: |
+ sudo curl --retry 3 https://s3.amazonaws.com/ossci-macos/sccache_v2.15 --output /usr/local/bin/sccache
+ sudo chmod +x /usr/local/bin/sccache
+ echo "SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2" >> "${GITHUB_ENV}"
+ - name: Build
+ run: |
+ echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname "$(which conda)")/../"}" >> "${GITHUB_ENV}"
+ .jenkins/pytorch/macos-build.sh
+{%- if build_generates_artifacts %}
+ - name: Archive artifacts into zip
+ run: |
+ zip -1 -r artifacts.zip dist/
+ - uses: actions/upload-artifact@v2
+ name: Store PyTorch Build Artifacts on S3
+ with:
+ name: ${{ env.BUILD_ENVIRONMENT }}
+ retention-days: 14
+ if-no-files-found: error
+ path:
+ artifacts.zip
+{%- endif %}
+{%- if not exclude_test %}
+ - name: Test
+ run: |
+ python -m pip install dist/*.whl
+ .jenkins/pytorch/macos-test.sh
+ !{{ common.render_test_results() }}
+ !{{ common.upload_test_reports("macos", artifact_name="test-reports", use_s3=False) }}
+{%- endif %}
+{% endblock +%}
+
+!{{ common.concurrency(build_environment) }}
diff --git a/.github/workflows/generated-ios-12-5-1-arm64-coreml.yml b/.github/workflows/generated-ios-12-5-1-arm64-coreml.yml
index 52c12f6..dff462e 100644
--- a/.github/workflows/generated-ios-12-5-1-arm64-coreml.yml
+++ b/.github/workflows/generated-ios-12-5-1-arm64-coreml.yml
@@ -21,6 +21,7 @@
IN_CI: 1
IS_GHA: 1
+
jobs:
ciflow_should_run:
diff --git a/.github/workflows/generated-ios-12-5-1-arm64-custom-ops.yml b/.github/workflows/generated-ios-12-5-1-arm64-custom-ops.yml
index f9d4caf..e8e92f5 100644
--- a/.github/workflows/generated-ios-12-5-1-arm64-custom-ops.yml
+++ b/.github/workflows/generated-ios-12-5-1-arm64-custom-ops.yml
@@ -21,6 +21,7 @@
IN_CI: 1
IS_GHA: 1
+
jobs:
ciflow_should_run:
diff --git a/.github/workflows/generated-ios-12-5-1-arm64-full-jit.yml b/.github/workflows/generated-ios-12-5-1-arm64-full-jit.yml
index 2bdd44f..5f7bf72 100644
--- a/.github/workflows/generated-ios-12-5-1-arm64-full-jit.yml
+++ b/.github/workflows/generated-ios-12-5-1-arm64-full-jit.yml
@@ -21,6 +21,7 @@
IN_CI: 1
IS_GHA: 1
+
jobs:
ciflow_should_run:
diff --git a/.github/workflows/generated-ios-12-5-1-arm64-metal.yml b/.github/workflows/generated-ios-12-5-1-arm64-metal.yml
index 9ebdd25..9b7cc68 100644
--- a/.github/workflows/generated-ios-12-5-1-arm64-metal.yml
+++ b/.github/workflows/generated-ios-12-5-1-arm64-metal.yml
@@ -21,6 +21,7 @@
IN_CI: 1
IS_GHA: 1
+
jobs:
ciflow_should_run:
diff --git a/.github/workflows/generated-ios-12-5-1-arm64.yml b/.github/workflows/generated-ios-12-5-1-arm64.yml
index 454c7c2..5b90b25 100644
--- a/.github/workflows/generated-ios-12-5-1-arm64.yml
+++ b/.github/workflows/generated-ios-12-5-1-arm64.yml
@@ -21,6 +21,7 @@
IN_CI: 1
IS_GHA: 1
+
jobs:
ciflow_should_run:
diff --git a/.github/workflows/generated-ios-12-5-1-x86-64-coreml.yml b/.github/workflows/generated-ios-12-5-1-x86-64-coreml.yml
index 6eeec4b..0fbae28 100644
--- a/.github/workflows/generated-ios-12-5-1-x86-64-coreml.yml
+++ b/.github/workflows/generated-ios-12-5-1-x86-64-coreml.yml
@@ -21,6 +21,7 @@
IN_CI: 1
IS_GHA: 1
+
jobs:
ciflow_should_run:
diff --git a/.github/workflows/generated-ios-12-5-1-x86-64-full-jit.yml b/.github/workflows/generated-ios-12-5-1-x86-64-full-jit.yml
index 14e021e..696894b 100644
--- a/.github/workflows/generated-ios-12-5-1-x86-64-full-jit.yml
+++ b/.github/workflows/generated-ios-12-5-1-x86-64-full-jit.yml
@@ -21,6 +21,7 @@
IN_CI: 1
IS_GHA: 1
+
jobs:
ciflow_should_run:
diff --git a/.github/workflows/generated-ios-12-5-1-x86-64.yml b/.github/workflows/generated-ios-12-5-1-x86-64.yml
index 969daf2..77d9bcb 100644
--- a/.github/workflows/generated-ios-12-5-1-x86-64.yml
+++ b/.github/workflows/generated-ios-12-5-1-x86-64.yml
@@ -21,6 +21,7 @@
IN_CI: 1
IS_GHA: 1
+
jobs:
ciflow_should_run:
diff --git a/.github/workflows/generated-macos-10-15-py3-arm64.yml b/.github/workflows/generated-macos-10-15-py3-arm64.yml
new file mode 100644
index 0000000..6a99e37
--- /dev/null
+++ b/.github/workflows/generated-macos-10-15-py3-arm64.yml
@@ -0,0 +1,97 @@
+# @generated DO NOT EDIT MANUALLY
+# Template is at: .github/templates/macos_ci_workflow.yml.j2
+# Generation script: .github/scripts/generate_ci_workflows.py
+name: macos-10-15-py3-arm64
+
+on:
+ pull_request:
+ types: [opened, synchronize, reopened, unassigned]
+ push:
+ branches:
+ - master
+ - release/*
+ workflow_dispatch:
+
+# For setup-miniconda, see https://github.com/conda-incubator/setup-miniconda/issues/179
+defaults:
+ run:
+ shell: bash -e -l {0}
+env:
+ BUILD_ENVIRONMENT: macos-10-15-py3-arm64
+ COMPACT_JOB_NAME: macos-10-15-py3-arm64
+ IN_CI: 1
+ IS_GHA: 1
+
+
+jobs:
+
+ ciflow_should_run:
+ runs-on: ubuntu-18.04
+ timeout-minutes: 240
+ env:
+ IS_PROBOT_TRIGGER_EVENT: ${{ (github.event.action == 'unassigned') && (github.event.assigneed.login == 'pytorchbot') }}
+ LABEL_CONDITIONS: ${{ contains(github.event.pull_request.labels.*.name, 'ciflow/all') || contains(github.event.pull_request.labels.*.name, 'ciflow/macos') }}
+ LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }}
+ if: ${{ (github.repository == 'pytorch/pytorch') && (
+ (github.event_name == 'push') ||
+ (github.event_name == 'schedule') ||
+ (contains(github.event.pull_request.labels.*.name, 'ciflow/all') || contains(github.event.pull_request.labels.*.name, 'ciflow/macos')) ||
+ (false))
+ }}
+ steps:
+ - name: noop
+ run: echo running ciflow_should_run
+ - name: print labels
+ run: echo "${LABELS}"
+
+ build:
+ runs-on: macos-10.15
+ needs: [ciflow_should_run]
+ env:
+ JOB_BASE_NAME: macos-10-15-py3-arm64
+ # For sccache access (only on non-forked PRs)
+ AWS_ACCESS_KEY_ID: ${{ secrets.MACOS_SCCACHE_S3_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.MACOS_SCCACHE_S3_SECRET_ACCESS_KEY }}
+ steps:
+ - name: Checkout PyTorch
+ uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9
+ with:
+ # deep clone, to allow use of git merge-base
+ fetch-depth: 0
+ submodules: recursive
+ - name: Setup miniconda
+ uses: conda-incubator/setup-miniconda@v2
+ with:
+ auto-update-conda: true
+ python-version: 3.8
+ activate-environment: build
+ - name: Install macOS homebrew dependencies
+ run: |
+ # Install dependencies
+ brew install libomp
+ - name: Install sccache (only for non-forked PRs)
+ if: github.event.pull_request.head.repo.full_name == github.repository
+ run: |
+ sudo curl --retry 3 https://s3.amazonaws.com/ossci-macos/sccache_v2.15 --output /usr/local/bin/sccache
+ sudo chmod +x /usr/local/bin/sccache
+ echo "SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2" >> "${GITHUB_ENV}"
+ - name: Build
+ run: |
+ echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname "$(which conda)")/../"}" >> "${GITHUB_ENV}"
+ .jenkins/pytorch/macos-build.sh
+ - name: Archive artifacts into zip
+ run: |
+ zip -1 -r artifacts.zip dist/
+ - uses: actions/upload-artifact@v2
+ name: Store PyTorch Build Artifacts on S3
+ with:
+ name: ${{ env.BUILD_ENVIRONMENT }}
+ retention-days: 14
+ if-no-files-found: error
+ path:
+ artifacts.zip
+
+
+concurrency:
+ group: macos-10-15-py3-arm64-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
+ cancel-in-progress: true
diff --git a/.github/workflows/generated-macos-10-15-py3-lite-interpreter-x86-64.yml b/.github/workflows/generated-macos-10-15-py3-lite-interpreter-x86-64.yml
new file mode 100644
index 0000000..3fd8c6e
--- /dev/null
+++ b/.github/workflows/generated-macos-10-15-py3-lite-interpreter-x86-64.yml
@@ -0,0 +1,88 @@
+# @generated DO NOT EDIT MANUALLY
+# Template is at: .github/templates/macos_ci_workflow.yml.j2
+# Generation script: .github/scripts/generate_ci_workflows.py
+name: macos-10-15-py3-lite-interpreter-x86-64
+
+on:
+ pull_request:
+ types: [opened, synchronize, reopened, unassigned]
+ push:
+ branches:
+ - master
+ - release/*
+ workflow_dispatch:
+
+# For setup-miniconda, see https://github.com/conda-incubator/setup-miniconda/issues/179
+defaults:
+ run:
+ shell: bash -e -l {0}
+env:
+ BUILD_ENVIRONMENT: macos-10-15-py3-lite-interpreter-x86-64
+ COMPACT_JOB_NAME: macos-10-15-py3-lite-interpreter-x86-64
+ IN_CI: 1
+ IS_GHA: 1
+
+ # Set xcode xcode version to 12
+ DEVELOPER_DIR: /Applications/Xcode_12.app/Contents/Developer
+
+jobs:
+
+ ciflow_should_run:
+ runs-on: ubuntu-18.04
+ timeout-minutes: 240
+ env:
+ IS_PROBOT_TRIGGER_EVENT: ${{ (github.event.action == 'unassigned') && (github.event.assigneed.login == 'pytorchbot') }}
+ LABEL_CONDITIONS: ${{ contains(github.event.pull_request.labels.*.name, 'ciflow/all') || contains(github.event.pull_request.labels.*.name, 'ciflow/macos') }}
+ LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }}
+ if: ${{ (github.repository == 'pytorch/pytorch') && (
+ (github.event_name == 'push') ||
+ (github.event_name == 'schedule') ||
+ (contains(github.event.pull_request.labels.*.name, 'ciflow/all') || contains(github.event.pull_request.labels.*.name, 'ciflow/macos')) ||
+ (false))
+ }}
+ steps:
+ - name: noop
+ run: echo running ciflow_should_run
+ - name: print labels
+ run: echo "${LABELS}"
+
+ build:
+ runs-on: macos-10.15
+ needs: [ciflow_should_run]
+ env:
+ JOB_BASE_NAME: macos-10-15-py3-lite-interpreter-x86-64
+ # For sccache access (only on non-forked PRs)
+ AWS_ACCESS_KEY_ID: ${{ secrets.MACOS_SCCACHE_S3_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.MACOS_SCCACHE_S3_SECRET_ACCESS_KEY }}
+ steps:
+ - name: Checkout PyTorch
+ uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9
+ with:
+ # deep clone, to allow use of git merge-base
+ fetch-depth: 0
+ submodules: recursive
+ - name: Setup miniconda
+ uses: conda-incubator/setup-miniconda@v2
+ with:
+ auto-update-conda: true
+ python-version: 3.8
+ activate-environment: build
+ - name: Install macOS homebrew dependencies
+ run: |
+ # Install dependencies
+ brew install libomp
+ - name: Install sccache (only for non-forked PRs)
+ if: github.event.pull_request.head.repo.full_name == github.repository
+ run: |
+ sudo curl --retry 3 https://s3.amazonaws.com/ossci-macos/sccache_v2.15 --output /usr/local/bin/sccache
+ sudo chmod +x /usr/local/bin/sccache
+ echo "SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2" >> "${GITHUB_ENV}"
+ - name: Build
+ run: |
+ echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname "$(which conda)")/../"}" >> "${GITHUB_ENV}"
+ .jenkins/pytorch/macos-build.sh
+
+
+concurrency:
+ group: macos-10-15-py3-lite-interpreter-x86-64-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
+ cancel-in-progress: true
diff --git a/.github/workflows/generated-macos-10-15-py3-x86-64.yml b/.github/workflows/generated-macos-10-15-py3-x86-64.yml
new file mode 100644
index 0000000..517814f
--- /dev/null
+++ b/.github/workflows/generated-macos-10-15-py3-x86-64.yml
@@ -0,0 +1,133 @@
+# @generated DO NOT EDIT MANUALLY
+# Template is at: .github/templates/macos_ci_workflow.yml.j2
+# Generation script: .github/scripts/generate_ci_workflows.py
+name: macos-10-15-py3-x86-64
+
+on:
+ pull_request:
+ types: [opened, synchronize, reopened, unassigned]
+ push:
+ branches:
+ - master
+ - release/*
+ workflow_dispatch:
+
+# For setup-miniconda, see https://github.com/conda-incubator/setup-miniconda/issues/179
+defaults:
+ run:
+ shell: bash -e -l {0}
+env:
+ BUILD_ENVIRONMENT: macos-10-15-py3-x86-64
+ COMPACT_JOB_NAME: macos-10-15-py3-x86-64
+ IN_CI: 1
+ IS_GHA: 1
+
+ # Set xcode xcode version to 12
+ DEVELOPER_DIR: /Applications/Xcode_12.app/Contents/Developer
+
+jobs:
+
+ ciflow_should_run:
+ runs-on: ubuntu-18.04
+ timeout-minutes: 240
+ env:
+ IS_PROBOT_TRIGGER_EVENT: ${{ (github.event.action == 'unassigned') && (github.event.assigneed.login == 'pytorchbot') }}
+ LABEL_CONDITIONS: ${{ contains(github.event.pull_request.labels.*.name, 'ciflow/all') || contains(github.event.pull_request.labels.*.name, 'ciflow/macos') }}
+ LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }}
+ if: ${{ (github.repository == 'pytorch/pytorch') && (
+ (github.event_name == 'push') ||
+ (github.event_name == 'schedule') ||
+ (contains(github.event.pull_request.labels.*.name, 'ciflow/all') || contains(github.event.pull_request.labels.*.name, 'ciflow/macos')) ||
+ (false))
+ }}
+ steps:
+ - name: noop
+ run: echo running ciflow_should_run
+ - name: print labels
+ run: echo "${LABELS}"
+
+ build-test:
+ runs-on: macos-10.15
+ needs: [ciflow_should_run]
+ env:
+ JOB_BASE_NAME: macos-10-15-py3-x86-64
+ # For sccache access (only on non-forked PRs)
+ AWS_ACCESS_KEY_ID: ${{ secrets.MACOS_SCCACHE_S3_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.MACOS_SCCACHE_S3_SECRET_ACCESS_KEY }}
+ steps:
+ - name: Checkout PyTorch
+ uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9
+ with:
+ # deep clone, to allow use of git merge-base
+ fetch-depth: 0
+ submodules: recursive
+ - name: Setup miniconda
+ uses: conda-incubator/setup-miniconda@v2
+ with:
+ auto-update-conda: true
+ python-version: 3.8
+ activate-environment: build
+ - name: Install macOS homebrew dependencies
+ run: |
+ # Install dependencies
+ brew install libomp
+ - name: Install sccache (only for non-forked PRs)
+ if: github.event.pull_request.head.repo.full_name == github.repository
+ run: |
+ sudo curl --retry 3 https://s3.amazonaws.com/ossci-macos/sccache_v2.15 --output /usr/local/bin/sccache
+ sudo chmod +x /usr/local/bin/sccache
+ echo "SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2" >> "${GITHUB_ENV}"
+ - name: Build
+ run: |
+ echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname "$(which conda)")/../"}" >> "${GITHUB_ENV}"
+ .jenkins/pytorch/macos-build.sh
+ - name: Archive artifacts into zip
+ run: |
+ zip -1 -r artifacts.zip dist/
+ - uses: actions/upload-artifact@v2
+ name: Store PyTorch Build Artifacts on S3
+ with:
+ name: ${{ env.BUILD_ENVIRONMENT }}
+ retention-days: 14
+ if-no-files-found: error
+ path:
+ artifacts.zip
+ - name: Test
+ run: |
+ python -m pip install dist/*.whl
+ .jenkins/pytorch/macos-test.sh
+ - name: Install render_test_results dependencies
+ if: always()
+ shell: bash
+ run: |
+ python3 -m pip install junitparser==2.1.1 rich==10.9.0
+ - name: "[[ Click me for rendered test results (useful for finding failing tests) ]]"
+ if: always()
+ shell: bash
+ # Encoding is weird on windows, just try to default to utf-8 if possible
+ env:
+ PYTHONIOENCODING: "utf-8"
+ run: |
+ python3 tools/render_junit.py test/
+ - name: Zip test reports for upload
+ if: always()
+ env:
+ FILE_SUFFIX: 'macos-${{ github.job }}'
+ run: |
+ # Remove any previous test reports if they exist
+ rm -f test-reports-*.zip
+ zip -r "test-reports-${FILE_SUFFIX}.zip" test -i '*.xml'
+ - uses: actions/upload-artifact@v2
+ name: Store Test Reports on Github
+ if: always()
+ with:
+ name: test-reports
+ retention-days: 14
+ if-no-files-found: error
+ path:
+ test-reports-*.zip
+
+
+concurrency:
+ group: macos-10-15-py3-x86-64-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
+ cancel-in-progress: true
diff --git a/.jenkins/pytorch/macos-build.sh b/.jenkins/pytorch/macos-build.sh
index 95d0567..06e2493 100755
--- a/.jenkins/pytorch/macos-build.sh
+++ b/.jenkins/pytorch/macos-build.sh
@@ -4,8 +4,6 @@
# shellcheck source=./macos-common.sh
source "$(dirname "${BASH_SOURCE[0]}")/macos-common.sh"
-export CMAKE_PREFIX_PATH=${WORKSPACE_DIR}/miniconda3/
-
# Build PyTorch
if [ -z "${IN_CI}" ]; then
export DEVELOPER_DIR=/Applications/Xcode9.app/Contents/Developer
@@ -16,28 +14,57 @@
# The way this is done is by detecting the command of the parent pid of the current process and checking whether
# that is sccache, and wrapping sccache around the process if its parent were not already sccache.
function write_sccache_stub() {
- printf "#!/bin/sh\nif [ \$(ps auxc \$(ps auxc -o ppid \$\$ | grep \$\$ | rev | cut -d' ' -f1 | rev) | tr '\\\\n' ' ' | rev | cut -d' ' -f2 | rev) != sccache ]; then\n exec sccache %s \"\$@\"\nelse\n exec %s \"\$@\"\nfi" "$(which "$1")" "$(which "$1")" > "${WORKSPACE_DIR}/$1"
- chmod a+x "${WORKSPACE_DIR}/$1"
+ output=$1
+ binary=$(basename "${output}")
+
+ printf "#!/bin/sh\nif [ \$(ps auxc \$(ps auxc -o ppid \$\$ | grep \$\$ | rev | cut -d' ' -f1 | rev) | tr '\\\\n' ' ' | rev | cut -d' ' -f2 | rev) != sccache ]; then\n exec sccache %s \"\$@\"\nelse\n exec %s \"\$@\"\nfi" "$(which "${binary}")" "$(which "${binary}")" > "${output}"
+ chmod a+x "${output}"
}
if which sccache > /dev/null; then
- write_sccache_stub clang++
- write_sccache_stub clang
+ # Create temp directory for sccache shims
+ tmp_dir=$(mktemp -d)
+ trap 'rm -rfv ${tmp_dir}' EXIT
+ write_sccache_stub "${tmp_dir}/clang++"
+ write_sccache_stub "${tmp_dir}/clang"
- export PATH="${WORKSPACE_DIR}:$PATH"
+ export PATH="${tmp_dir}:$PATH"
fi
-if [ -z "${CROSS_COMPILE_ARM64}" ]; then
- USE_DISTRIBUTED=1 python setup.py install
+cross_compile_arm64() {
+ # Cross compilation for arm64
+ USE_DISTRIBUTED=1 CMAKE_OSX_ARCHITECTURES=arm64 MACOSX_DEPLOYMENT_TARGET=11.0 USE_MKLDNN=OFF USE_NNPACK=OFF USE_QNNPACK=OFF BUILD_TEST=OFF python setup.py bdist_wheel
+}
+
+compile_x86_64() {
+ USE_DISTRIBUTED=1 python setup.py bdist_wheel
+}
+
+build_lite_interpreter() {
+ echo "Testing libtorch (lite interpreter)."
+
+ CPP_BUILD="$(pwd)/../cpp_build"
+ # Ensure the removal of the tmp directory
+ trap 'rm -rfv ${CPP_BUILD}' EXIT
+ rm -rf "${CPP_BUILD}"
+ mkdir -p "${CPP_BUILD}/caffe2"
+
+ # It looks libtorch need to be built in "${CPP_BUILD}/caffe2 folder.
+ BUILD_LIBTORCH_PY=$PWD/tools/build_libtorch.py
+ pushd "${CPP_BUILD}/caffe2" || exit
+ VERBOSE=1 DEBUG=1 python "${BUILD_LIBTORCH_PY}"
+ popd || exit
+
+ "${CPP_BUILD}/caffe2/build/bin/test_lite_interpreter_runtime"
+}
+
+if [[ ${BUILD_ENVIRONMENT} = *arm64* ]]; then
+ cross_compile_arm64
+elif [[ ${BUILD_ENVIRONMENT} = *lite-interpreter* ]]; then
+ export BUILD_LITE_INTERPRETER=1
+ build_lite_interpreter
else
- export MACOSX_DEPLOYMENT_TARGET=11.0
- USE_DISTRIBUTED=1 CMAKE_OSX_ARCHITECTURES=arm64 USE_MKLDNN=OFF USE_NNPACK=OFF USE_QNNPACK=OFF BUILD_TEST=OFF python setup.py bdist_wheel
+ compile_x86_64
fi
assert_git_not_dirty
-
-# Upload torch binaries when the build job is finished
-if [ -z "${IN_CI}" ]; then
- 7z a "${IMAGE_COMMIT_TAG}".7z "${WORKSPACE_DIR}"/miniconda3/lib/python3.6/site-packages/torch*
- aws s3 cp "${IMAGE_COMMIT_TAG}".7z s3://ossci-macos-build/pytorch/"${IMAGE_COMMIT_TAG}".7z --acl public-read
-fi
diff --git a/.jenkins/pytorch/macos-common.sh b/.jenkins/pytorch/macos-common.sh
index 01573b3..3dc7d0f 100755
--- a/.jenkins/pytorch/macos-common.sh
+++ b/.jenkins/pytorch/macos-common.sh
@@ -2,38 +2,24 @@
# Common prelude for macos-build.sh and macos-test.sh
-sysctl -a | grep machdep.cpu
-
-# shellcheck disable=SC2034
-COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
-
# shellcheck source=./common.sh
source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
-export PATH="/usr/local/bin:$PATH"
-export WORKSPACE_DIR="${HOME}/workspace"
-mkdir -p "${WORKSPACE_DIR}"
-if [[ "${COMPACT_JOB_NAME}" == *arm64* ]]; then
- MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-py38_4.9.2-MacOSX-x86_64.sh"
-else
- MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-py37_4.8.3-MacOSX-x86_64.sh"
-fi
-
-# If a local installation of conda doesn't exist, we download and install conda
-if [ ! -d "${WORKSPACE_DIR}/miniconda3" ]; then
- mkdir -p "${WORKSPACE_DIR}"
- curl --retry 3 ${MINICONDA_URL} -o "${WORKSPACE_DIR}"/miniconda3.sh
- retry bash "${WORKSPACE_DIR}"/miniconda3.sh -b -p "${WORKSPACE_DIR}"/miniconda3
-fi
-export PATH="${WORKSPACE_DIR}/miniconda3/bin:$PATH"
-# shellcheck disable=SC1091
-source "${WORKSPACE_DIR}"/miniconda3/bin/activate
+sysctl -a | grep machdep.cpu
# NOTE: mkl 2021.3.0+ cmake requires sub-command PREPEND, may break the build
retry conda install -y \
- mkl=2021.2.0 mkl-include=2021.2.0 \
- numpy=1.18.5 pyyaml=5.3 setuptools=46.0.0 \
- cmake cffi ninja typing_extensions dataclasses pip
+ mkl=2021.2.0 \
+ mkl-include=2021.2.0 \
+ numpy=1.18.5 \
+ pyyaml=5.3 \
+ setuptools=46.0.0 \
+ cmake=3.19 \
+ cffi \
+ ninja \
+ typing_extensions \
+ dataclasses \
+ pip
# The torch.hub tests make requests to GitHub.
#
@@ -52,10 +38,6 @@
# Building with USE_DISTRIBUTED=1 requires libuv (for Gloo).
retry conda install -y libuv pkg-config
-# Image commit tag is used to persist the build from the build job
-# and to retrieve the build from the test job.
-export IMAGE_COMMIT_TAG=${BUILD_ENVIRONMENT}-${IMAGE_COMMIT_ID}
-
# These are required for both the build job and the test job.
# In the latter to test cpp extensions.
export MACOSX_DEPLOYMENT_TARGET=10.9
diff --git a/.jenkins/pytorch/macos-lite-interpreter-build-test.sh b/.jenkins/pytorch/macos-lite-interpreter-build-test.sh
deleted file mode 100644
index 99c424a..0000000
--- a/.jenkins/pytorch/macos-lite-interpreter-build-test.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/bin/bash
-
-# shellcheck source=./macos-common.sh
-source "$(dirname "${BASH_SOURCE[0]}")/macos-common.sh"
-
-git submodule sync --recursive
-git submodule update --init --recursive --jobs 0
-export CMAKE_PREFIX_PATH=${WORKSPACE_DIR}/miniconda3/
-
-# Build PyTorch
-if [ -z "${IN_CI}" ]; then
- export DEVELOPER_DIR=/Applications/Xcode9.app/Contents/Developer
-fi
-
-echo "BUILD_LITE_INTERPRETER: ${BUILD_LITE_INTERPRETER}"
-if [ "${BUILD_LITE_INTERPRETER}" == 1 ]; then
- echo "Testing libtorch (lite interpreter)."
-
- CPP_BUILD="$PWD/../cpp-build"
- rm -rf "${CPP_BUILD}"
- mkdir -p "${CPP_BUILD}/caffe2"
-
- # It looks libtorch need to be built in "${CPP_BUILD}/caffe2"
- # folder.
- BUILD_LIBTORCH_PY=$PWD/tools/build_libtorch.py
- pushd "${CPP_BUILD}/caffe2" || exit
- VERBOSE=1 DEBUG=1 python "${BUILD_LIBTORCH_PY}"
- popd || exit
-
- "${CPP_BUILD}/caffe2/build/bin/test_lite_interpreter_runtime"
-
- # Change the permission manually from 755 to 644 to keep git clean
- chmod 644 "${HOME}/project/.jenkins/pytorch/macos-lite-interpreter-build-test.sh"
- assert_git_not_dirty
-else
- echo "Skipping libtorch (lite interpreter)."
-fi
diff --git a/.jenkins/pytorch/macos-test.sh b/.jenkins/pytorch/macos-test.sh
index c2d2722..fb9b4e2 100755
--- a/.jenkins/pytorch/macos-test.sh
+++ b/.jenkins/pytorch/macos-test.sh
@@ -153,19 +153,8 @@
assert_git_not_dirty
}
-if [ -z "${BUILD_ENVIRONMENT}" ] || [[ "${BUILD_ENVIRONMENT}" == *-test ]]; then
- test_python_all
- test_libtorch
- test_custom_script_ops
- test_jit_hooks
- test_custom_backend
-else
- if [[ "${BUILD_ENVIRONMENT}" == *-test1 ]]; then
- test_python_all
- elif [[ "${BUILD_ENVIRONMENT}" == *-test2 ]]; then
- test_libtorch
- test_custom_script_ops
- test_jit_hooks
- test_custom_backend
- fi
-fi
+test_python_all
+test_libtorch
+test_custom_script_ops
+test_jit_hooks
+test_custom_backend
