Consolidate macos pip dependencies (#88071)
After conda, consolidating all macos pip dependencies to cache every dependencies that macos CI needs. Two small issues are found along the way in `_mac-test-mps` workflow:
* It didn't have `Install macOS homebrew dependencies` to install libomp like the regular `_mac-test` workflow
* It didn't install `scipy`, thus silently skipping some `signal.windows` tests
Both are fixed in this PR
Pull Request resolved: https://github.com/pytorch/pytorch/pull/88071
Approved by: https://github.com/malfet
diff --git a/.github/requirements/README.md b/.github/requirements/README.md
index 9093b92..a4f3cb7 100644
--- a/.github/requirements/README.md
+++ b/.github/requirements/README.md
@@ -17,3 +17,6 @@
test jobs to setup the conda environment
* conda-env-macOS-X64. This is use by MacOS (x86-64) build and test
jobs to setup the conda environment
+* Pip:
+ * pip-requirements-macOS.txt. This is used by MacOS build and test jobs to
+ setup the pip environment
diff --git a/.github/requirements/pip-requirements-macOS.txt b/.github/requirements/pip-requirements-macOS.txt
new file mode 100644
index 0000000..7aa2306
--- /dev/null
+++ b/.github/requirements/pip-requirements-macOS.txt
@@ -0,0 +1,21 @@
+boto3==1.19.12
+hypothesis==6.56.4
+expecttest==0.1.3
+librosa>=0.6.2
+mpmath==1.2.1
+networkx==2.8.7
+# Use numba-0.49.1 or older on Intel Macs, but 0.56.0 on M1 machines, as older numba is not available
+numba==0.56.0; platform_machine == "arm64"
+numba<=0.49.1; platform_machine != "arm64"
+opt-einsum>=3.3
+psutil==5.9.1
+pynvml==11.4.1
+pygments==2.12.0
+pytest==7.2.0
+pytest-xdist==3.0.2
+pytest-rerunfailures==10.2
+pytest-shard==0.1.2
+scipy==1.9.0
+sympy==1.11.1
+unittest-xml-reporting<=3.2.0,>=2.0.0
+xdoctest==1.0.2
diff --git a/.github/workflows/_mac-test-mps.yml b/.github/workflows/_mac-test-mps.yml
index 57e97e4..3f7ba04 100644
--- a/.github/workflows/_mac-test-mps.yml
+++ b/.github/workflows/_mac-test-mps.yml
@@ -37,11 +37,21 @@
name: ${{ inputs.build-environment }}
use-gha: true
+ # This is copied from the main macos test workflow. It was missed in the earlier fix because macos M1
+ # runners are shared and not ephemeral, so the issue wasn't manifested if the runners with the fix were
+ # used
+ - name: Install macOS homebrew dependencies
+ run: |
+ # Install dependencies
+ brew install libomp
+ brew link --force libomp
+
- name: Setup miniconda
uses: pytorch/test-infra/.github/actions/setup-miniconda@main
with:
python-version: 3.9
environment-file: .github/requirements/conda-env-${{ runner.os }}-${{ runner.arch }}
+ pip-requirements-file: .github/requirements/pip-requirements-${{ runner.os }}.txt
- name: Install PyTorch
env:
@@ -51,10 +61,9 @@
run: |
# shellcheck disable=SC1090
set -ex
- ${CONDA_RUN} python3 -mpip install "unittest-xml-reporting<=3.2.0,>=2.0.0"
# As wheels are cross-compiled they are reported as x86_64 ones
ORIG_WHLNAME=$(ls -1 dist/*.whl); ARM_WHLNAME=${ORIG_WHLNAME/x86_64/arm64}; mv ${ORIG_WHLNAME} ${ARM_WHLNAME}
- ${CONDA_RUN} python3 -mpip install dist/*.whl
+ ${CONDA_RUN} python3 -mpip install --no-index --no-deps dist/*.whl
- name: Run MPS tests
env:
diff --git a/.github/workflows/_mac-test.yml b/.github/workflows/_mac-test.yml
index db524ca..52aba9c 100644
--- a/.github/workflows/_mac-test.yml
+++ b/.github/workflows/_mac-test.yml
@@ -94,6 +94,7 @@
with:
python-version: 3.8
environment-file: .github/requirements/conda-env-${{ runner.os }}-${{ runner.arch }}
+ pip-requirements-file: .github/requirements/pip-requirements-${{ runner.os }}.txt
- name: Setup miniconda (arm64, py3.9)
if: ${{ runner.arch == 'ARM64' }}
@@ -101,12 +102,11 @@
with:
python-version: 3.9
environment-file: .github/requirements/conda-env-${{ runner.os }}-${{ runner.arch }}
+ pip-requirements-file: .github/requirements/pip-requirements-${{ runner.os }}.txt
- name: Start monitoring script
id: monitor-script
run: |
- ${CONDA_RUN} python3 -m pip install psutil==5.9.1
- ${CONDA_RUN} python3 -m pip install pynvml==11.4.1
${CONDA_RUN} python3 -m tools.stats.monitor > usage_log.txt 2>&1 &
echo "monitor-script-pid=${!}" >> "${GITHUB_OUTPUT}"
@@ -144,7 +144,7 @@
export PR_BODY="${PR_BODY//[\'\"]}"
arch
- ${CONDA_RUN} python3 -mpip install $(echo dist/*.whl)[opt-einsum]
+ ${CONDA_RUN} python3 -mpip install --no-index --no-deps $(echo dist/*.whl)
${CONDA_RUN} .jenkins/pytorch/macos-test.sh
- name: Get workflow job id
@@ -190,6 +190,4 @@
GHA_WORKFLOW_JOB_ID: ${{ steps.get-job-id.outputs.job-id }}
run: |
set -x
- ${CONDA_RUN} python3 -m pip install -r requirements.txt
- ${CONDA_RUN} python3 -m pip install boto3==1.19.12
${CONDA_RUN} python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
diff --git a/.jenkins/pytorch/macos-test.sh b/.jenkins/pytorch/macos-test.sh
index b598eff..4beab88 100755
--- a/.jenkins/pytorch/macos-test.sh
+++ b/.jenkins/pytorch/macos-test.sh
@@ -4,23 +4,6 @@
# shellcheck source=./macos-common.sh
source "$(dirname "${BASH_SOURCE[0]}")/macos-common.sh"
-if [[ ${BUILD_ENVIRONMENT} = *arm64* ]]; then
- pip install hypothesis "expecttest==0.1.3" "librosa>=0.6.2" "numba==0.56.0" psutil "scipy==1.9.0"
-else
- pip install hypothesis "expecttest==0.1.3" "librosa>=0.6.2" "numba<=0.49.1" psutil "scipy==1.6.3"
-fi
-
-# TODO move this to docker
-# Pin unittest-xml-reporting to freeze printing test summary logic, related: https://github.com/pytorch/pytorch/issues/69014
-pip install "unittest-xml-reporting<=3.2.0,>=2.0.0" \
- pytest \
- pytest-xdist \
- pytest-shard \
- pytest-rerunfailures \
- "xdoctest==1.0.2" \
- "pygments==2.12.0" \
- "opt-einsum>=3.3"
-
if [ -z "${CI}" ]; then
rm -rf "${WORKSPACE_DIR}"/miniconda3/lib/python3.6/site-packages/torch*
fi
diff --git a/torch/testing/_internal/opinfo/definitions/signal.py b/torch/testing/_internal/opinfo/definitions/signal.py
index c915e93..0bab000 100644
--- a/torch/testing/_internal/opinfo/definitions/signal.py
+++ b/torch/testing/_internal/opinfo/definitions/signal.py
@@ -368,5 +368,12 @@
sample_inputs_func=partial(sample_inputs_window, beta=12.0),
reference_inputs_func=partial(reference_inputs_kaiser_window, beta=12.0),
error_inputs_func=error_inputs_kaiser_window,
+ skips=(
+ DecorateInfo(
+ unittest.skip("Unsupported on MPS for now pending aten::i0 support"),
+ "TestCommon",
+ "test_numpy_ref_mps",
+ ),
+ ),
),
]
