commit b6f41e2bcd69e3e38109232f6684063ab828473d
author Huy Do <huydhn@gmail.com> Tue Jan 24 05:14:10 2023 +0000
committer PyTorch MergeBot <pytorchmergebot@users.noreply.github.com> Tue Jan 24 05:14:14 2023 +0000
tree 375a0df9af79af858276b958d42c910d3d3fe104
parent 0bf7506051c0cd78224e86d6a14cd059b3bea3fe

[MacOS] Explicitly use cmake from cloned conda environment (#92737)

My first attempt to fix `Library not loaded: @rpath/libzstd.1.dylib` issue on MacOS M1 in https://github.com/pytorch/pytorch/pull/91142 provides some additional logs about flaky error but doesn't fix the issue as I see some of them recently, for example

* https://hud.pytorch.org/pytorch/pytorch/commit/e4d83d54a6214d8fa1a9063f0da65932b45b7207

Looking at the log, I can see that:

* CMAKE_EXEC correctly points to `CMAKE_EXEC=/Users/ec2-user/runner/_work/_temp/conda_environment_3971491892/bin/cmake`
* The library is there under the executable rpath
```
ls -la /Users/ec2-user/runner/_work/_temp/conda_environment_3971491892/bin/../lib
...
2023-01-20T23:22:03.9761370Z -rwxr-xr-x    2 ec2-user  staff    737776 Apr 22  2022 libzstd.1.5.2.dylib
2023-01-20T23:22:03.9761630Z lrwxr-xr-x    1 ec2-user  staff        19 Jan 20 22:47 libzstd.1.dylib -> libzstd.1.5.2.dylib
...
```

Then calling cmake after that suddenly uses the wrong cmake from miniconda package cache:

```
2023-01-20T23:22:04.0636880Z + cmake ..
2023-01-20T23:22:04.1924790Z dyld[85763]: Library not loaded: @rpath/libzstd.1.dylib
2023-01-20T23:22:04.1925540Z   Referenced from: /Users/ec2-user/runner/_work/_temp/miniconda/pkgs/cmake-3.22.1-hae769c0_0/bin/cmake
```

This is weird, so my second attempt will be more explicit and use the correct cmake executable in `CMAKE_EXEC`.  May be something manipulates the global path in between making ` /Users/ec2-user/runner/_work/_temp/miniconda/pkgs/cmake-3.22.1-hae769c0_0/bin/cmake` comes first in the PATH

Pull Request resolved: https://github.com/pytorch/pytorch/pull/92737
Approved by: https://github.com/ZainRizvi

.jenkins/pytorch/macos-test.sh

diff --git a/.jenkins/pytorch/macos-test.sh b/.jenkins/pytorch/macos-test.sh
index ebdba69..2da2be0 100755
--- a/.jenkins/pytorch/macos-test.sh
+++ b/.jenkins/pytorch/macos-test.sh
@@ -89,6 +89,8 @@
   CONDA_INSTALLATION_DIR=$(dirname "$CMAKE_EXEC")
   # Print all libraries under cmake rpath for debugging
   ls -la "$CONDA_INSTALLATION_DIR/../lib"
+
+  export CMAKE_EXEC
 }
 
 test_custom_backend() {
@@ -99,7 +101,7 @@
   rm -rf build && mkdir build
   pushd build
   SITE_PACKAGES="$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')"
-  CMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" cmake ..
+  CMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" "${CMAKE_EXEC}" ..
   make VERBOSE=1
   popd
 
@@ -122,7 +124,7 @@
   rm -rf build && mkdir build
   pushd build
   SITE_PACKAGES="$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')"
-  CMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" cmake ..
+  CMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" "${CMAKE_EXEC}" ..
   make VERBOSE=1
   popd
 
@@ -144,7 +146,7 @@
   rm -rf build && mkdir build
   pushd build
   SITE_PACKAGES="$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')"
-  CMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" cmake ..
+  CMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" "${CMAKE_EXEC}" ..
   make VERBOSE=1
   popd