Run CI in a new environment (#92378)
Needed to be able to install newer Python versions (Python-3.11 in this case), which do not have numerous packages that default environment must have
In addition, fix weird incursion of `conda-forge` by torch-deploy test.
Reincarnation of an old https://github.com/pytorch/pytorch/pull/66530
Pull Request resolved: https://github.com/pytorch/pytorch/pull/92378
Approved by: https://github.com/kit1980
diff --git a/.circleci/docker/centos-rocm/Dockerfile b/.circleci/docker/centos-rocm/Dockerfile
index 52ea467..af7073f 100644
--- a/.circleci/docker/centos-rocm/Dockerfile
+++ b/.circleci/docker/centos-rocm/Dockerfile
@@ -37,9 +37,10 @@
RUN bash ./install_user.sh && rm install_user.sh
# Install conda and other packages (e.g., numpy, pytest)
-ENV PATH /opt/conda/bin:$PATH
ARG ANACONDA_PYTHON_VERSION
ARG CONDA_CMAKE
+ENV ANACONDA_PYTHON_VERSION=$ANACONDA_PYTHON_VERSION
+ENV PATH /opt/conda/envs/py_$ANACONDA_PYTHON_VERSION/bin:/opt/conda/bin:$PATH
COPY requirements-ci.txt /opt/conda/requirements-ci.txt
COPY ./common/install_conda.sh install_conda.sh
RUN bash ./install_conda.sh && rm install_conda.sh
diff --git a/.circleci/docker/common/install_conda.sh b/.circleci/docker/common/install_conda.sh
index 90040c6..225a5a5 100755
--- a/.circleci/docker/common/install_conda.sh
+++ b/.circleci/docker/common/install_conda.sh
@@ -61,17 +61,17 @@
# as_jenkins conda update -y -n base conda
# Install correct Python version
- as_jenkins conda install -y python="$ANACONDA_PYTHON_VERSION"
+ as_jenkins conda create -n py_$ANACONDA_PYTHON_VERSION -y python="$ANACONDA_PYTHON_VERSION"
conda_install() {
# Ensure that the install command don't upgrade/downgrade Python
# This should be called as
# conda_install pkg1 pkg2 ... [-c channel]
- as_jenkins conda install -q -y python="$ANACONDA_PYTHON_VERSION" $*
+ as_jenkins conda install -q -n py_$ANACONDA_PYTHON_VERSION -y python="$ANACONDA_PYTHON_VERSION" $*
}
pip_install() {
- as_jenkins pip install --progress-bar off $*
+ as_jenkins conda run -n py_$ANACONDA_PYTHON_VERSION pip install --progress-bar off $*
}
# Install PyTorch conda deps, as per https://github.com/pytorch/pytorch README
diff --git a/.circleci/docker/common/install_rocm_magma.sh b/.circleci/docker/common/install_rocm_magma.sh
index c7b116b9..fa21321 100644
--- a/.circleci/docker/common/install_rocm_magma.sh
+++ b/.circleci/docker/common/install_rocm_magma.sh
@@ -23,7 +23,7 @@
# hipcc with openmp flag may cause isnan() on __device__ not to be found; depending on context, compiler may attempt to match with host definition
sed -i 's/^FOPENMP/#FOPENMP/g' make.inc
make -f make.gen.hipMAGMA -j $(nproc)
-LANG=C.UTF-8 make lib/libmagma.so -j $(nproc) MKLROOT=/opt/conda
-make testing/testing_dgemm -j $(nproc) MKLROOT=/opt/conda
+LANG=C.UTF-8 make lib/libmagma.so -j $(nproc) MKLROOT=/opt/conda/envs/py_$ANACONDA_PYTHON_VERSION
+make testing/testing_dgemm -j $(nproc) MKLROOT=/opt/conda/envs/py_$ANACONDA_PYTHON_VERSION
popd
mv magma /opt/rocm
diff --git a/.circleci/docker/ubuntu-cuda/Dockerfile b/.circleci/docker/ubuntu-cuda/Dockerfile
index dbe5985..b64e3ee 100644
--- a/.circleci/docker/ubuntu-cuda/Dockerfile
+++ b/.circleci/docker/ubuntu-cuda/Dockerfile
@@ -23,8 +23,9 @@
RUN bash ./install_docs_reqs.sh && rm install_docs_reqs.sh
# Install conda and other packages (e.g., numpy, pytest)
-ENV PATH /opt/conda/bin:$PATH
ARG ANACONDA_PYTHON_VERSION
+ENV ANACONDA_PYTHON_VERSION=$ANACONDA_PYTHON_VERSION
+ENV PATH /opt/conda/envs/py_$ANACONDA_PYTHON_VERSION/bin:/opt/conda/bin:$PATH
ARG CONDA_CMAKE
COPY requirements-ci.txt /opt/conda/requirements-ci.txt
COPY ./common/install_conda.sh install_conda.sh
diff --git a/.circleci/docker/ubuntu-rocm/Dockerfile b/.circleci/docker/ubuntu-rocm/Dockerfile
index da8b326..3d3cbf7 100644
--- a/.circleci/docker/ubuntu-rocm/Dockerfile
+++ b/.circleci/docker/ubuntu-rocm/Dockerfile
@@ -25,9 +25,10 @@
RUN bash ./install_user.sh && rm install_user.sh
# Install conda and other packages (e.g., numpy, pytest)
-ENV PATH /opt/conda/bin:$PATH
ARG ANACONDA_PYTHON_VERSION
ARG CONDA_CMAKE
+ENV ANACONDA_PYTHON_VERSION=$ANACONDA_PYTHON_VERSION
+ENV PATH /opt/conda/envs/py_$ANACONDA_PYTHON_VERSION/bin:/opt/conda/bin:$PATH
COPY requirements-ci.txt /opt/conda/requirements-ci.txt
COPY ./common/install_conda.sh install_conda.sh
RUN bash ./install_conda.sh && rm install_conda.sh
diff --git a/.circleci/docker/ubuntu/Dockerfile b/.circleci/docker/ubuntu/Dockerfile
index e8c264c..bce7d48 100644
--- a/.circleci/docker/ubuntu/Dockerfile
+++ b/.circleci/docker/ubuntu/Dockerfile
@@ -34,9 +34,10 @@
RUN bash ./install_docs_reqs.sh && rm install_docs_reqs.sh
# Install conda and other packages (e.g., numpy, pytest)
-ENV PATH /opt/conda/bin:$PATH
ARG ANACONDA_PYTHON_VERSION
ARG CONDA_CMAKE
+ENV ANACONDA_PYTHON_VERSION=$ANACONDA_PYTHON_VERSION
+ENV PATH /opt/conda/envs/py_$ANACONDA_PYTHON_VERSION/bin:/opt/conda/bin:$PATH
COPY requirements-ci.txt /opt/conda/requirements-ci.txt
COPY ./common/install_conda.sh install_conda.sh
RUN bash ./install_conda.sh && rm install_conda.sh
diff --git a/.circleci/scripts/python_doc_push_script.sh b/.circleci/scripts/python_doc_push_script.sh
index d255f77..07db737 100755
--- a/.circleci/scripts/python_doc_push_script.sh
+++ b/.circleci/scripts/python_doc_push_script.sh
@@ -77,6 +77,9 @@
export LC_ALL=C
export PATH=/opt/conda/bin:$PATH
+if [ -n $ANACONDA_PYTHON_VERSION ]; then
+ export PATH=/opt/conda/envs/py_$ANACONDA_PYTHON_VERSION/bin:$PATH
+fi
rm -rf pytorch || true
diff --git a/.jenkins/pytorch/common_utils.sh b/.jenkins/pytorch/common_utils.sh
index 5fd318b..6060a71 100644
--- a/.jenkins/pytorch/common_utils.sh
+++ b/.jenkins/pytorch/common_utils.sh
@@ -145,8 +145,7 @@
}
function setup_torchdeploy_deps(){
- conda install -y cmake
- conda install -y -c conda-forge libpython-static=3.10
+ conda install -y -n "py_${ANACONDA_PYTHON_VERSION}" "libpython-static=${ANACONDA_PYTHON_VERSION}"
local CC
local CXX
CC="$(which gcc)"
