[Caffe2] Consolidating conda build scripts (#6359)
* Consolidating conda build scripts
* Grep bug
* Naming bug
* Correcting quoting of variable passing
diff --git a/conda/caffe2/cuda/build.sh b/conda/caffe2/cuda/build.sh
deleted file mode 100755
index 0946d1d..0000000
--- a/conda/caffe2/cuda/build.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/bin/bash
-
-# Install script for Anaconda environments with CUDA on linux
-# This script is not supposed to be called directly, but should be run by:
-#
-# $ cd <path to caffe2, e.g. ~/caffe2>
-# $ conda build conda/build
-#
-# If you're debugging this, it may be useful to use the env that conda build is
-# using:
-# $ cd <anaconda_root>/conda-bld/caffe2_<timestamp>
-# $ source activate _h_env_... # some long path with lots of placeholders
-#
-# Also, failed builds will accumulate those caffe2_<timestamp> directories. You
-# can remove them after a succesfull build with
-# $ conda build purge
-#
-
-set -ex
-
-echo "Installing caffe2 to ${PREFIX}"
-
-# This is needed for build variants (packages with multiple variants in
-# conda_build_config.yaml) to remove any files that cmake cached, since
-# conda-build uses the same environment for all the build variants
-rm -rf build
-
-PYTHON_ARGS="$(python ./scripts/get_python_cmake_flags.py)"
-CMAKE_ARGS=()
-
-# Build with minimal required libraries
-# Add CMAKE flags here
-CMAKE_ARGS+=("-DUSE_MPI=OFF")
-
-# Build with CUDA
-CMAKE_ARGS+=("-DUSE_CUDA=ON")
-CMAKE_ARGS+=("-DUSE_NCCL=ON")
-
-# Install under specified prefix
-CMAKE_ARGS+=("-DCMAKE_INSTALL_PREFIX=$PREFIX")
-CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$PREFIX")
-
-mkdir -p build
-cd build
-cmake "${CMAKE_ARGS[@]}" $CONDA_CMAKE_BUILD_ARGS $PYTHON_ARGS ..
-make "-j$(nproc)"
-
-make install/fast
diff --git a/conda/caffe2/cuda_full/build.sh b/conda/caffe2/full/build.sh
similarity index 100%
rename from conda/caffe2/cuda_full/build.sh
rename to conda/caffe2/full/build.sh
diff --git a/conda/caffe2/cuda_full/conda_build_config.yaml b/conda/caffe2/full/conda_build_config.yaml
similarity index 100%
rename from conda/caffe2/cuda_full/conda_build_config.yaml
rename to conda/caffe2/full/conda_build_config.yaml
diff --git a/conda/caffe2/cuda_full/meta.yaml b/conda/caffe2/full/meta.yaml
similarity index 100%
rename from conda/caffe2/cuda_full/meta.yaml
rename to conda/caffe2/full/meta.yaml
diff --git a/conda/caffe2/no_cuda/build.sh b/conda/caffe2/no_cuda/build.sh
deleted file mode 100755
index 0ca1d5c..0000000
--- a/conda/caffe2/no_cuda/build.sh
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/bin/bash
-
-# Install script for Anaconda environments on macOS and linux.
-# This script is not supposed to be called directly, but should be run by:
-#
-# $ cd <path to caffe2, e.g. ~/caffe2>
-# $ conda build conda
-#
-# This installation uses MKL and does not use CUDA
-#
-# If you're debugging this, it may be useful to use the env that conda build is
-# using:
-# $ cd <anaconda_root>/conda-bld/caffe2_<timestamp>
-# $ source activate _h_env_... # some long path with lots of placeholders
-#
-# Also, failed builds will accumulate those caffe2_<timestamp> directories. You
-# can remove them after a succesfull build with
-# $ conda build purge
-#
-
-set -ex
-
-echo "Installing caffe2 to ${PREFIX}"
-
-# This is needed for build variants (packages with multiple variants in
-# conda_build_config.yaml) to remove any files that cmake cached, since
-# conda-build uses the same environment for all the build variants
-rm -rf build
-
-PYTHON_ARGS="$(python ./scripts/get_python_cmake_flags.py)"
-CMAKE_ARGS=()
-
-# This installation defaults to using MKL because it is much faster. If you
-# want to build without MKL then you should also remove mkl from meta.yaml in
-# addition to removing the flags below
-CMAKE_ARGS+=("-DBLAS=MKL")
-
-# Minimal packages
-CMAKE_ARGS+=("-DUSE_CUDA=OFF")
-CMAKE_ARGS+=("-DUSE_MPI=OFF")
-CMAKE_ARGS+=("-DUSE_NCCL=OFF")
-
-# Install under specified prefix
-CMAKE_ARGS+=("-DCMAKE_INSTALL_PREFIX=$PREFIX")
-CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$PREFIX")
-
-mkdir -p build
-cd build
-cmake "${CMAKE_ARGS[@]}" $CONDA_CMAKE_BUILD_ARGS $PYTHON_ARGS ..
-if [ "$(uname)" == 'Darwin' ]; then
- make "-j$(sysctl -n hw.ncpu)"
-else
- make "-j$(nproc)"
-fi
-
-make install/fast
diff --git a/conda/caffe2/no_cuda/meta.yaml b/conda/caffe2/no_cuda/meta.yaml
deleted file mode 100644
index ca62504..0000000
--- a/conda/caffe2/no_cuda/meta.yaml
+++ /dev/null
@@ -1,54 +0,0 @@
-{% set version = "0.8.dev" %}
-
-package:
- name: caffe2
- version: {{ version }}
-
-source:
- path: ../../..
-
-build:
- number: 0
- skip: True # [win]
- script_env:
- - CONDA_CMAKE_BUILD_ARGS
-
-requirements:
- build:
- - cmake
- - future
- - gflags
- - glog
- - leveldb
- - lmdb
- - mkl
- - mkl-include
- - numpy
- - opencv
- - python
- - six
- # other packages here
- run:
- - future
- - gflags
- - glog
- - leveldb
- - lmdb
- - mkl
- - mkl-include
- - numpy
- - opencv
- - protobuf
- - python
- - six
- # other packages here
-
-
-about:
- home: https://caffe2.ai/
- license: BSD
- summary: Caffe2 is a lightweight, modular, and scalable deep learning framework.
-
-extra:
- recipe-maintainers:
- - pjh5
diff --git a/conda/caffe2/normal/build.sh b/conda/caffe2/normal/build.sh
new file mode 100755
index 0000000..79b6ccf
--- /dev/null
+++ b/conda/caffe2/normal/build.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Install script for Anaconda environments on macOS and linux.
+# This script is not supposed to be called directly, but should be called by
+# scripts/build_anaconda.sh, which handles setting lots of needed flags
+# depending on the current system and user flags.
+#
+# If you're debugging this, it may be useful to use the env that conda build is
+# using:
+# $ cd <anaconda_root>/conda-bld/caffe2_<timestamp>
+# $ source activate _h_env_... # some long path with lots of placeholders
+#
+# Also, failed builds will accumulate those caffe2_<timestamp> directories. You
+# can remove them after a succesfull build with
+# $ conda build purge
+
+set -ex
+
+echo "Installing caffe2 to ${PREFIX}"
+
+PYTHON_ARGS="$(python ./scripts/get_python_cmake_flags.py)"
+
+# Install under specified prefix
+CMAKE_ARGS=()
+CMAKE_ARGS+=("-DCMAKE_INSTALL_PREFIX=$PREFIX")
+CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$PREFIX")
+
+# Build Caffe2
+mkdir -p build
+cd build
+cmake "${CMAKE_ARGS[@]}" "$PYTHON_ARGS" $CONDA_CMAKE_BUILD_ARGS ..
+if [ "$(uname)" == 'Darwin' ]; then
+ make "-j$(sysctl -n hw.ncpu)"
+else
+ make "-j$(nproc)"
+fi
+
+make install/fast
diff --git a/conda/caffe2/cuda/meta.yaml b/conda/caffe2/normal/meta.yaml
similarity index 78%
rename from conda/caffe2/cuda/meta.yaml
rename to conda/caffe2/normal/meta.yaml
index 4068f08..93bda14 100644
--- a/conda/caffe2/cuda/meta.yaml
+++ b/conda/caffe2/normal/meta.yaml
@@ -1,7 +1,7 @@
{% set version = "0.8.dev" %}
package:
- name: caffe2-cuda
+ name: caffe2
version: {{ version }}
source:
@@ -17,23 +17,13 @@
build:
- cmake
- future
- - glog
- - gflags
- - leveldb
- - lmdb
- numpy
- - opencv
- python
- six
# other packages here
run:
- future
- - glog
- - gflags
- - leveldb
- - lmdb
- numpy
- - opencv
- protobuf
- python
- six
@@ -45,7 +35,9 @@
about:
home: https://caffe2.ai/
- license: BSD
+ license: BSD 3-Clause
+ license_family: BSD
+ license_file: LICENSE
summary: Caffe2 is a lightweight, modular, and scalable deep learning framework.
extra:
diff --git a/scripts/build_anaconda.sh b/scripts/build_anaconda.sh
index d380deb..dec7f4b 100755
--- a/scripts/build_anaconda.sh
+++ b/scripts/build_anaconda.sh
@@ -1,12 +1,30 @@
#!/bin/bash
-# NOTE: All parameters to this function are forwared directly to conda-build
-# and so will never be seen by the build.sh
-# TODO change arguments to go to cmake by default
-# TODO handle setting flags in build.sh too
+# This script creates and (possibly) uploads a Caffe2 Anaconda package, and
+# then (optionally) installs the package locally into the current activated
+# conda environment. This script handles flags needed by CUDA and gcc versions,
+# and has good default behavior if called with no arguments.
+#
+# Usage:
+# ./build_anaconda.sh [--cuda X.Y] [--cudnn Z] [--conda <flag forwared to conda-build>]... [<flags forwarded to cmake>]...
+#
+# Parameters can also be passed through the BUILD_ENVIRONMENT environment
+# variable, e.g.
+# BUILD_ENVIRONMENT=conda2-cuda8.0-cudnn7-gcc4.8 ./scripts/build_anaconda.sh
+# - Parameters parsed from the BUILD_ENVIRONMENT will be overridden by command
+# line parameters.
+# - The conda version and gcc version given in BUILD_ENVIRONMENT are ignored.
+# These versions are determined by calling the binaries with --version
+#
+# The special flags SKIP_CONDA_TESTS, CAFFE2_ANACONDA_ORG_ACCESS_TOKEN, and
+# ANACONDA_USERNAME can only be passed in as environment variables.
set -ex
+#
+# Functions used in this script
+#
+
# portable_sed: A wrapper around sed that works on both mac and linux, used to
# alter conda-build files such as the meta.yaml. It always adds the inplace
# flag
@@ -19,8 +37,8 @@
fi
}
-# remove_package: Given a package name, removes any line that mentions that
-# file from the meta.yaml
+# remove_package: Given a string, removes any line that mentions that line from
+# the meta.yaml
remove_package () {
portable_sed "/$1/d" "${META_YAML}"
}
@@ -28,41 +46,93 @@
# add_package: Takes a package name and a version and finagles the
# meta.yaml to ask for that version specifically.
# NOTE: this assumes that $META_YAML has already been set
+# The \\"$'\n' is a properly escaped new line
+# Those 4 spaces are there to properly indent the comment
add_package () {
remove_package $1
# This magic string _M_STR is in the requirements sections of the meta.yaml
- # The \\"$'\n' is a properly escaped new line
- # Those 4 spaces are there to properly indent the comment
local _M_STR='# other packages here'
portable_sed "s/$_M_STR/- ${1} ${2}\\"$'\n'" $_M_STR/" "${META_YAML}"
}
-CAFFE2_ROOT="$( cd "$(dirname "$0")"/.. ; pwd -P)"
+# add_feature: Adds a given feature tag to the build section. Takes care to
+# only add the feature section once. Assumes that no feature section exists yet
+add_feature() {
+ local _M_STR='# features go here'
+ if [[ -z $ADDED_A_FEATURE ]]; then
+ portable_sed "s/$_M_STR/features:\\"$'\n'" $_M_STR/" "${META_YAML}"
+ ADDED_A_FEATURE=YES
+ fi
+ portable_sed "s/$_M_STR/- ${1}\\"$'\n'" $_M_STR/" "${META_YAML}"
+}
+
+
+#
+# Parse options from both command line and from BUILD_ENVIRONMENT
+#
CONDA_BUILD_ARGS=()
CMAKE_BUILD_ARGS=()
+CAFFE2_CONDA_CHANNEL=()
+if [[ $BUILD_ENVIRONMENT == *cuda* ]]; then
+ CUDA_VERSION="$(echo $BUILD_ENVIRONMENT | grep --only-matching -P '(?<=cuda)[0-9]\.[0-9]')"
+fi
+if [[ $BUILD_ENVIRONMENT == *cudnn* ]]; then
+ CUDNN_VERSION="$(echo $BUILD_ENVIRONMENT | grep --only-matching -P '(?<=cudnn)[0-9](\.[0-9])?')"
+fi
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --cuda)
+ shift
+ CUDA_VERSION="$1"
+ ;;
+ --cudnn)
+ shift
+ CUDNN_VERSION="$1"
+ ;;
+ --conda)
+ shift
+ CONDA_BUILD_ARGS+=("$1")
+ ;;
+ *)
+ CMAKE_BUILD_ARGS+=("$1")
+ ;;
+ esac
+ shift
+done
-# Reinitialize submodules
-git submodule update --init
+# Verify that the CUDA version is supported
+if [[ -n $CUDA_VERSION ]]; then
+ if [[ $CUDA_VERSION == 9.1* ]]; then
+ CUDA_FEATURE_NAME=cuda91
+ elif [[ $CUDA_VERSION == 9.0* ]]; then
+ CUDA_FEATURE_NAME=cuda90
+ elif [[ $CUDA_VERSION == 8.0* ]]; then
+ CUDA_FEATURE_NAME=cuda80
+ else
+ echo "Unsupported CUDA version $CUDA_VERSION"
+ echo "Changes have already been made to the meta.yaml, you may have to revert them"
+ exit 1
+ fi
+ if [[ -z $CUDNN_VERSION ]]; then
+ echo "No CuDNN version given. Caffe2 will still build against whatever"
+ echo "CuDNN that it finds first, and will break if there is no CuDNN found."
+ fi
+ echo "Detected CUDA_VERSION of $CUDA_VERSION"
+fi
#
# Read python and gcc version
#
-
# Read the gcc version to see what ABI to build for
-if [[ $BUILD_ENVIRONMENT == *gcc4.8* ]]; then
- GCC_USE_C11=0
- GCC_VERSION='4.8'
-fi
-if [ "$(uname)" != 'Darwin' -a -z "${GCC_USE_C11}" ]; then
+if [[ "$(uname)" != 'Darwin' ]]; then
GCC_VERSION="$(gcc --version | grep --only-matching '[0-9]\.[0-9]\.[0-9]*' | head -1)"
- if [[ "$GCC_VERSION" == 4* ]]; then
- GCC_USE_C11=0
- else
- GCC_USE_C11=1
- fi
fi
-
+if [[ "$GCC_VERSION" == 4* ]]; then
+ GCC_USE_C11=0
+else
+ GCC_USE_C11=1
+fi
# Read the python version
# Specifically 3.6 because the latest Anaconda version is 3.6, and so it's site
# packages have 3.6 in the name
@@ -77,16 +147,14 @@
#
# Pick the correct conda-build folder
#
-CAFFE2_CONDA_BUILD_DIR="${CAFFE2_ROOT}/conda/caffe2"
+PYTORCH_ROOT="$( cd "$(dirname "$0")"/.. ; pwd -P)"
+CAFFE2_CONDA_BUILD_DIR="${PYTORCH_ROOT}/conda/caffe2"
if [[ "${BUILD_ENVIRONMENT}" == *full* ]]; then
- CAFFE2_CONDA_BUILD_DIR="${CAFFE2_CONDA_BUILD_DIR}/cuda_full"
-elif [[ "${BUILD_ENVIRONMENT}" == *cuda* ]]; then
- CAFFE2_CONDA_BUILD_DIR="${CAFFE2_CONDA_BUILD_DIR}/cuda"
+ CAFFE2_CONDA_BUILD_DIR="${CAFFE2_CONDA_BUILD_DIR}/full"
else
- CAFFE2_CONDA_BUILD_DIR="${CAFFE2_CONDA_BUILD_DIR}/no_cuda"
+ CAFFE2_CONDA_BUILD_DIR="${CAFFE2_CONDA_BUILD_DIR}/normal"
fi
META_YAML="${CAFFE2_CONDA_BUILD_DIR}/meta.yaml"
-CONDA_BUILD_CONFIG_YAML="${CAFFE2_CONDA_BUILD_DIR}/conda_build_config.yaml"
#
@@ -99,7 +167,7 @@
# the package name in meta.yaml based off of these values, we let Caffe2
# take the CUDA and cuDNN versions that it finds in the build environment,
# and manually set the package name ourself.
- CAFFE2_PACKAGE_NAME="${CAFFE2_PACKAGE_NAME}-cuda${CAFFE2_CUDA_VERSION}-cudnn${CAFFE2_CUDNN_VERSION}"
+ CAFFE2_PACKAGE_NAME="${CAFFE2_PACKAGE_NAME}-cuda${CUDA_VERSION}-cudnn${CUDNN_VERSION}"
fi
if [[ "$(uname)" != 'Darwin' ]]; then
if [[ $GCC_USE_C11 -eq 0 ]]; then
@@ -114,7 +182,7 @@
#
-# Handle skipping tests and uploading
+# Handle skipping tests and uploading built packages to Anaconda.org
#
# If skipping tests, remove the test related lines from the meta.yaml and don't
# upload to Anaconda.org
@@ -122,7 +190,6 @@
portable_sed '/test:/d' "${META_YAML}"
portable_sed '/imports:/d' "${META_YAML}"
portable_sed '/caffe2.python.core/d' "${META_YAML}"
-
elif [ -n "$UPLOAD_TO_CONDA" ]; then
# Upload to Anaconda.org if needed. This is only allowed if testing is
# enabled
@@ -132,29 +199,54 @@
#
-# Change flags based on target gcc ABI
+# Set flags and package requirements
#
+# Add normal packages for all builds
+add_package glog
+add_package gflags
+add_package leveldb
+add_package lmdb
+add_package opencv
+#
+# Handle CUDA related flags
+if [[ -n $CUDA_VERSION ]]; then
+ CMAKE_BUILD_ARGS+=("-DUSE_CUDA=ON")
+ CMAKE_BUILD_ARGS+=("-DUSE_NCCL=ON")
+ #add_feature $CUDA_FEATURE_NAME
+ #add_package $CUDA_FEATURE_NAME
+ #add_feature nccl2
+ #CAFFE2_CONDA_CHANNEL+=('-c pytorch')
+else
+ CMAKE_BUILD_ARGS+=("-DUSE_CUDA=OFF")
+ CMAKE_BUILD_ARGS+=("-DUSE_NCCL=OFF")
+ CMAKE_BUILD_ARGS+=("-DBLAS=MKL")
+ add_package 'mkl'
+ add_package 'mkl-include'
+fi
+#
+# Change flags based on target gcc ABI
if [[ "$(uname)" != 'Darwin' ]]; then
if [ "$GCC_USE_C11" -eq 0 ]; then
- CMAKE_BUILD_ARGS+=("-DCMAKE_CXX_FLAGS=-D_GLIBCXX_USE_CXX11_ABI=0")
- # Default conda channels use gcc 7.2 (for recent packages), conda-forge uses
- # gcc 4.8.5
- CAFFE2_CONDA_CHANNEL='-c conda-forge'
-
- # opencv 3.3.1 in conda-forge doesn't have imgcodecs
+ # opencv 3.3.1 in conda-forge doesn't have imgcodecs, and opencv 3.1.0
+ # requires numpy 1.12
add_package 'opencv' '==3.1.0'
if [[ "$PYTHON_VERSION" == 3.* ]]; then
- # opencv 3.1.0 for python 3 requires numpy 1.12
add_package 'numpy' '>1.11'
fi
+ # Default conda channels use gcc 7.2 (for recent packages), conda-forge uses
+ # gcc 4.8.5
+ CMAKE_BUILD_ARGS+=("-DCMAKE_CXX_FLAGS=-D_GLIBCXX_USE_CXX11_ABI=0")
+ CAFFE2_CONDA_CHANNEL+=('-c conda-forge')
fi
fi
+
+#
# Build Caffe2 with conda-build
#
# If --user and --token are set, then this will also upload the built package
# to Anaconda.org, provided there were no failures and all the tests passed
-CONDA_CMAKE_BUILD_ARGS="$CMAKE_BUILD_ARGS" conda build "${CAFFE2_CONDA_BUILD_DIR}" $CAFFE2_CONDA_CHANNEL ${CONDA_BUILD_ARGS[@]} "$@"
+CONDA_CMAKE_BUILD_ARGS=${CMAKE_BUILD_ARGS[@]} conda build "${CAFFE2_CONDA_BUILD_DIR}" $CAFFE2_CONDA_CHANNEL ${CONDA_BUILD_ARGS[@]} "$@"
# Install Caffe2 from the built package into the local conda environment
if [ -n "$CONDA_INSTALL_LOCALLY" ]; then
