| # Copyright (C) 2022 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| function gettop |
| { |
| local TOPFILE=build/make/core/envsetup.mk |
| # The ${TOP-} expansion allows this to work even with set -u |
| if [ -n "${TOP:-}" -a -f "${TOP:-}/$TOPFILE" ] ; then |
| # The following circumlocution ensures we remove symlinks from TOP. |
| (cd "$TOP"; PWD= /bin/pwd) |
| else |
| if [ -f $TOPFILE ] ; then |
| # The following circumlocution (repeated below as well) ensures |
| # that we record the true directory name and not one that is |
| # faked up with symlink names. |
| PWD= /bin/pwd |
| else |
| local HERE=$PWD |
| local T= |
| while [ \( ! \( -f $TOPFILE \) \) -a \( "$PWD" != "/" \) ]; do |
| \cd .. |
| T=`PWD= /bin/pwd -P` |
| done |
| \cd "$HERE" |
| if [ -f "$T/$TOPFILE" ]; then |
| echo "$T" |
| fi |
| fi |
| fi |
| } |
| |
| # Asserts that the root of the tree can be found. |
| if [ -z "${IMPORTING_ENVSETUP:-}" ] ; then |
| function require_top |
| { |
| TOP=$(gettop) |
| if [[ ! $TOP ]] ; then |
| echo "Can not locate root of source tree. $(basename $0) must be run from within the Android source tree or TOP must be set." >&2 |
| exit 1 |
| fi |
| } |
| fi |
| |
| # Asserts that the lunch variables have been set |
| if [ -z "${IMPORTING_ENVSETUP:-}" ] ; then |
| function require_lunch |
| { |
| if [[ ! $TARGET_PRODUCT || ! $TARGET_RELEASE || ! $TARGET_BUILD_VARIANT ]] ; then |
| echo "Please run lunch and try again." >&2 |
| exit 1 |
| fi |
| } |
| fi |
| |
| function set_network_file_system_type_env_var() { |
| local top=$(gettop) |
| local out_dir=$(getoutdir) |
| local cartfs_mount_point=$(cartfs_mount_point) |
| |
| local nfs_type=local |
| |
| # The options are: |
| # - cog-cartfs-symlink: out is a symlink to a CartFS path in a Cog workspace. |
| # - local-cartfs-symlink: out is a symlink to a CartFS path in a local workspace. |
| # - cog-symlink: $top starts with /google/cog. |
| # - abfs: .abfs.sock exists in the workspace. |
| if [[ -n "$cartfs_mount_point" && -L "$out_dir" && "$(readlink "$out_dir")" =~ ^/google/cartfs/mount ]]; then |
| if [[ "$top" =~ ^/google/cog ]]; then |
| nfs_type=cog-cartfs-symlink |
| else |
| nfs_type=local-cartfs-symlink |
| fi |
| elif [[ "$top" =~ ^/google/cog ]]; then |
| nfs_type=cog-symlink |
| elif [[ -f "$top/.abfs.sock" ]]; then |
| nfs_type=abfs |
| fi |
| |
| export NETWORK_FILE_SYSTEM_TYPE=$nfs_type |
| } |
| |
| # This function sets up the build environment to be appropriate for Cog. |
| function setup_cog_env_if_needed() { |
| local top=$(gettop) |
| |
| # return early if not in a cog workspace |
| if [[ ! "$top" =~ ^/google/cog ]]; then |
| return 0 |
| fi |
| |
| clean_deleted_workspaces_in_cartfs |
| |
| setup_cog_symlink |
| |
| export ANDROID_BUILD_ENVIRONMENT_CONFIG="googler-cog" |
| |
| # Running repo command within Cog workspaces is not supported, so override |
| # it with this function. If the user is running repo within a Cog workspace, |
| # we'll fail with an error, otherwise, we run the original repo command with |
| # the given args. |
| if ! ORIG_REPO_PATH=`which repo`; then |
| return 0 |
| fi |
| function repo { |
| if [[ "${PWD}" == /google/cog/* ]]; then |
| echo -e "\e[01;31mERROR:\e[0mrepo command is disallowed within Cog workspaces." |
| kill -INT $$ # exits the script without exiting the user's shell |
| fi |
| ${ORIG_REPO_PATH} "$@" |
| } |
| } |
| |
| # creates a symlink for the out/ dir when inside a cog workspace. |
| function setup_cog_symlink() { |
| local out_dir=$(getoutdir) |
| local top=$(gettop) |
| |
| # return early if out dir is already a symlink. |
| if [[ -L "$out_dir" ]]; then |
| destination=$(readlink "$out_dir") |
| # ensure the destination exists. |
| mkdir -p "$destination" |
| return 0 |
| fi |
| |
| # return early if out dir is not in the workspace |
| if [[ ! "$out_dir" =~ ^$top/ ]]; then |
| return 0 |
| fi |
| |
| local link_destination="${HOME}/.cog/android-build-out" |
| |
| # When cartfs is mounted, use it as the destination for output directory. |
| local cartfs_mount_point=$(cartfs_mount_point) |
| if [[ -n "$cartfs_mount_point" ]]; then |
| local cog_workspace_name="$(basename "$(dirname "${top}")")" |
| link_destination="${cartfs_mount_point}/${cog_workspace_name}/out" |
| setup_cartfs_incremental_build "${link_destination}" "${cartfs_mount_point}" |
| else |
| # If CartFS is not mounted, check to see if it is installed (no mount but |
| # being installed implies it was disabled). If it is installed, then don't |
| # display any messages to the user. If it isn't installed, then message the |
| # user to install it, but don't stop the script. |
| if ! command -v cartfs &> /dev/null; then |
| echo "***" |
| echo "🚨 Install CartFS for more reliable builds. See go/cartfs-with-cog for installation instructions! 🚨" |
| echo "***" |
| fi |
| fi |
| |
| # remove existing out/ dir if it exists |
| if [[ -d "$out_dir" ]]; then |
| echo "Detected existing out/ directory in the Cog workspace which is not supported. Repairing workspace by removing it and creating the symlink to ${link_destination}" |
| if ! rm -rf "$out_dir"; then |
| echo "Failed to remove existing out/ directory: $out_dir" >&2 |
| kill -INT $$ # exits the script without exiting the user's shell |
| fi |
| fi |
| |
| # create symlink |
| echo "Creating symlink: $out_dir -> $link_destination" |
| mkdir -p ${link_destination} |
| if ! ln -s "$link_destination" "$out_dir"; then |
| echo "Failed to create cog symlink: $out_dir -> $link_destination" >&2 |
| kill -INT $$ # exits the script without exiting the user's shell |
| fi |
| } |
| |
| function getoutdir |
| { |
| local top=$(gettop) |
| local out_dir="${OUT_DIR:-}" |
| if [[ -z "${out_dir}" ]]; then |
| if [[ -n "${OUT_DIR_COMMON_BASE:-}" && -n "${top}" ]]; then |
| out_dir="${OUT_DIR_COMMON_BASE}/$(basename ${top})" |
| else |
| out_dir="out" |
| fi |
| fi |
| if [[ "${out_dir}" != /* ]]; then |
| out_dir="${top}/${out_dir}" |
| fi |
| echo "${out_dir}" |
| } |
| |
| # Pretty print the build status and duration |
| function _wrap_build() |
| { |
| if [[ "${ANDROID_QUIET_BUILD:-}" == true ]]; then |
| "$@" |
| return $? |
| fi |
| local start_time=$(date +"%s") |
| "$@" |
| local ret=$? |
| local end_time=$(date +"%s") |
| local tdiff=$(($end_time-$start_time)) |
| local hours=$(($tdiff / 3600 )) |
| local mins=$((($tdiff % 3600) / 60)) |
| local secs=$(($tdiff % 60)) |
| local ncolors=$(tput colors 2>/dev/null) |
| if [ -n "$ncolors" ] && [ $ncolors -ge 8 ]; then |
| color_failed=$'\E'"[0;31m" # red |
| color_success=$'\E'"[0;32m" # green |
| color_warning=$'\E'"[0;33m" # yellow |
| color_info=$'\E'"[0;36m" # cyan |
| color_reset=$'\E'"[00m" |
| else |
| color_failed="" |
| color_success="" |
| color_warning="" |
| color_info="" |
| color_reset="" |
| fi |
| |
| echo |
| if [ $ret -eq 0 ] ; then |
| echo -n "${color_success}#### build completed successfully " |
| else |
| echo -n "${color_failed}#### failed to build some targets " |
| fi |
| if [ $hours -gt 0 ] ; then |
| printf "(%02d:%02d:%02d (hh:mm:ss))" $hours $mins $secs |
| elif [ $mins -gt 0 ] ; then |
| printf "(%02d:%02d (mm:ss))" $mins $secs |
| elif [ $secs -gt 0 ] ; then |
| printf "(%d seconds)" $secs |
| fi |
| echo " ####" |
| # Because SOONG_PARTIAL_COMPILE and SOONG_USE_PARTIAL_COMPILE are set via |
| # ANDROID_BUILD_ENVIRONMENT, we only have access to values explicitly set by the user. |
| if [[ ${TARGET_BUILD_VARIANT} = eng ]] && [[ $SOONG_USE_PARTIAL_COMPILE == false ]]; then |
| echo "${color_info}Partial compilation was disabled due to SOONG_USE_PARTIAL_COMPILE=false" |
| echo "See http://go/soong-partial-compile" |
| fi |
| if [[ ${SOONG_INCREMENTAL_ANALYSIS#true} = ${SOONG_INCREMENTAL_ANALYSIS} ]]; then |
| echo "${color_info}Try enabling incremental analysis for faster builds after changing Android.bp files." |
| echo "See http://go/soong-incremental-analysis" |
| fi |
| echo -n "${color_reset}" |
| |
| echo |
| return $ret |
| } |
| |
| |
| function log_tool_invocation() |
| { |
| if [[ -z $ANDROID_TOOL_LOGGER ]]; then |
| return |
| fi |
| |
| LOG_TOOL_TAG=$1 |
| LOG_START_TIME=$(date +%s.%N) |
| trap ' |
| exit_code=$?; |
| # Remove the trap to prevent duplicate log. |
| trap - EXIT; |
| $ANDROID_TOOL_LOGGER \ |
| --tool_tag="${LOG_TOOL_TAG}" \ |
| --start_timestamp="${LOG_START_TIME}" \ |
| --end_timestamp="$(date +%s.%N)" \ |
| --tool_args="$*" \ |
| --exit_code="${exit_code}" \ |
| ${ANDROID_TOOL_LOGGER_EXTRA_ARGS} \ |
| > /dev/null 2>&1 & |
| exit ${exit_code} |
| ' SIGINT SIGTERM SIGQUIT EXIT |
| } |
| |
| # Import the build variables supplied as arguments into this shell's environment. |
| # For absolute variables, prefix the variable name with a '/'. For example: |
| # import_build_vars OUT_DIR DIST_DIR /HOST_OUT_EXECUTABLES |
| # Returns nonzero if the build command failed. Stderr is passed through. |
| function import_build_vars() |
| { |
| require_top |
| local script |
| script=$(cd $TOP && build/soong/bin/get_build_vars "$@") |
| local ret=$? |
| if [ $ret -ne 0 ] ; then |
| return $ret |
| fi |
| eval "$script" |
| return $? |
| } |
| |
| function cartfs_mount_point() { |
| if cartfs --is_running &>/dev/null; then |
| # TODO(b/465427340): Read the mount point from the output once CartFS |
| # exposes it. CartFS team is aware of this plan and will not change the |
| # mount point until we can programmatically read it. |
| echo "/google/cartfs/mount" |
| return |
| fi |
| |
| # Fallback to the old findmnt while we wait for all users to update to the |
| # latest CartFS version. |
| # TODO(b/465427340): Remove this fallback at the same time we start reading |
| # the mount point from CartFS. |
| |
| # Make sure findmnt is installed. |
| if ! command -v findmnt &> /dev/null; then |
| return |
| fi |
| |
| local cartfs_user_id="$(id -u cartfs 2>/dev/null)" |
| |
| # To find the main CartFS mount point, filter mounts by cartfs user_id and |
| # look for the one where source is just /dev/fuse - that excludes bind mounts |
| # that may originate in CartFS. |
| local cartfs_mount_point="$(findmnt -t fuse -O "user_id=${cartfs_user_id}" --output "target,source" --raw | grep '/dev/fuse$' | tail -n +1 | awk '{print $1}')" |
| # Making sure $cartfs_user_id is not empty since findmnt will return mounts |
| # started by root when it is. |
| if [[ -n "$cartfs_user_id" ]] && [[ -n "$cartfs_mount_point" ]] && findmnt "$cartfs_mount_point" >/dev/null 2>&1; then |
| echo "$cartfs_mount_point" |
| fi |
| } |
| |
| # Deletes cartfs folders that are mapped to deleted workspaces. |
| function clean_deleted_workspaces_in_cartfs() { |
| local cartfs_mount_point=$(cartfs_mount_point) |
| if [[ -n "$cartfs_mount_point" ]]; then |
| local folders_list |
| folders_list=$(find "$cartfs_mount_point" -maxdepth 1 -type d) |
| if [[ -n "$folders_list" ]]; then |
| local log_file="${HOME}/.cartfs/cartfs_workspace_deletion.log" |
| mkdir -p "$(dirname "${log_file}")" |
| while read -r folder; do |
| if [[ "$folder" != "$cartfs_mount_point" ]]; then |
| local workspace_name="$(basename "${folder}")" |
| local workspaces_path="$(dirname "$(dirname "${top}")")" |
| local full_path="${workspaces_path}/${workspace_name}" |
| if [[ ! -d "${full_path}" ]]; then |
| local log_timestamp=$(date +"%Y-%m-%d %H:%M:%S") |
| echo "${log_timestamp}: The workspace ${workspace_name} does not exist, deleting ${folder} from cartfs" >> "${log_file}" |
| rm -Rf "${folder}" & |
| fi |
| fi |
| done <<< "$folders_list" |
| fi |
| fi |
| } |
| |
| # Configure the output directory of the new workspace in Cartfs to be an |
| # incremental build by copying the build output of a previous build of the same |
| # repo and forking the mtimes in Cog. |
| function setup_cartfs_incremental_build() { |
| # Make sure grpc_cli is installed. |
| if ! command -v grpc_cli &> /dev/null; then |
| return |
| fi |
| |
| local link_destination=$1 |
| if [[ -d "$link_destination" ]]; then |
| return |
| fi |
| |
| local cartfs_endpoint="127.0.0.1:65001" |
| local cartfs_rpc_copy_directory="cartfs.Cartfs.CopyDirectory" |
| |
| # Make sure CartFS is listening on the endpoint and that the CopyDirectory |
| # function is available. |
| if ! grpc_cli ls ${cartfs_endpoint} ${cartfs_rpc_copy_directory} --channel_creds_type=insecure &> /dev/null; then |
| return |
| fi |
| |
| local cogfsd_endpoint="unix:///google/cog/status/uds/${UID}" |
| local cogfsd_rpc_forkmtimes="devtools_srcfs.CogLocalRpcService.ForkMtimes" |
| |
| # Make sure Cogfsd is listening on the endpoint and that the ForMtimes |
| # function is available. |
| if ! grpc_cli ls ${cogfsd_endpoint} ${cogfsd_rpc_forkmtimes} --channel_creds_type=insecure &> /dev/null; then |
| return |
| fi |
| |
| echo "Searching for recent build outputs in CartFS for incremental builds" |
| |
| local cartfs_mount_point=$2 |
| local top=$(gettop) |
| local repo="$(basename "${top}")" |
| local current_cartfs_folder="$(dirname "${link_destination}")" |
| local target_workspace="$(basename "${current_cartfs_folder}")" |
| local source_workspace="" |
| local copy_from="" |
| local folders="" |
| |
| folders=$(stat -c "%Y %n" ${cartfs_mount_point}/*/out 2>/dev/null | sort -nr | sed 's/^[0-9]\+ //') |
| if [[ -n "${folders}" ]]; then |
| while read -r cartfs_out; do |
| if [[ "${cartfs_out}" != "${link_destination}" ]]; then |
| local workspace_name="$(basename "$(dirname "${cartfs_out}")")" |
| local workspace_path="$(dirname "$(dirname "${top}")")" |
| local full_path="${workspace_path}/${workspace_name}/${repo}" |
| |
| # Make sure the CoG workspace still exists and is of the same repo. |
| if [[ ! -d "${full_path}" ]]; then |
| continue |
| fi |
| |
| # Make sure the out directory exists, is not empty, and we can stat it. |
| if [[ -d "${cartfs_out}" && -n "$(ls -A "${cartfs_out}")" ]]; then |
| copy_from="${cartfs_out}" |
| source_workspace="${workspace_name}" |
| break |
| fi |
| fi |
| done <<< "$folders" |
| fi |
| |
| if [[ -n "$copy_from" ]]; then |
| echo "Found suitable build output from workspace ${source_workspace}" |
| echo "Copying content from $copy_from to $link_destination" |
| # Filtering output to only include relevant information. |
| local output_filter="connecting to|Rpc succeeded with OK status|Not yet implemented|success|Received trailing metadata from server" |
| if ! grpc_cli call ${cogfsd_endpoint} ${cogfsd_rpc_forkmtimes} \ |
| 'source_workspace: "'${source_workspace}'" |
| target_workspace: "'${target_workspace}'"' \ |
| --channel_creds_type=insecure 2>&1 | grep -v -E "${output_filter}"; then |
| return 1 |
| fi |
| if ! grpc_cli call ${cartfs_endpoint} ${cartfs_rpc_copy_directory} \ |
| 'from_path: "'${copy_from#$cartfs_mount_point/}'" |
| to_path: "'${link_destination#$cartfs_mount_point/}'"' \ |
| --channel_creds_type=insecure 2>&1 | grep -v -E "${output_filter}"; then |
| return 1 |
| fi |
| # Adding a file to track that the workspace is an incremental |
| # build from Cartfs. This will be used for metrics. |
| echo "${source_workspace}" > "${link_destination}/.cartfs-copied" |
| # Don't carry over the leftovers file from previous workspace. |
| rm -f "${link_destination}/.leftovers" |
| else |
| echo "No suitable build outputs matching the same repository found." |
| echo "Starting from a fresh build output directory." |
| fi |
| } |
