| #!/bin/bash |
| |
| set -eu |
| |
| # Copyright 2020 Google Inc. All rights reserved. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| # Tool to evaluate the transitive closure of the ninja dependency graph of the |
| # files and targets a given target depends on. |
| # |
| # i.e. the list of things that, if changed, could cause a change to a target. |
| |
| readonly me=$(basename "${0}") |
| |
| readonly usage="usage: ${me} {options} target [target...] |
| |
| Evaluate the transitive closure of files and ninja targets that one or more |
| targets depend on. |
| |
| Dependency Options: |
| |
| -(no)order_deps Whether to include order-only dependencies. (Default false) |
| -(no)implicit Whether to include implicit dependencies. (Default true) |
| -(no)explicit Whether to include regular / explicit deps. (Default true) |
| |
| -nofollow Unanchored regular expression. Matching paths and targets |
| always get reported. Their dependencies do not get reported |
| unless first encountered in a 'container' file type. |
| Multiple allowed and combined using '|'. |
| e.g. -nofollow='*.so' not -nofollow='.so$' |
| -nofollow='*.so|*.dex' or -nofollow='*.so' -nofollow='.dex' |
| (Defaults to no matches) |
| -container Unanchored regular expression. Matching file extensions get |
| treated as 'container' files for -nofollow option. |
| Multiple allowed and combines using '|' |
| (Default 'apex|apk|zip|jar|tar|tgz') |
| |
| Output Options: |
| |
| -(no)quiet Suppresses progress output to stderr and interactive |
| alias -(no)q prompts. By default, when stderr is a tty, progress gets |
| reported to stderr; when both stderr and stdin are tty, |
| the script asks user whether to delete intermediate files. |
| When suppressed or not prompted, script always deletes the |
| temporary / intermediate files. |
| -sep=<delim> Use 'delim' as output field separator between notice |
| checksum and notice filename in notice output. |
| e.g. sep='\\t' |
| (Default space) |
| -csv Shorthand for -sep=',' |
| -directories=<f> Output directory names of dependencies to 'f'. |
| alias -d User '/dev/stdout' to send directories to stdout. Defaults |
| to no directory output. |
| -notices=<file> Output license and notice file paths to 'file'. |
| alias -n Use '/dev/stdout' to send notices to stdout. Defaults to no |
| license/notice output. |
| -projects=<file> Output git project names to 'file'. Use '/dev/stdout' to |
| alias -p send projects to stdout. Defaults to no project output. |
| -targets=<fils> Output target dependencies to 'file'. Use '/dev/stdout' to |
| alias -t send targets to stdout. |
| When no directory, notice, project or target output options |
| given, defaults to stdout. Otherwise, defaults to no target |
| output. |
| |
| At minimum, before running this script, you must first run: |
| $ source build/envsetup.sh |
| $ lunch |
| $ m nothing |
| to setup the build environment, choose a target platform, and build the ninja |
| dependency graph. |
| " |
| |
| function die() { echo -e "${*}" >&2; exit 2; } |
| |
| # Reads one input target per line from stdin; outputs (isnotice target) tuples. |
| # |
| # output target is a ninja target that the input target depends on |
| # isnotice in {0,1} with 1 for output targets believed to be license or notice |
| function getDeps() { |
| (tr '\n' '\0' | xargs -0 -r "${ninja_bin}" -f "${ninja_file}" -t query) \ |
| | awk -v include_order="${include_order_deps}" \ |
| -v include_implicit="${include_implicit_deps}" \ |
| -v include_explicit="${include_deps}" \ |
| -v containers="${container_types}" \ |
| ' |
| BEGIN { |
| ininput = 0 |
| isnotice = 0 |
| currFileName = "" |
| currExt = "" |
| } |
| $1 == "outputs:" || $1 == "validations:" { |
| ininput = 0 |
| } |
| ininput == 0 && $0 ~ /^\S\S*:$/ { |
| isnotice = ($0 ~ /.*NOTICE.*[.]txt:$/) |
| currFileName = gensub(/^.*[/]([^/]*)[:]$/, "\\1", "g") |
| currExt = gensub(/^.*[.]([^./]*)[:]$/, "\\1", "g") |
| } |
| ininput != 0 && $1 !~ /^[|][|]?/ { |
| if (include_explicit == "true") { |
| fileName = gensub(/^.*[/]([^/]*)$/, "\\1", "g") |
| print ( \ |
| (isnotice && $0 !~ /^\s*build[/]soong[/]scripts[/]/) \ |
| || $0 ~ /NOTICE|LICEN[CS]E/ \ |
| || $0 ~ /(notice|licen[cs]e)[.]txt/ \ |
| )" "(fileName == currFileName||currExt ~ "^(" containers ")$")" "gensub(/^\s*/, "", "g") |
| } |
| } |
| ininput != 0 && $1 == "|" { |
| if (include_implicit == "true") { |
| fileName = gensub(/^.*[/]([^/]*)$/, "\\1", "g") |
| $1 = "" |
| print ( \ |
| (isnotice && $0 !~ /^\s*build[/]soong[/]scripts[/]/) \ |
| || $0 ~ /NOTICE|LICEN[CS]E/ \ |
| || $0 ~ /(notice|licen[cs]e)[.]txt/ \ |
| )" "(fileName == currFileName||currExt ~ "^(" containers ")$")" "gensub(/^\s*/, "", "g") |
| } |
| } |
| ininput != 0 && $1 == "||" { |
| if (include_order == "true") { |
| fileName = gensub(/^.*[/]([^/]*)$/, "\\1", "g") |
| $1 = "" |
| print ( \ |
| (isnotice && $0 !~ /^\s*build[/]soong[/]scripts[/]/) \ |
| || $0 ~ /NOTICE|LICEN[CS]E/ \ |
| || $0 ~ /(notice|licen[cs]e)[.]txt/ \ |
| )" "(fileName == currFileName||currExt ~ "^(" containers ")$")" "gensub(/^\s*/, "", "g") |
| } |
| } |
| $1 == "input:" { |
| ininput = 1 |
| } |
| ' |
| } |
| |
| # Reads one input directory per line from stdin; outputs unique git projects. |
| function getProjects() { |
| while read d; do |
| while [ "${d}" != '.' ] && [ "${d}" != '/' ]; do |
| if [ -d "${d}/.git/" ]; then |
| echo "${d}" |
| break |
| fi |
| d=$(dirname "${d}") |
| done |
| done | sort -u |
| } |
| |
| |
| if [ -z "${ANDROID_BUILD_TOP}" ]; then |
| die "${me}: Run 'lunch' to configure the build environment" |
| fi |
| |
| if [ -z "${TARGET_PRODUCT}" ]; then |
| die "${me}: Run 'lunch' to configure the build environment" |
| fi |
| |
| readonly ninja_file="${ANDROID_BUILD_TOP}/out/combined-${TARGET_PRODUCT}.ninja" |
| if [ ! -f "${ninja_file}" ]; then |
| die "${me}: Run 'm nothing' to build the dependency graph" |
| fi |
| |
| readonly ninja_bin="${ANDROID_BUILD_TOP}/prebuilts/build-tools/linux-x86/bin/ninja" |
| if [ ! -x "${ninja_bin}" ]; then |
| die "${me}: Cannot find ninja executable expected at ${ninja_bin}" |
| fi |
| |
| |
| # parse the command-line |
| |
| declare -a targets # one or more targets to evaluate |
| |
| include_order_deps=false # whether to trace through || "order dependencies" |
| include_implicit_deps=true # whether to trace through | "implicit deps" |
| include_deps=true # whether to trace through regular explicit deps |
| quiet=false # whether to suppress progress |
| |
| projects_out='' # where to output the list of projects |
| directories_out='' # where to output the list of directories |
| targets_out='' # where to output the list of targets/source files |
| notices_out='' # where to output the list of license/notice files |
| |
| sep=" " # separator between md5sum and notice filename |
| |
| nofollow='' # regularexp must fully match targets to skip |
| |
| container_types='' # regularexp must full match file extension |
| # defaults to 'apex|apk|zip|jar|tar|tgz' below. |
| |
| use_stdin=false # whether to read targets from stdin i.e. target - |
| |
| while [ $# -gt 0 ]; do |
| case "${1:-}" in |
| -) |
| use_stdin=true |
| ;; |
| -*) |
| flag=$(expr "${1}" : '^-*\(.*\)$') |
| case "${flag:-}" in |
| order_deps) |
| include_order_deps=true;; |
| noorder_deps) |
| include_order_deps=false;; |
| implicit) |
| include_implicit_deps=true;; |
| noimplicit) |
| include_implicit_deps=false;; |
| explicit) |
| include_deps=true;; |
| noexplicit) |
| include_deps=false;; |
| csv) |
| sep=",";; |
| sep) |
| sep="${2?"${usage}"}"; shift;; |
| sep=) |
| sep=$(expr "${flag}" : '^sep=\(.*\)$');; |
| q) ;& |
| quiet) |
| quiet=true;; |
| noq) ;& |
| noquiet) |
| quiet=false;; |
| nofollow) |
| case "${nofollow}" in |
| '') |
| nofollow="${2?"${usage}"}";; |
| *) |
| nofollow="${nofollow}|${2?"${usage}"}";; |
| esac |
| shift |
| ;; |
| nofollow=*) |
| case "${nofollow}" in |
| '') |
| nofollow=$(expr "${flag}" : '^nofollow=\(.*\)$');; |
| *) |
| nofollow="${nofollow}|"$(expr "${flag}" : '^nofollow=\(.*\)$');; |
| esac |
| ;; |
| container) |
| container_types="${container_types}|${2?"${usage}"}";; |
| container=*) |
| container_types="${container_types}|"$(expr "${flag}" : '^container=\(.*\)$');; |
| p) ;& |
| projects) |
| projects_out="${2?"${usage}"}"; shift;; |
| p=*) ;& |
| projects=*) |
| projects_out=$(expr "${flag}" : '^.*=\(.*\)$');; |
| d) ;& |
| directores) |
| directories_out="${2?"${usage}"}"; shift;; |
| d=*) ;& |
| directories=*) |
| directories_out=$(expr "${flag}" : '^.*=\(.*\)$');; |
| t) ;& |
| targets) |
| targets_out="${2?"${usage}"}"; shift;; |
| t=*) ;& |
| targets=) |
| targets_out=$(expr "${flag}" : '^.*=\(.*\)$');; |
| n) ;& |
| notices) |
| notices_out="${2?"${usage}"}"; shift;; |
| n=*) ;& |
| notices=) |
| notices_out=$(expr "${flag}" : '^.*=\(.*\)$');; |
| *) |
| die "${usage}\n\nUnknown flag ${1}";; |
| esac |
| ;; |
| *) |
| targets+=("${1:-}") |
| ;; |
| esac |
| shift |
| done |
| |
| |
| # fail fast if command-line arguments are invalid |
| |
| if [ ! -v targets[0] ] && ! ${use_stdin}; then |
| die "${usage}\n\nNo target specified." |
| fi |
| |
| if [ -z "${projects_out}" ] \ |
| && [ -z "${directories_out}" ] \ |
| && [ -z "${targets_out}" ] \ |
| && [ -z "${notices_out}" ] |
| then |
| targets_out='/dev/stdout' |
| fi |
| |
| if [ -z "${container_types}" ]; then |
| container_types='apex|apk|zip|jar|tar|tgz' |
| fi |
| |
| # showProgress when stderr is a tty |
| if [ -t 2 ] && ! ${quiet}; then |
| showProgress=true |
| else |
| showProgress=false |
| fi |
| |
| # interactive when both stderr and stdin are tty |
| if ${showProgress} && [ -t 0 ]; then |
| interactive=true |
| else |
| interactive=false |
| fi |
| |
| |
| readonly tmpFiles=$(mktemp -d "${TMPDIR}.tdeps.XXXXXXXXX") |
| if [ -z "${tmpFiles}" ]; then |
| die "${me}: unable to create temporary directory" |
| fi |
| |
| # The deps files contain unique (isnotice target) tuples where |
| # isnotice in {0,1} with 1 when ninja target 'target' is a license or notice. |
| readonly oldDeps="${tmpFiles}/old" |
| readonly newDeps="${tmpFiles}/new" |
| readonly allDeps="${tmpFiles}/all" |
| |
| if ${use_stdin}; then # start deps by reading 1 target per line from stdin |
| awk ' |
| NF > 0 { |
| print ( \ |
| $0 ~ /NOTICE|LICEN[CS]E/ \ |
| || $0 ~ /(notice|licen[cs]e)[.]txt/ \ |
| )" "gensub(/\s*$/, "", "g", gensub(/^\s*/, "", "g")) |
| } |
| ' > "${newDeps}" |
| else # start with no deps by clearing file |
| : > "${newDeps}" |
| fi |
| |
| # extend deps by appending targets from command-line |
| for idx in "${!targets[*]}"; do |
| isnotice='0' |
| case "${targets[${idx}]}" in |
| *NOTICE*) ;& |
| *LICEN[CS]E*) ;& |
| *notice.txt) ;& |
| *licen[cs]e.txt) |
| isnotice='1';; |
| esac |
| echo "${isnotice} 1 ${targets[${idx}]}" >> "${newDeps}" |
| done |
| |
| # remove duplicates and start with new, old and all the same |
| sort -u < "${newDeps}" > "${allDeps}" |
| cp "${allDeps}" "${newDeps}" |
| cp "${allDeps}" "${oldDeps}" |
| |
| # report depth of dependenciens when showProgress |
| depth=0 |
| |
| # 1st iteration always unfiltered |
| filter='cat' |
| while [ $(wc -l < "${newDeps}") -gt 0 ]; do |
| if ${showProgress}; then |
| echo "depth ${depth} has "$(wc -l < "${newDeps}")" targets" >&2 |
| depth=$(expr ${depth} + 1) |
| fi |
| ( # recalculate dependencies by combining unique inputs of new deps w. old |
| set +e |
| sh -c "${filter}" < "${newDeps}" | cut -d\ -f3- | getDeps |
| set -e |
| cat "${oldDeps}" |
| ) | sort -u > "${allDeps}" |
| # recalculate new dependencies as net additions to old dependencies |
| set +e |
| diff "${oldDeps}" "${allDeps}" --old-line-format='' --new-line-format='%L' \ |
| --unchanged-line-format='' > "${newDeps}" |
| set -e |
| # apply filters on subsequent iterations |
| case "${nofollow}" in |
| '') |
| filter='cat';; |
| *) |
| filter="egrep -v '^[01] 0 (${nofollow})$'" |
| ;; |
| esac |
| # recalculate old dependencies for next iteration |
| cp "${allDeps}" "${oldDeps}" |
| done |
| |
| # found all deps -- clean up last iteration of old and new |
| rm -f "${oldDeps}" |
| rm -f "${newDeps}" |
| |
| if ${showProgress}; then |
| echo $(wc -l < "${allDeps}")" targets" >&2 |
| fi |
| |
| if [ -n "${targets_out}" ]; then |
| cut -d\ -f3- "${allDeps}" | sort -u > "${targets_out}" |
| fi |
| |
| if [ -n "${directories_out}" ] \ |
| || [ -n "${projects_out}" ] \ |
| || [ -n "${notices_out}" ] |
| then |
| readonly allDirs="${tmpFiles}/dirs" |
| ( |
| cut -d\ -f3- "${allDeps}" | tr '\n' '\0' | xargs -0 dirname |
| ) | sort -u > "${allDirs}" |
| if ${showProgress}; then |
| echo $(wc -l < "${allDirs}")" directories" >&2 |
| fi |
| |
| case "${directories_out}" in |
| '') : do nothing;; |
| *) |
| cat "${allDirs}" > "${directories_out}" |
| ;; |
| esac |
| fi |
| |
| if [ -n "${projects_out}" ] \ |
| || [ -n "${notices_out}" ] |
| then |
| readonly allProj="${tmpFiles}/projects" |
| set +e |
| egrep -v '^out[/]' "${allDirs}" | getProjects > "${allProj}" |
| set -e |
| if ${showProgress}; then |
| echo $(wc -l < "${allProj}")" projects" >&2 |
| fi |
| |
| case "${projects_out}" in |
| '') : do nothing;; |
| *) |
| cat "${allProj}" > "${projects_out}" |
| ;; |
| esac |
| fi |
| |
| case "${notices_out}" in |
| '') : do nothing;; |
| *) |
| readonly allNotice="${tmpFiles}/notices" |
| set +e |
| egrep '^1' "${allDeps}" | cut -d\ -f3- | egrep -v '^out/' > "${allNotice}" |
| set -e |
| cat "${allProj}" | while read proj; do |
| for f in LICENSE LICENCE NOTICE license.txt notice.txt; do |
| if [ -f "${proj}/${f}" ]; then |
| echo "${proj}/${f}" |
| fi |
| done |
| done >> "${allNotice}" |
| if ${showProgress}; then |
| echo $(cat "${allNotice}" | sort -u | wc -l)" notice targets" >&2 |
| fi |
| readonly hashedNotice="${tmpFiles}/hashednotices" |
| ( # md5sum outputs checksum space indicator(space or *) filename newline |
| set +e |
| sort -u "${allNotice}" | tr '\n' '\0' | xargs -0 -r md5sum 2>/dev/null |
| set -e |
| # use sed to replace space and indicator with separator |
| ) > "${hashedNotice}" |
| if ${showProgress}; then |
| echo $(cut -d\ -f2- "${hashedNotice}" | sort -u | wc -l)" notice files" >&2 |
| echo $(cut -d\ -f1 "${hashedNotice}" | sort -u | wc -l)" distinct notices" >&2 |
| fi |
| sed 's/^\([^ ]*\) [* ]/\1'"${sep}"'/g' "${hashedNotice}" | sort > "${notices_out}" |
| ;; |
| esac |
| |
| if ${interactive}; then |
| echo -n "$(date '+%F %-k:%M:%S') Delete ${tmpFiles} ? [n] " >&2 |
| read answer |
| case "${answer}" in [yY]*) rm -fr "${tmpFiles}";; esac |
| else |
| rm -fr "${tmpFiles}" |
| fi |
