development/referenceDocs/stageReferenceDocsWithDackka.sh

#!/bin/bash
#
# Script to fetch generated API references docs and stage them.
#
# Examples:
#
# Stage refdocs from a given build ID (to the default staging DB):
#
#   ./stageReferenceDocsWithDackka.sh --buildId 1234567
#
# Stage locally-generated refdocs (to the default staging DB) *.
# The directory path must be absolute (can't contain ~):
#
#   /stageReferenceDocsWithDackka.sh --buildId 0
#   --sourceDir=/dir/to/androidx-main/out/androidx/docs-public/build
#
# Stage ToT refdocs from a given build ID, to a specified DB, using a specific
# date string for the generated CL:
#
#   ./stageReferenceDocsWithDackka.sh --buildId 1234567 --db androidx-docs
#   --dateStr "April 29, 2021" --useToT
#
# ===
#
# * buildId still needs to be specified when staging locally-generated refdocs,
#   but the value is unused and ignored.
#
# ===
#

source gbash.sh || exit

readonly defaultDb=""
DEFINE_string buildId --required "" "The build ID from the Android build server. This is ignored when specifying the 'sourceDir' flag."
DEFINE_string dateStr "<insert date here>" "Date string used for CL message. Enclose date in double quotes (ex: \"April 29, 2021\")"
DEFINE_string db "$defaultDb" "The database used for staging. Omitting this value will stage changes to the staging DB."
DEFINE_string sourceDir "" "Local directory to fetch doc artifacts from. Directory must be absolute (can't contain ~)."
DEFINE_bool useToT false "Stage docs from tip-of-tree docs build rather than public docs build"
DEFINE_bool buildNativeDocs false "Build and stage native docs generated with doxygen"

gbash::init_google "$@"

# Allowlist for including specific directories which are generated by
# Doclava/Dokka instead of Dackka.
#
# There are separate lists for Java and Kotlin refdocs as some libraries (such
# as Compose) only publish refdocs for a single language.
#
# Each directory's spelling must match the library's directory in
# frameworks/support.
readonly javaLibraryDirsThatDontUseDackka=(
  "android/support/v4"
  "androidx/ads"
  "androidx/appcompat"
  "androidx/appsearch"
  "androidx/biometric"
  "androidx/browser"
  "androidx/camera"
  "androidx/car"
  "androidx/concurrent"
  "androidx/contentpager"
  "androidx/cursoradapter"
  "androidx/customview"
  "androidx/datastore"
  "androidx/documentfile"
  "androidx/draganddrop"
  "androidx/dynamicanimation"
  "androidx/enterprise"
  "androidx/exifinterface"
  "androidx/gridlayout"
  "androidx/heifwriter"
  "androidx/hilt"
  "androidx/leanback"
  "androidx/media"
  "androidx/media2"
  "androidx/mediarouter"
  "androidx/palette"
  "androidx/percentlayout"
  "androidx/preference"
  "androidx/print"
  "androidx/profileinstaller"
  "androidx/recommendation"
  "androidx/recyclerview"
  "androidx/remotecallback"
  "androidx/resourceinspection"
  "androidx/room"
  "androidx/security"
  "androidx/slice"
  "androidx/sqlite"
  "androidx/startup"
  "androidx/swiperefreshlayout"
  "androidx/textclassifier"
  "androidx/tracing"
  "androidx/transition"
  "androidx/tvprovider"
  "androidx/versionedparcelable"
  "androidx/webkit"
  "androidx/work"
)
readonly kotlinLibraryDirsThatDontUseDackka=(
  "android/support/v4"
  "androidx/ads"
  "androidx/appcompat"
  "androidx/appsearch"
  "androidx/benchmark"
  "androidx/biometric"
  "androidx/browser"
  "androidx/camera"
  "androidx/car"
  "androidx/concurrent"
  "androidx/contentpager"
  "androidx/cursoradapter"
  "androidx/customview"
  "androidx/datastore"
  "androidx/documentfile"
  "androidx/dynamicanimation"
  "androidx/draganddrop"
  "androidx/enterprise"
  "androidx/exifinterface"
  "androidx/glance"
  "androidx/gridlayout"
  "androidx/heifwriter"
  "androidx/hilt"
  "androidx/leanback"
  "androidx/media"
  "androidx/media2"
  "androidx/mediarouter"
  "androidx/palette"
  "androidx/percentlayout"
  "androidx/preference"
  "androidx/print"
  "androidx/profileinstaller"
  "androidx/recommendation"
  "androidx/recyclerview"
  "androidx/remotecallback"
  "androidx/resourceinspection"
  "androidx/room"
  "androidx/security"
  "androidx/slice"
  "androidx/sqlite"
  "androidx/startup"
  "androidx/swiperefreshlayout"
  "androidx/textclassifier"
  "androidx/tracing"
  "androidx/transition"
  "androidx/tvprovider"
  "androidx/versionedparcelable"
  "androidx/webkit"
  "androidx/work"
)

# Change directory to this script's location and store the directory
cd "$(dirname $0)"
scriptDirectory=$(pwd)

# Working directories for the refdocs
outDir="$scriptDirectory/out"
readonly newDir="reference-docs"
readonly dokkaNewDir="reference-docs-dokka"
readonly doclavaNewDir="reference-docs-doclava"
readonly doxygenNewDir="reference-docs-doxygen"

# Remove and recreate the existing out directory to avoid conflicts from previous runs
rm -rf $outDir
mkdir -p $outDir/$newDir
mkdir -p $outDir/$dokkaNewDir
mkdir -p $outDir/$doclavaNewDir
mkdir -p $outDir/$doxygenNewDir
cd $outDir

if [ "$FLAGS_sourceDir" == "" ]; then
  printf "=================================================================== \n"
  printf "== Download the doc zip files from the build server \n"
  printf "=================================================================== \n"

  if (( FLAGS_useToT )); then
    printf "Downloading docs-tip-of-tree zip files \n"
    androidxDoclavaZip="doclava-tip-of-tree-docs-${FLAGS_buildId}.zip"
    androidxDokkaZip="dokka-tip-of-tree-docs-${FLAGS_buildId}.zip"
    androidxDackkaZip="dackka-tip-of-tree-docs-${FLAGS_buildId}.zip"
  else
    printf "Downloading docs-public zip files \n"
    androidxDoclavaZip="doclava-public-docs-${FLAGS_buildId}.zip"
    androidxDokkaZip="dokka-public-docs-${FLAGS_buildId}.zip"
    androidxDackkaZip="dackka-public-docs-${FLAGS_buildId}.zip"
  fi

  if (( "${FLAGS_buildId::1}" == "P" )); then
    /google/data/ro/projects/android/fetch_artifact --bid $FLAGS_buildId --target androidx_incremental incremental/$androidxDoclavaZip
    /google/data/ro/projects/android/fetch_artifact --bid $FLAGS_buildId --target androidx_incremental incremental/$androidxDokkaZip
    /google/data/ro/projects/android/fetch_artifact --bid $FLAGS_buildId --target androidx_incremental incremental/$androidxDackkaZip
  else
    /google/data/ro/projects/android/fetch_artifact --bid $FLAGS_buildId --target androidx $androidxDoclavaZip
    /google/data/ro/projects/android/fetch_artifact --bid $FLAGS_buildId --target androidx $androidxDokkaZip
    /google/data/ro/projects/android/fetch_artifact --bid $FLAGS_buildId --target androidx $androidxDackkaZip
  fi

  # Let's double check we succeeded before continuing
  if [[ -f "$androidxDoclavaZip" ]] && [[ -f "$androidxDokkaZip" ]] && [[ -f "$androidxDackkaZip" ]]; then
    echo "Download completed"
  else
    printf "\n"
    printf "=================================================================== \n"
    echo "Failed to download doc zip files. Please double check your build ID and try again."
    exit 1
  fi

  printf "\n"
  printf "=================================================================== \n"
  printf "== Unzip the doc zip files \n"
  printf "=================================================================== \n"

  unzip $androidxDoclavaZip -d $doclavaNewDir
  unzip $androidxDokkaZip -d $dokkaNewDir
  unzip $androidxDackkaZip -d $newDir
else
  printf "=================================================================== \n"
  printf "== Copying doc sources from local directory $FLAGS_sourceDir \n"
  printf "=================================================================== \n"

  cp -r "$FLAGS_sourceDir/javadoc/." $doclavaNewDir
  cp -r "$FLAGS_sourceDir/dokkaKotlinDocs/." $dokkaNewDir
  cp -r "$FLAGS_sourceDir/dackkaDocs/." $newDir

fi

printf "\n"
printf "=================================================================== \n"
printf "== Copy over Doclava/Dokka generated refdocs \n"
printf "=================================================================== \n"

# The Dackka versions of the docs for these libraries are intentionally not
# removed before copying the non-Dackka versions because some files aren't
# generated by Doclava/Dokka (such as Java refdocs based on Kotlin sources)

cd $outDir
for dir in "${javaLibraryDirsThatDontUseDackka[@]}"
do
  printf "Copying Java refdocs for $dir\n"
  cp -r $doclavaNewDir/reference/$dir/* $newDir/reference/$dir
done

for dir in "${kotlinLibraryDirsThatDontUseDackka[@]}"
do
  printf "Copying Kotlin refdocs for $dir\n"
  cp -r $dokkaNewDir/reference/kotlin/$dir/* $newDir/reference/kotlin/$dir
done

if (( FLAGS_buildNativeDocs )); then
  printf "\n"
  printf "=================================================================== \n"
  printf "== Generate Doxygen docs \n"
  printf "=================================================================== \n"

  ../generateDoxygenDocs.sh
fi

printf "\n"
printf "=================================================================== \n"
printf "== Generate the language switcher \n"
printf "=================================================================== \n"

cd $newDir/reference
python3 ./../../../switcher.py --work androidx
python3 ./../../../switcher.py --work support

if (( FLAGS_buildNativeDocs )); then
  cd $outDir
  # Copy over doxygen generated refdocs after switcher is added
  rsync -avh --ignore-existing $doxygenNewDir/reference/ $newDir/reference/

  # Include doxygen toc files in main toc
  cd $newDir
  find reference -name _doxygen.yaml  -exec python3 $scriptDirectory/helpers/insert_include_into_toc.py {}  \;
fi

printf "\n"
printf "=================================================================== \n"
printf "== Create (if needed) and sync g4 workspace \n"
printf "=================================================================== \n"

client="$(p4 g4d -f androidx-ref-docs-stage)"
cd "$client"

# Revert all local changes to prevent merge conflicts when syncing.
# This is OK since we always want to start with a fresh CitC client
g4 revert ...
g4 sync

printf "\n"
printf "=================================================================== \n"
printf "== Prep directories and copy refdocs to CitC client \n"
printf "=================================================================== \n"

cd third_party/devsite/android/en/reference

cd kotlin/androidx
ls | grep -v "package\|class\|book\|toc\|constraint\|test\|index\|redirects" | xargs -I {} rm -rf {}
cd ../../androidx
ls | grep -v "package\|class\|book\|toc\|constraint\|test\|index\|redirects" | xargs -I {} rm -rf {}
cd ..

cp -r $outDir/$newDir/reference/* .

printf "\n"
printf "=================================================================== \n"
printf "== Create a changelist of pending refdoc changes \n"
printf "=================================================================== \n"

stagingLinkJava="go/dac-stage/reference/androidx/packages"
stagingLinkKotlin="go/dac-stage/reference/kotlin/androidx/packages"

# Add the db param to links if the target database is not the default staging DB.
if [ "$FLAGS_db" != "$defaultdb" ]; then
  stagingLinkJava+="?db=$FLAGS_db"
  stagingLinkKotlin+="?db=$FLAGS_db"
fi

# Construct CL description
clDesc="Androidx $FLAGS_dateStr Ref Docs

DO NOT SUBMIT

GO LIVE TIME: $FLAGS_dateStr @ 10:00 AM PST

Staged:
* Java: $stagingLinkJava
* Kotlin: $stagingLinkKotlin

All docs build id: $FLAGS_buildId

The following scripts were used to create these docs:

https://cs.android.com/androidx/platform/frameworks/support/+/androidx-main:development/referenceDocs/
"

# Grab the CL number generated from running `g4 change`.
clNum=$(g4 change --desc "$clDesc" | tail -1 | awk '{print $2}')
printf "View pending changes at http://cl/${clNum} \n"

printf "\n"
printf "=================================================================== \n"
printf "== Stage changes \n"
printf "=================================================================== \n"

# Construct the devsite command and parameters
devsiteCmd="/google/data/ro/projects/devsite/devsite2 stage"
devsiteCmd+=" --use_large_thread_pools"
devsiteCmd+=" --upload_safety_check_mode=ignore"

# Add the --db flag if the target database is not the default staging DB.
if [ "$FLAGS_db" != "$defaultDb" ]; then
  devsiteCmd+=" --db=$FLAGS_db"
fi

# Directories to stage
devsiteCmd+=" android/support"
devsiteCmd+=" androidx"
devsiteCmd+=" kotlin/android/support"
devsiteCmd+=" kotlin/androidx"

printf "Running devsite command:\n"
printf "$devsiteCmd\n"

$devsiteCmd

# Print devsite command and CL link again in case they scrolled off the screen or
# scrollback buffer
printf "\n"
printf "Ran devsite command:\n"
printf "$devsiteCmd\n"
printf "\n"
printf "View pending changes at http://cl/${clNum} \n"