tools/repo_diff/repo_diff_android.py

#!/usr/bin/python
"""Diff a repo (downstream) and its upstream.

This script:
  1. Downloads a repo source tree with specified manifest URL, branch
     and release tag.
  2. Retrieves the BUILD_ID from $downstream/build/core/build_id.mk.
  3. Downloads the upstream using the BUILD_ID.
  4. Diffs each project in these two repos.
"""

import argparse
import datetime
import os
import subprocess
import repo_diff_trees

HELP_MSG = "Diff a repo (downstream) and its upstream"

DOWNSTREAM_WORKSPACE = "downstream"
UPSTREAM_WORKSPACE = "upstream"

DEFAULT_MANIFEST_URL = "https://android.googlesource.com/platform/manifest"
DEFAULT_MANIFEST_BRANCH = "android-8.0.0_r10"
DEFAULT_UPSTREAM_MANIFEST_URL = "https://android.googlesource.com/platform/manifest"
DEFAULT_UPSTREAM_MANIFEST_BRANCH = "android-8.0.0_r1"
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
DEFAULT_EXCLUSIONS_FILE = os.path.join(SCRIPT_DIR, "android_exclusions.txt")


def parse_args():
  """Parse args."""

  parser = argparse.ArgumentParser(description=HELP_MSG)

  parser.add_argument("-u", "--manifest-url",
                      help="manifest url",
                      default=DEFAULT_MANIFEST_URL)
  parser.add_argument("-b", "--manifest-branch",
                      help="manifest branch",
                      default=DEFAULT_MANIFEST_BRANCH)
  parser.add_argument("-r", "--upstream-manifest-url",
                      help="upstream manifest url",
                      default=DEFAULT_UPSTREAM_MANIFEST_URL)
  parser.add_argument("-a", "--upstream-manifest-branch",
                      help="upstream manifest branch",
                      default=DEFAULT_UPSTREAM_MANIFEST_BRANCH)
  parser.add_argument("-e", "--exclusions-file",
                      help="exclusions file",
                      default=DEFAULT_EXCLUSIONS_FILE)
  parser.add_argument("-t", "--tag",
                      help="release tag (optional). If not set then will "
                      "sync the latest in the branch.")
  parser.add_argument("-i", "--ignore_error_during_sync",
                      action="store_true",
                      help="repo sync might fail due to varios reasons. "
                      "Ignore these errors and move on. Use with caution.")

  return parser.parse_args()


def repo_init(url, rev, workspace):
  """Repo init with specific url and rev.

  Args:
    url: manifest url
    rev: manifest branch, or rev
    workspace: the folder to init and sync code
  """

  try:
    subprocess.check_output("repo", stderr=subprocess.PIPE,
                            cwd=os.path.dirname(workspace), shell=True)
  except subprocess.CalledProcessError:
    pass
  else:
    raise ValueError("cannot repo-init workspace (%s), workspace is within an "
                     "existing tree" % workspace)

  print("repo init:\n  url: %s\n  rev: %s\n  workspace: %s" %
        (url, rev, workspace))

  subprocess.check_output("repo init --manifest-url=%s --manifest-branch=%s" %
                          (url, rev), cwd=workspace, shell=True)


def repo_sync(workspace, ignore_error, retry=5):
  """Repo sync."""

  count = 0
  while count < retry:
    count += 1
    print("repo sync (retry=%d/%d):\n  workspace: %s" %
          (count, retry, workspace))

    try:
      command = "repo sync --jobs=24 --current-branch --quiet"
      command += " --no-tags --no-clone-bundle"
      if ignore_error:
        command += " --force-broken"
      subprocess.check_output(command, cwd=workspace, shell=True)
    except subprocess.CalledProcessError as e:
      print "Error: %s" % e.output
      if count == retry and not ignore_error:
        raise e
    # Stop retrying if the repo sync was successful
    else:
      break


def get_commit_with_keyword(project_path, keyword):
  """Get the latest commit in $project_path with the specific keyword."""

  return subprocess.check_output(("git -C %s "
                                  "rev-list --max-count=1 --grep=\"%s\" "
                                  "HEAD") %
                                 (project_path, keyword), shell=True).rstrip()


def get_build_id(workspace):
  """Get BUILD_ID defined in $workspace/build/core/build_id.mk."""

  path = os.path.join(workspace, "build", "core", "build_id.mk")
  return subprocess.check_output("source %s && echo $BUILD_ID" % path,
                                 shell=True).rstrip()


def repo_sync_specific_release(url, branch, tag, workspace, ignore_error):
  """Repo sync source with the specific release tag."""

  if not os.path.exists(workspace):
    os.makedirs(workspace)

  manifest_path = os.path.join(workspace, ".repo", "manifests")

  repo_init(url, branch, workspace)

  if tag:
    rev = get_commit_with_keyword(manifest_path, tag)
    if not rev:
      raise(ValueError("could not find a manifest revision for tag " + tag))
    repo_init(url, rev, workspace)

  repo_sync(workspace, ignore_error)


def diff(manifest_url, manifest_branch, tag, 
         upstream_manifest_url, upstream_manifest_branch,
         exclusions_file, ignore_error_during_sync):
  """Syncs and diffs an Android workspace against an upstream workspace."""

  workspace = os.path.abspath(DOWNSTREAM_WORKSPACE)
  upstream_workspace = os.path.abspath(UPSTREAM_WORKSPACE)
  # repo sync downstream source tree
  repo_sync_specific_release(
      manifest_url,
      manifest_branch,
      tag,
      workspace,
      ignore_error_during_sync)

  build_id = None

  if tag:
    # get the build_id so that we know which rev of upstream we need
    build_id = get_build_id(workspace)
    if not build_id:
      raise(ValueError("Error: could not find the Build ID of " + workspace))

  # repo sync upstream source tree
  repo_sync_specific_release(
      upstream_manifest_url,
      upstream_manifest_branch,
      build_id,
      upstream_workspace,
      ignore_error_during_sync)


  # make output folder
  if tag:
    output_folder = os.path.abspath(tag.replace(" ", "_"))
  else:
    current_time = datetime.datetime.today().strftime('%Y%m%d_%H%M%S')
    output_folder = os.path.abspath(current_time)

  if not os.path.exists(output_folder):
      os.makedirs(output_folder)

  # do the comparison
  repo_diff_trees.diff(
      upstream_workspace,
      workspace,
      os.path.join(output_folder, "project.csv"),
      os.path.join(output_folder, "commit.csv"),
      os.path.abspath(exclusions_file),
  )


def main():
  args = parse_args()

  diff(args.manifest_url,
       args.manifest_branch,
       args.tag,
       args.upstream_manifest_url,
       args.upstream_manifest_branch,
       args.exclusions_file,
       args.ignore_error_during_sync)

if __name__ == "__main__":
  main()