development/diff_published_artifacts/diff_published_artifacts.py

# Copyright 2022, The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
import os
import subprocess
import shutil
import re
import requests
import zipfile
import sys

FETCH_ARTIFACT = "/google/data/ro/projects/android/fetch_artifact"
BASS = "/google/data/ro/projects/android/bass"
DEFAULT_CLONE_DEPTH = 100 # chosen arbitrarily, may need to be adjusted
DEFAULT_BUILD_ARTIFACT_SEARCH_TIME_SPAN_IN_DAYS = 7 # chosen arbitrarily, may need to be adjusted
GIT_LOG_URL = "https://android.googlesource.com/platform/frameworks/support/+log"
IGNORE_PATHS = [
 # includes timestamps
  "*.xml",
   # are different because the xml files include timestamps
  "*.xml.*",
  "*.sha*",
  "*.md5",
  # is different because it references the .sha* files.
  "*.module"
]


def main(build_id):
  """
  This is a script to take a given build_id, and search for a build of the previous commit on ab/
  If a commit is found, we download the top-of-tree-m2repository-all-{build_id}.zip file from both
  builds and diff the contents. If an .aar / .jar is different, we will unzip those and diff the
  contents as well.
  """
  staging_dir = prep_staging_dir()
  build_info_file_path = fetch_build_info(build_id, staging_dir)

  # presubmit BUILD_INFO files include the commit being built as well as the and parent commit
  if is_presubmit_build(build_id):
    previous_revision = get_previous_revision_from_build_info(build_info_file_path)
  # other builds only include the commit being built (as far as I can tell),  we need to
  # get the previous commit from git log
  else:
    current_revision = get_current_revision(build_info_file_path)
    previous_revision = get_previous_revision_from_git_history(current_revision, staging_dir)
  previous_build_id = get_previous_build_id(previous_revision)
  (before, after) = download_and_unzip_repos(staging_dir, build_id, previous_build_id)
  diff_repos(before, after, staging_dir)

def prep_staging_dir():
  """
  remove and recreate the ./download_staging which is located as a sibling of this script.
  """
  current_dir = os.path.dirname(os.path.realpath(__file__))
  staging_dir = current_dir + "/download_staging"
  if os.path.isdir(staging_dir):
    shutil.rmtree(staging_dir)
  os.makedirs(staging_dir, exist_ok=True)
  return staging_dir


def fetch_m2repo(build_id, staging_dir):
  file_path = f"top-of-tree-m2repository-all-{build_id}.zip"
  if is_presubmit_build(build_id):
    file_path = f"incremental/{file_path}"
  return fetch_artifact(build_id, staging_dir, file_path)


def fetch_build_info(build_id, staging_dir):
  return fetch_artifact(build_id, staging_dir, "BUILD_INFO")


def fetch_artifact(build_id, output_dir, file_path):
  file_name = file_path.split("/")[-1]
  print(f"fetching {file_name}")

  if is_presubmit_build(build_id):
    target = "androidx_incremental"
  else:
    target = "androidx"
  return FetchArtifactService().fetch_artifact(build_id, "aosp-androidx-main", target, output_dir, file_path)


def get_current_revision(build_info_file_path):
  print("Getting current revision from BUILD_INFO")
  with open(build_info_file_path) as f:
    build_info = json.load(f)
    support_project = next(
        project for project in build_info["parsed_manifest"]["projects"] if
        project["name"] == "platform/frameworks/support")
    current_revision = support_project["revision"]
    print(f"Found revision: {current_revision}")
    return current_revision


def get_previous_revision_from_build_info(build_info_file_path):
  print("Getting previous revision from BUILD_INFO")
  with open(build_info_file_path) as f:
    build_info = json.load(f)
    revision = build_info["git-pull"][0]["revisions"][0]["commit"]["parents"][0]["commitId"]
    print(f"Found previous revision: {revision}")
    return revision

def get_previous_revision_from_git_history(current_revision, staging_dir):
  """
  Gets previous revision from git log endpoint for androidx-main.
  """
  response = requests.get(git_log_url(current_revision))
  # endpoint returns  some junk in the first line making it invalid json
  text_with_first_line_removed = "\n".join(response.text.split("\n")[1:])
  response_json = json.loads(text_with_first_line_removed)
  previous_revision = response_json["log"][0]["parents"][0]
  print(f"Found previous revision: {previous_revision}")
  return previous_revision


def get_previous_build_id(previous_revision):
  print("Searching Android Build server for build matching previous revision")
  output = BassService().search_builds(
    DEFAULT_BUILD_ARTIFACT_SEARCH_TIME_SPAN_IN_DAYS,
    "aosp-androidx-main",
    "androidx",
    "BUILD_INFO",
    previous_revision
  )
  match = re.search("BuildID\: (\d+)", output.stdout)
  if match is None:
    raise Exception(f"Couldn't find previous build ID for revision {previous_revision}")

  previous_build_id = match.group(1)
  print(f"Found build matching previous revision: {previous_build_id}")
  return previous_build_id

def download_and_unzip_repos(staging_dir, build_id, previous_build_id):
  before_dir = staging_dir + "/before"
  after_dir = staging_dir + "/after"
  os.makedirs(before_dir)
  os.makedirs(after_dir)
  after_zip = fetch_m2repo(build_id, staging_dir)
  before_zip = fetch_m2repo(previous_build_id, staging_dir)
  return (unzip(before_zip, before_dir), unzip(after_zip, after_dir))

def diff_repos(before, after, staging_dir):
  output = DiffService().diff(before, after, IGNORE_PATHS)
  for line in output.stdout.splitlines():
    if line.startswith("Binary files "):
      for (before_file, after_file) in re.findall("Binary files (.+) and (.+) differ", line):
        diff_binary(before_file, after_file, staging_dir)
    else:
      print(line)


def diff_binary(before, after, staging_dir):
  file_name = before.split("/")[-1]
  if is_unzippable(before) and is_unzippable(after):
    before_contents = unzip(before, staging_dir + "/" + file_name + "-before")
    after_contents = unzip(after, staging_dir + "/" + file_name + "-after")
    output = DiffService().diff(before_contents, after_contents)
    # sometimes the binary is "different" but the contents are identical.
    # It might be interesting to add diff the metadata, but for now just ignore it.
    if output.stdout.strip() != "":
      print(output.stdout)
  else:
    print(f"Binary files {before} and {after} differ")

def is_unzippable(filename):
  return filename.endswith(".zip") or filename.endswith(".aar") or filename.endswith(".jar")

def unzip(file, destination):
  with zipfile.ZipFile(file, 'r') as zip:
    zip.extractall(destination)
    return destination

def is_presubmit_build(build_id):
    return build_id.startswith("P")

def git_log_url(revision):
    return f"{GIT_LOG_URL}/{revision}?format=JSON"

class DiffService():
    @staticmethod
    def diff(before_dir, after_dir, exclude=[]):
      args = ["diff", "-r"]
      for pattern in exclude:
        args.extend(["-x", pattern])
      args.extend([before_dir, after_dir])
      return subprocess.run(args, text=True, capture_output=True)


class FetchArtifactService():
    @staticmethod
    def fetch_artifact(build_id, branch, target, output_dir, file_path):
      file_name = file_path.split("/")[-1]
      subprocess.run(
        [
          FETCH_ARTIFACT,
          "--bid",
          build_id,
          "--branch",
          branch,
          "--target",
          target,
          file_path,
        ],
        cwd=output_dir,
        capture_output=True,
        check=True
      )
      return f"{output_dir}/{file_name}"

class BassService():
    @staticmethod
    def search_builds(days, branch, target, file_name, query):
     return subprocess.run([
          BASS,
          "--days",
          str(days),
          "--successful",
          "true",
          "--branch",
          branch,
          "--target",
          target,
          "--filename",
          file_name,
          "--query",
          query
      ],
          capture_output=True,
          text=True,
          check=True
      )

if __name__ == "__main__":
    main(sys.argv[1])