update_build_tools.py

#!/usr/bin/python
#
# Copyright 2014 Google Inc. All Rights Reserved.

import argparse
import os
import re
import shutil
import subprocess
import sys
import zipfile

parser = argparse.ArgumentParser(
    description=('Update prebuilt android build tools'))
parser.add_argument(
    'buildId',
    type=int,
    nargs='?',
    help='Build server build ID')
parser.add_argument(
    '--target',
    default='sdk_phone_x86-sdk',
    help='Download from the specified build server target')
parser.add_argument(
    '--from-local-dir',
    dest='localDir',
    help='Copy prebuilts from local directory instead of buildserver')
parser.add_argument(
    '--no-git',
    action='store_true',
    help='Do not create a git commit for the import')
args = parser.parse_args()

if not args.buildId and not args.localDir:
    parser.error("you must specify either a build ID or --from-local-dir")
    sys.exit(1)


if not args.no_git:
    # Make sure we don't overwrite any pending changes
    try:
        subprocess.check_call(['git', 'diff', '--quiet', '--', '**'])
        subprocess.check_call(['git', 'diff', '--quiet', '--cached', '--', '**'])
    except subprocess.CalledProcessError:
        print >> sys.stderr, "FAIL: There are uncommitted changes here; please revert or stash"
        sys.exit(1)


def GetSSOCmdline(url):
    return ['sso_client', '--location', '--request_timeout', '90', '--url', url]
def SSOFetch(url):
    return subprocess.check_output(GetSSOCmdline(url), stderr=subprocess.STDOUT)
def FetchFile(srcFile, dstFile):
    if not args.localDir:
        fileURL = buildURL + '/' + srcFile
        subprocess.check_call(' '.join(GetSSOCmdline(fileURL) + ['>', dstFile]), shell=True)
    else:
        shutil.copyfile(os.path.join(args.localDir, srcFile), dstFile)


if not args.localDir:
    # to start, find the build server branch we're downloading from
    try:
        branch = SSOFetch("http://android-build/buildbot-update?op=GET-BRANCH&bid=%d" % args.buildId)
    except subprocess.CalledProcessError:
        print >> sys.stderr, "FAIL: Unable to retrieve branch for build ID %d" % args.buildId
        sys.exit(1)
    buildURL = "http://android-build/builds/%s-linux-%s/%d" % (branch, args.target, args.buildId)

    # fetch the list build artifacts
    try:
        listingText = SSOFetch(buildURL + "?output=text")
    except subprocess.CalledProcessError:
        print >> sys.stderr, "FAIL: Unable to retrieve listing for build ID %d" % args.buildId
        sys.exit(1)

    listing = listingText.strip().split('\n')
else:
    listing = os.listdir(args.localDir)


buildToolsZipRE = re.compile(r'^sdk-repo-linux-build-tools-.*\.zip$')
buildToolsZipFile = None
for fname in listing:
    if buildToolsZipRE.match(fname):
        buildToolsZipFile = fname
        break
if not buildToolsZipFile:
    src = 'directory' if args.localDir else 'build'
    print >> sys.stderr, "FAIL: The specified %s contains no platform ZIP" % src
    sys.exit(1)

try:
    FetchFile(buildToolsZipFile, buildToolsZipFile)
except:
    print >> sys.stderr, "FAIL: Unable to fetch %s" % buildToolsZipFile
    sys.exit(1)


try:    # make sure we delete the downloaded ZIP
    BUILD_TOOLS_DIRS = [
        ('android-LollipopMR1/lib', 'lib'),
        ('android-LollipopMR1/renderscript', 'renderscript'),
    ]

    executableMap = {}
    with zipfile.ZipFile(buildToolsZipFile) as buildToolsZip:
        files = filter(lambda x: x.count('/') == 1 and not x[-1] == '/', buildToolsZip.namelist())
        for fileEntry in map(lambda x: (x, x[x.find('/')+1:]) , files):
            executable = False
            if os.path.exists(fileEntry[1]):
                if os.path.isdir(fileEntry[1]):
                    subprocess.check_call(['git', 'rm', '-rf', fileEntry[1]])
                else:
                    executable = os.access(fileEntry[1], os.X_OK)
                    subprocess.check_call(['git', 'rm', fileEntry[1]])

            with buildToolsZip.open(fileEntry[0], 'r') as infile:
                with open(fileEntry[1], 'w+') as outfile:
                    shutil.copyfileobj(infile, outfile)
            if executable:
                subprocess.check_call(['chmod', '+x', fileEntry[1]])
            subprocess.check_call(['git', 'add', fileEntry[1]])

        for dirEntry in BUILD_TOOLS_DIRS:
            if os.path.exists(dirEntry[1]):
                if os.path.isdir(dirEntry[1]):
                    subprocess.check_call(['git', 'rm', '-rf', dirEntry[1]])
                else:
                    subprocess.check_call(['git', 'rm', dirEntry[1]])

            # extract the new version of the directory
            fileZipDir = dirEntry[0] + '/'

            for fname in buildToolsZip.namelist():
                if (fname[-1] != '/'
                        and fname.startswith(dirEntry[0])):
                    relativePath = fname[len(fileZipDir):]
                    subdirs = os.path.join(dirEntry[1], os.path.dirname(relativePath))
                    if not os.path.exists(subdirs):
                        os.makedirs(subdirs)
                    with buildToolsZip.open(fname, 'r') as infile:
                        with open(os.path.join(dirEntry[1], relativePath), 'w+') as outfile:
                            shutil.copyfileobj(infile, outfile)

            subprocess.check_call(['git', 'add', dirEntry[1]])

    # modify the source.properties
    subprocess.check_call(['sed', '-i', 's/Pkg.Revision=/\/\/Pkg.Revision=/',
            'source.properties'])

    subprocess.check_call(' '.join(['echo', '-e',
            '"Archive.Os=LINUX\nPkg.Revision=22.0.0\nArchive.Arch=ANY"'] +
            ['>>', 'source.properties']), shell=True)

    subprocess.check_call(['git', 'add', 'source.properties'])

    if not args.no_git:
        # commit all changes
        if not args.localDir:
            # gather useful info
            buildInfo = dict(
                buildURL = buildURL,
                buildId = args.buildId,
                branch = branch,
                buildToolsZipFile = buildToolsZipFile,
            )
            msg = """Import L Build Tools from {branch} build {buildId}

{buildURL}/{buildToolsZipFile}

source.properties has been modified to make this appear as API 22
""".format(**buildInfo)
        else:
            msg = "DO NOT SUBMIT Import locally built android platform from %s" % args.localDir

        subprocess.check_call(['git', 'commit', '-m', msg])
        print 'Update successful.'

        if not args.localDir:
            print 'Be sure to upload this manually to gerrit.'

finally:
    # revert all stray files, including the downloaded zip
    try:
        with open(os.devnull, 'w') as bitbucket:
            subprocess.check_call(['git', 'add', '-Af', '.'], stdout=bitbucket)
            subprocess.check_call(
                    ['git', 'commit', '-m', 'COMMIT TO REVERT - RESET ME!!!'], stdout=bitbucket)
            subprocess.check_call(['git', 'reset', '--hard', 'HEAD~1'], stdout=bitbucket)
    except subprocess.CalledProcessError:
        print >> sys.stderr, "ERROR: Failed cleaning up, manual cleanup required!!!"