update-prebuilts.py - toolchain/gcc - Git at Google

 #!/usr/bin/env python
 #
 # Copyright (C) 2015 The Android Open Source Project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #      http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #

 """Update the prebuilt GCC from the build server."""
 from __future__ import print_function

 import argparse
 import inspect
 import os
 import shutil
 import subprocess
 import sys


 THIS_DIR = os.path.realpath(os.path.dirname(__name__))
 ANDROID_DIR = os.path.realpath(os.path.join(THIS_DIR, '../..'))

 BRANCH = 'aosp-gcc'


 def android_path(*args):
     return os.path.join(ANDROID_DIR, *args)


 class ArgParser(argparse.ArgumentParser):
     def __init__(self):
         super(ArgParser, self).__init__(
             description=inspect.getdoc(sys.modules[__name__]))

         self.add_argument(
             'build', metavar='BUILD',
             help='Build number to pull from the build server.')

         self.add_argument(
             '--dryrun', action='store_true',
             help='Dry run mode: echo commands but do not execute them.')

         self.add_argument(
             '--use-current-branch', action='store_true',
             help='Do not repo start a new branch for the update.')

         self.add_argument(
             '--cachedir', metavar='CACHEDIR',
             help='Draw build artifacts from cache dir.')

         self.add_argument(
             '--message', '-m', metavar='MESSAGE',
             help='Override the git commit message.')


 def host_to_build_host(host):
     """Gets the build host name for an NDK host tag.

     The Windows builds are done from Linux.
     """
     return {
         'darwin': 'mac',
         'linux': 'linux',
         'windows': 'linux',
     }[host]


 def build_name(host, arch):
     """Gets the release build name for an NDK host tag.

     The builds are named by a short identifier like "linux" or "win64".

     >>> build_name('darwin', 'arm')
     'arm'

     >>> build_name('darwin', 'aarch64')
     'arm64'

     >>> build_name('linux', 'x86')
     'linux_x86'
     """
     build_arch = arch
     if arch == 'aarch64':
         build_arch = 'arm64'

     if host == 'darwin':
         return build_arch

     return host + '_' + build_arch


 def package_name(host, arch):
     """Returns the file name for a given package configuration.

     >>> package_name('linux', 'arm')
     'gcc-arm-linux-x86_64.tar.bz2'

     >>> package_name('linux', 'aarch64')
     'gcc-arm64-linux-x86_64.tar.bz2'

     >>> package_name('darwin', 'x86')
     'gcc-x86-darwin-x86_64.tar.bz2'
     """
     build_arch = arch
     if arch == 'aarch64':
         build_arch = 'arm64'
     return 'gcc-{}-{}-x86_64.tar.bz2'.format(build_arch, host)


 def invoke_cmd(dryrun, cmds, outfile=None):
     """Invoke specified command (or echo command if dry-run mode)."""
     if dryrun:
         print('cmd: %s' % ' '.join(cmds))
         return
     subprocess.check_call(cmds, stdout=outfile)


 def download_build(host, arch, build_number, download_dir, dryrun, cachedir):
     """Download a specific build artifact."""
     pkg_name = package_name(host, arch)
     if cachedir:
         cached_pkg = os.path.join(cachedir, pkg_name)
         if os.path.exists(cached_pkg):
             print('Reusing existing copy of {} from '
                   '{}'.format(pkg_name, cachedir))
             return cached_pkg

     url_base = 'https://android-build-uber.corp.google.com'
     path = 'builds/{branch}-{build_host}-{build_name}/{build_num}'.format(
         branch=BRANCH,
         build_host=host_to_build_host(host),
         build_name=build_name(host, arch),
         build_num=build_number)

     url = '{}/{}/{}'.format(url_base, path, pkg_name)
     TIMEOUT = '60'  # In seconds.
     out_file_path = os.path.join(download_dir, pkg_name)
     with open(out_file_path, 'w') as out_file:
         print('Downloading {} to {}'.format(url, out_file_path))
         invoke_cmd(dryrun,
                    ['sso_client', '--location',
                     '--request_timeout', TIMEOUT, url],
                    outfile=out_file)
     return out_file_path


 def extract_package(package, install_dir, dryrun):
     # The --strip-components is needed because the git project is in
     # prebuilts/gcc/$HOST/$ARCH/$TRIPLE, rather than a directory above that
     # like it really should have been.
     cmd = ['tar', 'xf', package, '-C', install_dir, '--strip-components=1']
     print('Extracting {}...'.format(package))
     invoke_cmd(dryrun, cmd)


 def delete_old_toolchain(path, dryrun):
     print('Removing old files in {}...'.format(path))
     invoke_cmd(dryrun, ['git', '-C', path,
                         'rm', '-rf', '--ignore-unmatch', '.'])

     # Git doesn't believe in directories, so `git rm -rf` might leave behind
     # empty directories.
     invoke_cmd(dryrun, ['git', '-C', path, 'clean', '-df'])


 def get_prebuilt_arch(arch):
     return {
         'arm': 'arm',
         'aarch64': 'aarch64',
         'mips64': 'mips',
         'x86_64': 'x86',
     }[arch]


 def get_triple(arch):
     triple_arch = arch
     if arch == 'mips64':
         triple_arch = 'mips64el'

     triple = '{}-linux-android'.format(triple_arch)
     if arch == 'arm':
         triple += 'eabi'

     return triple


 def get_prebuilt_subdir(host, arch):
     """Returns the install path for a GCC prebuilt relative to the root.

     Thanks to historical project naming conventions, these paths are a little
     non-obvious. The toolchains are installed to paths such as:
     "prebuilts/gcc/$HOST/$ARCH/$TRIPLE-$VERSION", but $ARCH isn't the arch
     you'd expect for anything but the two ARMs.

     For x86 and mips, we use a multilib toolchain. In other words, we don't
     have both an x86 and x86_64 toolchain, we have just the x86_64 toolchain.
     Unfortunately, the install path for the x86_64 toolchain is
     prebuilts/gcc/$HOST/x86/x86_64-linux-android, because reasons.

     >>> get_prebuilt_subdir('linux-x86', 'arm')
     'prebuilts/gcc/linux-x86/arm/arm-linux-androideabi-4.9'

     >>> get_prebuilt_subdir('linux-x86', 'aarch64')
     'prebuilts/gcc/linux-x86/aarch64/aarch64-linux-android-4.9'

     >>> get_prebuilt_subdir('linux-x86', 'x86_64')
     'prebuilts/gcc/linux-x86/x86/x86_64-linux-android-4.9'

     >>> get_prebuilt_subdir('linux-x86', 'mips64')
     'prebuilts/gcc/linux-x86/mips/mips64el-linux-android-4.9'

     >>> get_prebuilt_subdir('darwin-x86', 'mips64')
     'prebuilts/gcc/darwin-x86/mips/mips64el-linux-android-4.9'
     """

     prebuilt_arch = get_prebuilt_arch(arch)
     triple = get_triple(arch) + '-4.9'

     return os.path.join('prebuilts/gcc', host, prebuilt_arch, triple)


 # This is a separate function from get_prebuilt_subdir just to simplify the
 # doctests (no need to know absolute path).
 def get_prebuilt_path(host, arch):
     return android_path(get_prebuilt_subdir(host, arch))


 def generate_androidkernel_symlinks(arch, prebuilt_dir, dryrun):
     """Generate an -androidkernel toolchain.

     The kernel doesn't correctly link with gold, the default on x86 and ARM.
     Generate a fake toolchain consisting of symlinks, with ld pointing to bfd.
     """

     files = {
         'ar': 'ar',
         'as': 'as',
         'size': 'size',
         'strip': 'strip',
         'nm': 'nm',
         'cpp': 'cpp',
         'ld': 'ld.bfd',
         'gcc': 'gcc',
         'objcopy': 'objcopy',
         'objdump': 'objdump',
         'readelf': 'readelf',
     }

     original_triple = get_triple(arch)
     new_triple = original_triple + "kernel"

     if arch == 'arm':
         # We don't want arm-linux-androideabikernel.
         new_triple = 'arm-linux-androidkernel'

     bin_dir = os.path.join(prebuilt_dir, 'bin')
     src_prefix = '{}-'.format(original_triple)
     link_prefix = '{}-'.format(new_triple)
     for link, src in files.iteritems():
         link_path = os.path.join(bin_dir, link_prefix + link)
         src_path = src_prefix + src

         if dryrun:
             print('ln -s {} {}'.format(src_path, link_path))
         else:
             # Make sure our symlink actually points to something.
             full_path = os.path.join(bin_dir, src_path)
             if not os.path.exists(full_path):
                 sys.exit("ERROR: missing file '{}'".format(full_path))

             os.symlink(src_path, link_path)


 def update_gcc(host, arch, build_number, use_current_branch,
                dryrun, download_dir, message, cachedir):
     host_tag = host + '-x86'
     prebuilt_dir = get_prebuilt_path(host_tag, arch)
     if dryrun:
         print('cd %s' % prebuilt_dir)
     os.chdir(prebuilt_dir)

     if not use_current_branch:
         invoke_cmd(dryrun,
                    ['repo', 'start',
                     'update-gcc-{}'.format(build_number), '.'])

     package = download_build(host, arch, build_number,
                              download_dir, dryrun, cachedir)

     # Remove the old toolchain so we know the package we're building isn't
     # missing anything.
     delete_old_toolchain(prebuilt_dir, dryrun)
     extract_package(package, prebuilt_dir, dryrun)
     generate_androidkernel_symlinks(arch, prebuilt_dir, dryrun)

     print('Adding files to index...')
     invoke_cmd(dryrun, ['git', 'add', '.'])

     print('Committing update...')
     if message is not None:
         message = message.decode('string_escape')
     else:
         message = 'Update prebuilt GCC to build {}.'.format(build_number)
     invoke_cmd(dryrun, ['git', 'commit', '-m', message])


 def main():
     args = ArgParser().parse_args()

     download_dir = os.path.realpath('.download')
     if os.path.isdir(download_dir):
         shutil.rmtree(download_dir)
     os.makedirs(download_dir)

     try:
         hosts = ('linux', 'darwin')
         arches = ('arm', 'aarch64', 'x86_64', 'mips64')
         for host in hosts:
             for arch in arches:
                 update_gcc(host, arch, args.build, args.use_current_branch,
                            args.dryrun, download_dir, args.message,
                            args.cachedir)

     finally:
         shutil.rmtree(download_dir)


 if __name__ == '__main__':
     main()
	#!/usr/bin/env python
	#
	# Copyright (C) 2015 The Android Open Source Project
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.
	#

	"""Update the prebuilt GCC from the build server."""
	from __future__ import print_function

	import argparse
	import inspect
	import os
	import shutil
	import subprocess
	import sys


	THIS_DIR = os.path.realpath(os.path.dirname(__name__))
	ANDROID_DIR = os.path.realpath(os.path.join(THIS_DIR, '../..'))

	BRANCH = 'aosp-gcc'


	def android_path(*args):
	return os.path.join(ANDROID_DIR, *args)


	class ArgParser(argparse.ArgumentParser):
	def __init__(self):
	super(ArgParser, self).__init__(
	description=inspect.getdoc(sys.modules[__name__]))

	self.add_argument(
	'build', metavar='BUILD',
	help='Build number to pull from the build server.')

	self.add_argument(
	'--dryrun', action='store_true',
	help='Dry run mode: echo commands but do not execute them.')

	self.add_argument(
	'--use-current-branch', action='store_true',
	help='Do not repo start a new branch for the update.')

	self.add_argument(
	'--cachedir', metavar='CACHEDIR',
	help='Draw build artifacts from cache dir.')

	self.add_argument(
	'--message', '-m', metavar='MESSAGE',
	help='Override the git commit message.')


	def host_to_build_host(host):
	"""Gets the build host name for an NDK host tag.

	The Windows builds are done from Linux.
	"""
	return {
	'darwin': 'mac',
	'linux': 'linux',
	'windows': 'linux',
	}[host]


	def build_name(host, arch):
	"""Gets the release build name for an NDK host tag.

	The builds are named by a short identifier like "linux" or "win64".

	>>> build_name('darwin', 'arm')
	'arm'

	>>> build_name('darwin', 'aarch64')
	'arm64'

	>>> build_name('linux', 'x86')
	'linux_x86'
	"""
	build_arch = arch
	if arch == 'aarch64':
	build_arch = 'arm64'

	if host == 'darwin':
	return build_arch

	return host + '_' + build_arch


	def package_name(host, arch):
	"""Returns the file name for a given package configuration.

	>>> package_name('linux', 'arm')
	'gcc-arm-linux-x86_64.tar.bz2'

	>>> package_name('linux', 'aarch64')
	'gcc-arm64-linux-x86_64.tar.bz2'

	>>> package_name('darwin', 'x86')
	'gcc-x86-darwin-x86_64.tar.bz2'
	"""
	build_arch = arch
	if arch == 'aarch64':
	build_arch = 'arm64'
	return 'gcc-{}-{}-x86_64.tar.bz2'.format(build_arch, host)


	def invoke_cmd(dryrun, cmds, outfile=None):
	"""Invoke specified command (or echo command if dry-run mode)."""
	if dryrun:
	print('cmd: %s' % ' '.join(cmds))
	return
	subprocess.check_call(cmds, stdout=outfile)


	def download_build(host, arch, build_number, download_dir, dryrun, cachedir):
	"""Download a specific build artifact."""
	pkg_name = package_name(host, arch)
	if cachedir:
	cached_pkg = os.path.join(cachedir, pkg_name)
	if os.path.exists(cached_pkg):
	print('Reusing existing copy of {} from '
	'{}'.format(pkg_name, cachedir))
	return cached_pkg

	url_base = 'https://android-build-uber.corp.google.com'
	path = 'builds/{branch}-{build_host}-{build_name}/{build_num}'.format(
	branch=BRANCH,
	build_host=host_to_build_host(host),
	build_name=build_name(host, arch),
	build_num=build_number)

	url = '{}/{}/{}'.format(url_base, path, pkg_name)
	TIMEOUT = '60' # In seconds.
	out_file_path = os.path.join(download_dir, pkg_name)
	with open(out_file_path, 'w') as out_file:
	print('Downloading {} to {}'.format(url, out_file_path))
	invoke_cmd(dryrun,
	['sso_client', '--location',
	'--request_timeout', TIMEOUT, url],
	outfile=out_file)
	return out_file_path


	def extract_package(package, install_dir, dryrun):
	# The --strip-components is needed because the git project is in
	# prebuilts/gcc/$HOST/$ARCH/$TRIPLE, rather than a directory above that
	# like it really should have been.
	cmd = ['tar', 'xf', package, '-C', install_dir, '--strip-components=1']
	print('Extracting {}...'.format(package))
	invoke_cmd(dryrun, cmd)


	def delete_old_toolchain(path, dryrun):
	print('Removing old files in {}...'.format(path))
	invoke_cmd(dryrun, ['git', '-C', path,
	'rm', '-rf', '--ignore-unmatch', '.'])

	# Git doesn't believe in directories, so `git rm -rf` might leave behind
	# empty directories.
	invoke_cmd(dryrun, ['git', '-C', path, 'clean', '-df'])


	def get_prebuilt_arch(arch):
	return {
	'arm': 'arm',
	'aarch64': 'aarch64',
	'mips64': 'mips',
	'x86_64': 'x86',
	}[arch]


	def get_triple(arch):
	triple_arch = arch
	if arch == 'mips64':
	triple_arch = 'mips64el'

	triple = '{}-linux-android'.format(triple_arch)
	if arch == 'arm':
	triple += 'eabi'

	return triple


	def get_prebuilt_subdir(host, arch):
	"""Returns the install path for a GCC prebuilt relative to the root.

	Thanks to historical project naming conventions, these paths are a little
	non-obvious. The toolchains are installed to paths such as:
	"prebuilts/gcc/$HOST/$ARCH/$TRIPLE-$VERSION", but $ARCH isn't the arch
	you'd expect for anything but the two ARMs.

	For x86 and mips, we use a multilib toolchain. In other words, we don't
	have both an x86 and x86_64 toolchain, we have just the x86_64 toolchain.
	Unfortunately, the install path for the x86_64 toolchain is
	prebuilts/gcc/$HOST/x86/x86_64-linux-android, because reasons.

	>>> get_prebuilt_subdir('linux-x86', 'arm')
	'prebuilts/gcc/linux-x86/arm/arm-linux-androideabi-4.9'

	>>> get_prebuilt_subdir('linux-x86', 'aarch64')
	'prebuilts/gcc/linux-x86/aarch64/aarch64-linux-android-4.9'

	>>> get_prebuilt_subdir('linux-x86', 'x86_64')
	'prebuilts/gcc/linux-x86/x86/x86_64-linux-android-4.9'

	>>> get_prebuilt_subdir('linux-x86', 'mips64')
	'prebuilts/gcc/linux-x86/mips/mips64el-linux-android-4.9'

	>>> get_prebuilt_subdir('darwin-x86', 'mips64')
	'prebuilts/gcc/darwin-x86/mips/mips64el-linux-android-4.9'
	"""

	prebuilt_arch = get_prebuilt_arch(arch)
	triple = get_triple(arch) + '-4.9'

	return os.path.join('prebuilts/gcc', host, prebuilt_arch, triple)


	# This is a separate function from get_prebuilt_subdir just to simplify the
	# doctests (no need to know absolute path).
	def get_prebuilt_path(host, arch):
	return android_path(get_prebuilt_subdir(host, arch))


	def generate_androidkernel_symlinks(arch, prebuilt_dir, dryrun):
	"""Generate an -androidkernel toolchain.

	The kernel doesn't correctly link with gold, the default on x86 and ARM.
	Generate a fake toolchain consisting of symlinks, with ld pointing to bfd.
	"""

	files = {
	'ar': 'ar',
	'as': 'as',
	'size': 'size',
	'strip': 'strip',
	'nm': 'nm',
	'cpp': 'cpp',
	'ld': 'ld.bfd',
	'gcc': 'gcc',
	'objcopy': 'objcopy',
	'objdump': 'objdump',
	'readelf': 'readelf',
	}

	original_triple = get_triple(arch)
	new_triple = original_triple + "kernel"

	if arch == 'arm':
	# We don't want arm-linux-androideabikernel.
	new_triple = 'arm-linux-androidkernel'

	bin_dir = os.path.join(prebuilt_dir, 'bin')
	src_prefix = '{}-'.format(original_triple)
	link_prefix = '{}-'.format(new_triple)
	for link, src in files.iteritems():
	link_path = os.path.join(bin_dir, link_prefix + link)
	src_path = src_prefix + src

	if dryrun:
	print('ln -s {} {}'.format(src_path, link_path))
	else:
	# Make sure our symlink actually points to something.
	full_path = os.path.join(bin_dir, src_path)
	if not os.path.exists(full_path):
	sys.exit("ERROR: missing file '{}'".format(full_path))

	os.symlink(src_path, link_path)


	def update_gcc(host, arch, build_number, use_current_branch,
	dryrun, download_dir, message, cachedir):
	host_tag = host + '-x86'
	prebuilt_dir = get_prebuilt_path(host_tag, arch)
	if dryrun:
	print('cd %s' % prebuilt_dir)
	os.chdir(prebuilt_dir)

	if not use_current_branch:
	invoke_cmd(dryrun,
	['repo', 'start',
	'update-gcc-{}'.format(build_number), '.'])

	package = download_build(host, arch, build_number,
	download_dir, dryrun, cachedir)

	# Remove the old toolchain so we know the package we're building isn't
	# missing anything.
	delete_old_toolchain(prebuilt_dir, dryrun)
	extract_package(package, prebuilt_dir, dryrun)
	generate_androidkernel_symlinks(arch, prebuilt_dir, dryrun)

	print('Adding files to index...')
	invoke_cmd(dryrun, ['git', 'add', '.'])

	print('Committing update...')
	if message is not None:
	message = message.decode('string_escape')
	else:
	message = 'Update prebuilt GCC to build {}.'.format(build_number)
	invoke_cmd(dryrun, ['git', 'commit', '-m', message])


	def main():
	args = ArgParser().parse_args()

	download_dir = os.path.realpath('.download')
	if os.path.isdir(download_dir):
	shutil.rmtree(download_dir)
	os.makedirs(download_dir)

	try:
	hosts = ('linux', 'darwin')
	arches = ('arm', 'aarch64', 'x86_64', 'mips64')
	for host in hosts:
	for arch in arches:
	update_gcc(host, arch, args.build, args.use_current_branch,
	args.dryrun, download_dir, args.message,
	args.cachedir)

	finally:
	shutil.rmtree(download_dir)


	if __name__ == '__main__':
	main()