build/scripts/slave/annotated_run.py - platform/external/adt-infra - Git at Google

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.

 import contextlib
 import json
 import optparse
 import os
 import subprocess
 import sys
 import tempfile

 BUILD_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(
     os.path.abspath(__file__))))
 sys.path.append(os.path.join(BUILD_ROOT, 'scripts'))
 sys.path.append(os.path.join(BUILD_ROOT, 'third_party'))

 from common import annotator
 from common import chromium_utils
 from common import master_cfg_utils

 SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))
 BUILD_LIMITED_ROOT = os.path.join(
     os.path.dirname(BUILD_ROOT), 'build_internal', 'scripts', 'slave')

 PACKAGE_CFG = os.path.join(
     os.path.dirname(os.path.dirname(SCRIPT_PATH)),
     'infra', 'config', 'recipes.cfg')

 @contextlib.contextmanager
 def namedTempFile():
   fd, name = tempfile.mkstemp()
   os.close(fd)  # let the exceptions fly
   try:
     yield name
   finally:
     try:
       os.remove(name)
     except OSError as e:
       print >> sys.stderr, "LEAK: %s: %s" % (name, e)

 def get_recipe_properties(factory_properties, build_properties,
                           master_overrides_slave):
   """Constructs the recipe's properties from buildbot's properties.

   This retrieves the current factory properties from the master_config
   in the slave's checkout (the factory properties handed to us from the
   master might be out of date), and merges in the build properties.

   Using the values from the checkout allows us to do things like change
   the recipe and other factory properties for a builder without needing
   a master restart.
   """
   master_properties = factory_properties.copy()
   master_properties.update(build_properties)

   mastername = master_properties.get('mastername')
   buildername = master_properties.get('buildername')
   slave_properties = {}
   if mastername and buildername:
     try:
       slave_properties = get_factory_properties_from_disk(
           mastername, buildername)
     except LookupError as e:
       if master_overrides_slave:
         print 'WARNING in annotated_run.py (non-fatal): %s' % e
       else:
         raise e

   properties = master_properties.copy()
   conflicting_properties = {}
   for name in slave_properties:
     if master_properties.get(name) != slave_properties[name]:
       conflicting_properties[name] = (master_properties.get(name),
                                    slave_properties[name])

   if conflicting_properties:
     print 'The following build properties differ between master and slave:'
     for name, (master_value, slave_value) in conflicting_properties.items():
       print ('  "%s": master: "%s", slave: "%s"' % (
           name,
           "<unset>" if (master_value is None) else master_value,
           slave_value))
     print ("Using the values from the %s." %
            ("master" if master_overrides_slave else "slave"))

   if not master_overrides_slave:
     for name, (_, slave_value) in conflicting_properties.items():
       properties[name] = slave_value

   return properties


 def get_factory_properties_from_disk(mastername, buildername):
   master_list = master_cfg_utils.GetMasters()
   master_path = None
   for name, path in master_list:
     if name == mastername:
       master_path = path

   if not master_path:
     raise LookupError('master "%s" not found.' % mastername)

   script_path = os.path.join(BUILD_ROOT, 'scripts', 'tools',
                              'dump_master_cfg.py')

   with namedTempFile() as fname:
     dump_cmd = [sys.executable,
                 script_path,
                 master_path, fname]
     proc = subprocess.Popen(dump_cmd, cwd=BUILD_ROOT, stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
     out, err = proc.communicate()
     exit_code = proc.returncode

     if exit_code:
       raise LookupError('Failed to get the master config; dump_master_cfg %s'
                         'returned %d):\n%s\n%s\n'% (
                         mastername, exit_code, out, err))

     with open(fname, 'rU') as f:
       config = json.load(f)

   # Now extract just the factory properties for the requested builder
   # from the master config.
   props = {}
   found = False
   for builder_dict in config['builders']:
     if builder_dict['name'] == buildername:
       found = True
       factory_properties = builder_dict['factory']['properties']
       for name, (value, _) in factory_properties.items():
         props[name] = value

   if not found:
     raise LookupError('builder "%s" not found on in master "%s"' %
                       (buildername, mastername))

   if 'recipe' not in props:
     raise LookupError('Cannot find recipe for %s on %s' %
                       (buildername, mastername))

   return props


 def get_args(argv):
   """Process command-line arguments."""

   parser = optparse.OptionParser(
       description='Entry point for annotated builds.')
   parser.add_option('--build-properties',
                     action='callback', callback=chromium_utils.convert_json,
                     type='string', default={},
                     help='build properties in JSON format')
   parser.add_option('--factory-properties',
                     action='callback', callback=chromium_utils.convert_json,
                     type='string', default={},
                     help='factory properties in JSON format')
   parser.add_option('--build-properties-gz',
                     action='callback', callback=chromium_utils.convert_gz_json,
                     type='string', default={}, dest='build_properties',
                     help='build properties in b64 gz JSON format')
   parser.add_option('--factory-properties-gz',
                     action='callback', callback=chromium_utils.convert_gz_json,
                     type='string', default={}, dest='factory_properties',
                     help='factory properties in b64 gz JSON format')
   parser.add_option('--keep-stdin', action='store_true', default=False,
                     help='don\'t close stdin when running recipe steps')
   parser.add_option('--master-overrides-slave', action='store_true',
                     help='use the property values given on the command line '
                          'from the master, not the ones looked up on the slave')
   return parser.parse_args(argv)


 def update_scripts():
   if os.environ.get('RUN_SLAVE_UPDATED_SCRIPTS'):
     os.environ.pop('RUN_SLAVE_UPDATED_SCRIPTS')
     return False

   stream = annotator.StructuredAnnotationStream()
   git_cmd = 'git.bat' if os.name == "nt" else 'git'
   with stream.step('update_scripts') as s:
     fetch_cmd = [git_cmd, 'fetch', '--all']
     reset_cmd = [git_cmd, 'reset', '--hard', 'origin/emu-master-dev']
     checkout_cmd = [git_cmd, 'checkout', 'emu-master-dev']
     if subprocess.call(fetch_cmd) != 0 or subprocess.call(reset_cmd) != 0 or subprocess.call(checkout_cmd) != 0:
       s.step_text('git update source failed!')
       s.step_warnings()
     s.step_text('git pull')

     os.environ['RUN_SLAVE_UPDATED_SCRIPTS'] = '1'

     # After running update scripts, set PYTHONIOENCODING=UTF-8 for the real
     # annotated_run.
     os.environ['PYTHONIOENCODING'] = 'UTF-8'

     return True


 def clean_old_recipe_engine():
   """Clean stale pycs from the old location of recipe_engine.

   This function should only be needed for a little while after the recipe
   packages rollout (2015-09-16).
   """
   for (dirpath, _, filenames) in os.walk(
       os.path.join(BUILD_ROOT, 'third_party', 'recipe_engine')):
     for filename in filenames:
       if filename.endswith('.pyc'):
         path = os.path.join(dirpath, filename)
         os.remove(path)


 def main(argv):
   opts, _ = get_args(argv)
   properties = get_recipe_properties(
       opts.factory_properties, opts.build_properties,
       opts.master_overrides_slave)

   clean_old_recipe_engine()

   # Find out if the recipe we intend to run is in build_internal's recipes. If
   # so, use recipes.py from there, otherwise use the one from build.
   recipe_file = properties['recipe'].replace('/', os.path.sep) + '.py'
   if os.path.exists(os.path.join(BUILD_LIMITED_ROOT, 'recipes', recipe_file)):
     recipe_runner = os.path.join(BUILD_LIMITED_ROOT, 'recipes.py')
   else:
     recipe_runner = os.path.join(SCRIPT_PATH, 'recipes.py')

   with namedTempFile() as props_file:
     with open(props_file, 'w') as fh:
       fh.write(json.dumps(properties))
     cmd = [
         sys.executable, '-u', recipe_runner,
         'run',
         '--workdir=%s' % os.getcwd(),
         '--properties-file=%s' % props_file,
         properties['recipe'] ]
     return subprocess.call(cmd)


 def shell_main(argv):
   if update_scripts():
     return subprocess.call([sys.executable] + argv)
   else:
     return main(argv)


 if __name__ == '__main__':
   sys.exit(shell_main(sys.argv))
	#!/usr/bin/env python
	# Copyright (c) 2013 The Chromium Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.

	import contextlib
	import json
	import optparse
	import os
	import subprocess
	import sys
	import tempfile

	BUILD_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(
	os.path.abspath(__file__))))
	sys.path.append(os.path.join(BUILD_ROOT, 'scripts'))
	sys.path.append(os.path.join(BUILD_ROOT, 'third_party'))

	from common import annotator
	from common import chromium_utils
	from common import master_cfg_utils

	SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))
	BUILD_LIMITED_ROOT = os.path.join(
	os.path.dirname(BUILD_ROOT), 'build_internal', 'scripts', 'slave')

	PACKAGE_CFG = os.path.join(
	os.path.dirname(os.path.dirname(SCRIPT_PATH)),
	'infra', 'config', 'recipes.cfg')

	@contextlib.contextmanager
	def namedTempFile():
	fd, name = tempfile.mkstemp()
	os.close(fd) # let the exceptions fly
	try:
	yield name
	finally:
	try:
	os.remove(name)
	except OSError as e:
	print >> sys.stderr, "LEAK: %s: %s" % (name, e)

	def get_recipe_properties(factory_properties, build_properties,
	master_overrides_slave):
	"""Constructs the recipe's properties from buildbot's properties.

	This retrieves the current factory properties from the master_config
	in the slave's checkout (the factory properties handed to us from the
	master might be out of date), and merges in the build properties.

	Using the values from the checkout allows us to do things like change
	the recipe and other factory properties for a builder without needing
	a master restart.
	"""
	master_properties = factory_properties.copy()
	master_properties.update(build_properties)

	mastername = master_properties.get('mastername')
	buildername = master_properties.get('buildername')
	slave_properties = {}
	if mastername and buildername:
	try:
	slave_properties = get_factory_properties_from_disk(
	mastername, buildername)
	except LookupError as e:
	if master_overrides_slave:
	print 'WARNING in annotated_run.py (non-fatal): %s' % e
	else:
	raise e

	properties = master_properties.copy()
	conflicting_properties = {}
	for name in slave_properties:
	if master_properties.get(name) != slave_properties[name]:
	conflicting_properties[name] = (master_properties.get(name),
	slave_properties[name])

	if conflicting_properties:
	print 'The following build properties differ between master and slave:'
	for name, (master_value, slave_value) in conflicting_properties.items():
	print (' "%s": master: "%s", slave: "%s"' % (
	name,
	"<unset>" if (master_value is None) else master_value,
	slave_value))
	print ("Using the values from the %s." %
	("master" if master_overrides_slave else "slave"))

	if not master_overrides_slave:
	for name, (_, slave_value) in conflicting_properties.items():
	properties[name] = slave_value

	return properties


	def get_factory_properties_from_disk(mastername, buildername):
	master_list = master_cfg_utils.GetMasters()
	master_path = None
	for name, path in master_list:
	if name == mastername:
	master_path = path

	if not master_path:
	raise LookupError('master "%s" not found.' % mastername)

	script_path = os.path.join(BUILD_ROOT, 'scripts', 'tools',
	'dump_master_cfg.py')

	with namedTempFile() as fname:
	dump_cmd = [sys.executable,
	script_path,
	master_path, fname]
	proc = subprocess.Popen(dump_cmd, cwd=BUILD_ROOT, stdout=subprocess.PIPE,
	stderr=subprocess.PIPE)
	out, err = proc.communicate()
	exit_code = proc.returncode

	if exit_code:
	raise LookupError('Failed to get the master config; dump_master_cfg %s'
	'returned %d):\n%s\n%s\n'% (
	mastername, exit_code, out, err))

	with open(fname, 'rU') as f:
	config = json.load(f)

	# Now extract just the factory properties for the requested builder
	# from the master config.
	props = {}
	found = False
	for builder_dict in config['builders']:
	if builder_dict['name'] == buildername:
	found = True
	factory_properties = builder_dict['factory']['properties']
	for name, (value, _) in factory_properties.items():
	props[name] = value

	if not found:
	raise LookupError('builder "%s" not found on in master "%s"' %
	(buildername, mastername))

	if 'recipe' not in props:
	raise LookupError('Cannot find recipe for %s on %s' %
	(buildername, mastername))

	return props


	def get_args(argv):
	"""Process command-line arguments."""

	parser = optparse.OptionParser(
	description='Entry point for annotated builds.')
	parser.add_option('--build-properties',
	action='callback', callback=chromium_utils.convert_json,
	type='string', default={},
	help='build properties in JSON format')
	parser.add_option('--factory-properties',
	action='callback', callback=chromium_utils.convert_json,
	type='string', default={},
	help='factory properties in JSON format')
	parser.add_option('--build-properties-gz',
	action='callback', callback=chromium_utils.convert_gz_json,
	type='string', default={}, dest='build_properties',
	help='build properties in b64 gz JSON format')
	parser.add_option('--factory-properties-gz',
	action='callback', callback=chromium_utils.convert_gz_json,
	type='string', default={}, dest='factory_properties',
	help='factory properties in b64 gz JSON format')
	parser.add_option('--keep-stdin', action='store_true', default=False,
	help='don\'t close stdin when running recipe steps')
	parser.add_option('--master-overrides-slave', action='store_true',
	help='use the property values given on the command line '
	'from the master, not the ones looked up on the slave')
	return parser.parse_args(argv)


	def update_scripts():
	if os.environ.get('RUN_SLAVE_UPDATED_SCRIPTS'):
	os.environ.pop('RUN_SLAVE_UPDATED_SCRIPTS')
	return False

	stream = annotator.StructuredAnnotationStream()
	git_cmd = 'git.bat' if os.name == "nt" else 'git'
	with stream.step('update_scripts') as s:
	fetch_cmd = [git_cmd, 'fetch', '--all']
	reset_cmd = [git_cmd, 'reset', '--hard', 'origin/emu-master-dev']
	checkout_cmd = [git_cmd, 'checkout', 'emu-master-dev']
	if subprocess.call(fetch_cmd) != 0 or subprocess.call(reset_cmd) != 0 or subprocess.call(checkout_cmd) != 0:
	s.step_text('git update source failed!')
	s.step_warnings()
	s.step_text('git pull')

	os.environ['RUN_SLAVE_UPDATED_SCRIPTS'] = '1'

	# After running update scripts, set PYTHONIOENCODING=UTF-8 for the real
	# annotated_run.
	os.environ['PYTHONIOENCODING'] = 'UTF-8'

	return True


	def clean_old_recipe_engine():
	"""Clean stale pycs from the old location of recipe_engine.

	This function should only be needed for a little while after the recipe
	packages rollout (2015-09-16).
	"""
	for (dirpath, _, filenames) in os.walk(
	os.path.join(BUILD_ROOT, 'third_party', 'recipe_engine')):
	for filename in filenames:
	if filename.endswith('.pyc'):
	path = os.path.join(dirpath, filename)
	os.remove(path)


	def main(argv):
	opts, _ = get_args(argv)
	properties = get_recipe_properties(
	opts.factory_properties, opts.build_properties,
	opts.master_overrides_slave)

	clean_old_recipe_engine()

	# Find out if the recipe we intend to run is in build_internal's recipes. If
	# so, use recipes.py from there, otherwise use the one from build.
	recipe_file = properties['recipe'].replace('/', os.path.sep) + '.py'
	if os.path.exists(os.path.join(BUILD_LIMITED_ROOT, 'recipes', recipe_file)):
	recipe_runner = os.path.join(BUILD_LIMITED_ROOT, 'recipes.py')
	else:
	recipe_runner = os.path.join(SCRIPT_PATH, 'recipes.py')

	with namedTempFile() as props_file:
	with open(props_file, 'w') as fh:
	fh.write(json.dumps(properties))
	cmd = [
	sys.executable, '-u', recipe_runner,
	'run',
	'--workdir=%s' % os.getcwd(),
	'--properties-file=%s' % props_file,
	properties['recipe'] ]
	return subprocess.call(cmd)


	def shell_main(argv):
	if update_scripts():
	return subprocess.call([sys.executable] + argv)
	else:
	return main(argv)


	if __name__ == '__main__':
	sys.exit(shell_main(sys.argv))