import_official_snapshot.py - platform/external/wayland - Git at Google

 #!/usr/bin/python3
 # Copyright 2021 The Android Open Source Project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #      http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.

 import argparse
 import datetime
 import os
 import os.path
 import re
 import subprocess
 import sys
 """
 Helper script for importing a new snapshot of the official Wayland sources.

 Usage: ./import_official_snapshot.py 1.18.0
 """


 def git(cmd, check=True):
     return subprocess.run(['git'] + cmd,
                           capture_output=True,
                           check=check,
                           text=True)


 def git_get_hash(commit):
     return git(['show-ref', '--head', '--hash', commit]).stdout.strip()


 def git_is_ref(ref):
     return git(
         ['show-ref', '--head', '--hash', '--verify', f'refs/heads/{ref}'],
         check=False).returncode == 0


 def get_git_files(version):
     return set(
         git(['ls-tree', '-r', '--name-only',
              f'{version}^{{tree}}']).stdout.split())


 def assert_no_uncommitted_changes():
     r = git(['diff-files', '--quiet', '--ignore-submodules'], check=False)
     if r.returncode:
         sys.exit('Error: Your tree is dirty')

     r = git(
         ['diff-index', '--quiet', '--ignore-submodules', '--cached', 'HEAD'],
         check=False)
     if r.returncode:
         sys.exit('Error: You have staged changes')


 def metadata_read_current_version():
     with open('METADATA', 'rt') as metadata_file:
         metadata = metadata_file.read()

     match = re.search(r'version: "([^"]*)"', metadata)
     if not match:
         sys.exit('Error: Unable to determine current version from METADATA')
     return match.group(1)


 def metadata_read_git_url():
     with open('METADATA', 'rt') as metadata_file:
         metadata = metadata_file.read()

     match = re.search(r'url\s*{\s*type:\s*GIT\s*value:\s*"([^"]*)"\s*}',
                       metadata)
     if not match:
         sys.exit('Error: Unable to determine GIT url from METADATA')
     return match.group(1)


 def setup_and_update_official_source_remote(official_source_git_url):
     r = git(['remote', 'get-url', 'official-source'], check=False)
     if r.returncode or r.stdout != official_source_git_url:
         # Not configured as expected.
         print(
             f'  Configuring official-source remote {official_source_git_url}')
         git(['remote', 'remove', 'official-source'], check=False)
         git(['remote', 'add', 'official-source', official_source_git_url])

     print('  Syncing official-source')
     git(['remote', 'update', 'official-source'])


 def get_local_files():
     result = []
     for root, dirs, files in os.walk('.'):
         # Don't include ./.git
         if root == '.' and '.git' in dirs:
             dirs.remove('.git')
         for name in files:
             result.append(os.path.join(root, name)[2:])
     return result


 def determine_files_to_preserve(current_version):
     local_files = set(get_local_files())

     current_official_files = get_git_files(current_version)

     android_added_files = local_files - current_official_files

     return android_added_files


 def update_metadata_version_and_import_date(version):
     now = datetime.datetime.now()

     with open('METADATA', 'rt') as metadata_file:
         metadata = metadata_file.read()

     metadata = re.sub(r'version: "[^"]*"', f'version: "{version}"', metadata)
     metadata = re.sub(
         r'last_upgrade_date {[^}]*}',
         (f'last_upgrade_date {{ year: {now.year} month: {now.month} '
          f'day: {now.day} }}'), metadata)

     with open('METADATA', 'wt') as metadata_file:
         metadata_file.write(metadata)


 def configure_wayland_version_header(version):
     with open('./src/wayland-version.h.in', 'rt') as template_file:
         content = template_file.read()

     (major, minor, micro) = version.split('.')

     content = re.sub(r'@WAYLAND_VERSION_MAJOR@', major, content)
     content = re.sub(r'@WAYLAND_VERSION_MINOR@', minor, content)
     content = re.sub(r'@WAYLAND_VERSION_MICRO@', micro, content)
     content = re.sub(r'@WAYLAND_VERSION@', version, content)

     with open('./src/wayland-version.h', 'wt') as version_header:
         version_header.write(content)

     # wayland-version.h is in .gitignore, so we explicitly have to force-add it.
     git(['add', '-f', './src/wayland-version.h'])


 def import_sources(version, preserve_files, update_metdata=True):
     start_hash = git_get_hash('HEAD')

     # Use `git-read-tree` to start with a pure copy of the imported version
     git(['read-tree', '-m', '-u', f'{version}^{{tree}}'])

     git(['commit', '-m', f'To squash: Clean import of {version}'])

     print('  Adding Android metadata')

     # Restore the needed Android files
     git(['restore', '--staged', '--worktree', '--source', start_hash] +
         list(sorted(preserve_files)))

     if update_metdata:
         update_metadata_version_and_import_date(version)
         configure_wayland_version_header(version)

     git(['commit', '-a', '-m', f'To squash: Update versions {version}'])


 def apply_and_reexport_patches(version, patches, use_cherry_pick=False):
     if not patches:
         return

     print(f'  Applying {len(patches)} Android patches')

     try:
         if use_cherry_pick:
             git(['cherry-pick'] + patches)
         else:
             git(['am'] + patches)
     except subprocess.CalledProcessError as e:
         if 'patch failed' not in e.stderr:
             raise
         # Print out the captured error mess
         sys.stderr.write(f'''
 Failure applying patches to Wayland {version} via:
     {e.cmd}

 Once the patches have been resolved, please re-export the patches with:

     git rm patches/*.diff
     git format-patch HEAD~{len(patches)}..HEAD --no-stat --no-signature \\
         --numbered --zero-commit --suffix=.diff --output-directory patches

 ... and also add them to the final squashed commit.
                 '''.strip())

         sys.stdout.write(e.stdout)
         sys.exit(e.stderr)

     patch_hashes = list(
         reversed(
             git(['log', f'-{len(patches)}',
                  '--pretty=format:%H']).stdout.split()))

     # Clean out the existing patches
     git(['rm', 'patches/*.diff'])

     # Re-export the patches, omitting information that might change
     git([
         'format-patch', f'HEAD~{len(patches)}..HEAD', '--no-stat',
         '--no-signature', '--numbered', '--zero-commit', '--suffix=.diff',
         '--output-directory', 'patches'
     ])

     # Add back all the exported patches
     git(['add', 'patches/*.diff'])

     # Create a commit for the exported patches if there are any differences.
     r = git(['diff-files', '--quiet', '--ignore-submodules'], check=False)
     if r.returncode:
         git(['commit', '-a', '-m', f'To squash: Update patches for {version}'])

     return patch_hashes


 def main():
     parser = argparse.ArgumentParser(description=(
         "Helper script for importing a snapshot of the Wayland sources."))
     parser.add_argument('version',
                         nargs='?',
                         default=None,
                         help='The official version to import')
     parser.add_argument(
         '--no-validate-existing',
         dest='validate_existing',
         default=True,
         action='store_false',
         help='Whether to validate the current tree against upstream + patches')
     parser.add_argument('--no-squash',
                         dest='squash',
                         default=True,
                         action='store_false',
                         help='Whether to squash the import to a single commit')
     args = parser.parse_args()

     print(
         f'Preparing to importing Wayland core sources version {args.version}')

     assert_no_uncommitted_changes()

     official_source_git_url = metadata_read_git_url()
     current_version = metadata_read_current_version()

     setup_and_update_official_source_remote(official_source_git_url)

     # Get the list of Android added files to preserve
     preserve_files = determine_files_to_preserve(current_version)

     # Filter the list to get all patches that we will need to apply
     patch_files = sorted(path for path in preserve_files
                          if path.startswith('patches/'))

     # Detect any add/add conflicts before we begin
     new_files = get_git_files(args.version or current_version)
     add_add_conflicts = preserve_files.intersection(new_files)
     if add_add_conflicts:
         sys.exit(f'''
 Error: The new version of Wayland adds files that are also added for Android:
 {add_add_conflicts}
                 '''.strip())

     import_branch_name = f'import_{args.version}' if args.version else None

     if import_branch_name and git_is_ref(import_branch_name):
         sys.exit(f'''
 Error: Branch name {import_branch_name} already exists. Please delete or rename.
                 '''.strip())

     initial_commit_hash = git_get_hash('HEAD')

     # Begin a branch for the version import, if a new version is being imported
     if import_branch_name:
         git(['checkout', '-b', import_branch_name])
         git([
             'commit', '--allow-empty', '-m',
             f'Update to Wayland {args.version}'
         ])

     patch_hashes = None

     if args.validate_existing:
         print(f'Importing {current_version} to validate all current fixups')
         import_sources(current_version, preserve_files, update_metdata=False)
         patch_hashes = apply_and_reexport_patches(current_version, patch_files)

         r = git(
             ['diff', '--quiet', '--ignore-submodules', initial_commit_hash],
             check=False)
         if r.returncode:
             sys.exit(f'''
 Failed to recreate the pre-import tree by importing the prior Wayland version
 {current_version} and applying the Android fixups.

 This is likely due to changes having been made to the Wayland sources without
 a corresponding patch file in patches/.

 To see the differences detected, run:

 git diff {initial_commit_hash}
                     '''.strip())

     if args.version:
         print(f'Importing {args.version}')
         import_sources(args.version, preserve_files)
         apply_and_reexport_patches(
             args.version,
             (patch_hashes if patch_hashes is not None else patch_files),
             use_cherry_pick=(patch_hashes is not None))

         if args.squash:
             print('Squashing to one commit')
             git(['reset', '--soft', initial_commit_hash])
             git([
                 'commit', '--allow-empty', '-m', f'''
 Update to Wayland {args.version}

 Automatic import using "./import_official_snapshot.py {args.version}"
                 '''.strip()
             ])


 if __name__ == '__main__':
     main()
	#!/usr/bin/python3
	# Copyright 2021 The Android Open Source Project
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	import argparse
	import datetime
	import os
	import os.path
	import re
	import subprocess
	import sys
	"""
	Helper script for importing a new snapshot of the official Wayland sources.

	Usage: ./import_official_snapshot.py 1.18.0
	"""


	def git(cmd, check=True):
	return subprocess.run(['git'] + cmd,
	capture_output=True,
	check=check,
	text=True)


	def git_get_hash(commit):
	return git(['show-ref', '--head', '--hash', commit]).stdout.strip()


	def git_is_ref(ref):
	return git(
	['show-ref', '--head', '--hash', '--verify', f'refs/heads/{ref}'],
	check=False).returncode == 0


	def get_git_files(version):
	return set(
	git(['ls-tree', '-r', '--name-only',
	f'{version}^{{tree}}']).stdout.split())


	def assert_no_uncommitted_changes():
	r = git(['diff-files', '--quiet', '--ignore-submodules'], check=False)
	if r.returncode:
	sys.exit('Error: Your tree is dirty')

	r = git(
	['diff-index', '--quiet', '--ignore-submodules', '--cached', 'HEAD'],
	check=False)
	if r.returncode:
	sys.exit('Error: You have staged changes')


	def metadata_read_current_version():
	with open('METADATA', 'rt') as metadata_file:
	metadata = metadata_file.read()

	match = re.search(r'version: "([^"]*)"', metadata)
	if not match:
	sys.exit('Error: Unable to determine current version from METADATA')
	return match.group(1)


	def metadata_read_git_url():
	with open('METADATA', 'rt') as metadata_file:
	metadata = metadata_file.read()

	match = re.search(r'url\s{\stype:\sGIT\svalue:\s"([^"])"\s*}',
	metadata)
	if not match:
	sys.exit('Error: Unable to determine GIT url from METADATA')
	return match.group(1)


	def setup_and_update_official_source_remote(official_source_git_url):
	r = git(['remote', 'get-url', 'official-source'], check=False)
	if r.returncode or r.stdout != official_source_git_url:
	# Not configured as expected.
	print(
	f' Configuring official-source remote {official_source_git_url}')
	git(['remote', 'remove', 'official-source'], check=False)
	git(['remote', 'add', 'official-source', official_source_git_url])

	print(' Syncing official-source')
	git(['remote', 'update', 'official-source'])


	def get_local_files():
	result = []
	for root, dirs, files in os.walk('.'):
	# Don't include ./.git
	if root == '.' and '.git' in dirs:
	dirs.remove('.git')
	for name in files:
	result.append(os.path.join(root, name)[2:])
	return result


	def determine_files_to_preserve(current_version):
	local_files = set(get_local_files())

	current_official_files = get_git_files(current_version)

	android_added_files = local_files - current_official_files

	return android_added_files


	def update_metadata_version_and_import_date(version):
	now = datetime.datetime.now()

	with open('METADATA', 'rt') as metadata_file:
	metadata = metadata_file.read()

	metadata = re.sub(r'version: "[^"]*"', f'version: "{version}"', metadata)
	metadata = re.sub(
	r'last_upgrade_date {[^}]*}',
	(f'last_upgrade_date {{ year: {now.year} month: {now.month} '
	f'day: {now.day} }}'), metadata)

	with open('METADATA', 'wt') as metadata_file:
	metadata_file.write(metadata)


	def configure_wayland_version_header(version):
	with open('./src/wayland-version.h.in', 'rt') as template_file:
	content = template_file.read()

	(major, minor, micro) = version.split('.')

	content = re.sub(r'@WAYLAND_VERSION_MAJOR@', major, content)
	content = re.sub(r'@WAYLAND_VERSION_MINOR@', minor, content)
	content = re.sub(r'@WAYLAND_VERSION_MICRO@', micro, content)
	content = re.sub(r'@WAYLAND_VERSION@', version, content)

	with open('./src/wayland-version.h', 'wt') as version_header:
	version_header.write(content)

	# wayland-version.h is in .gitignore, so we explicitly have to force-add it.
	git(['add', '-f', './src/wayland-version.h'])


	def import_sources(version, preserve_files, update_metdata=True):
	start_hash = git_get_hash('HEAD')

	# Use `git-read-tree` to start with a pure copy of the imported version
	git(['read-tree', '-m', '-u', f'{version}^{{tree}}'])

	git(['commit', '-m', f'To squash: Clean import of {version}'])

	print(' Adding Android metadata')

	# Restore the needed Android files
	git(['restore', '--staged', '--worktree', '--source', start_hash] +
	list(sorted(preserve_files)))

	if update_metdata:
	update_metadata_version_and_import_date(version)
	configure_wayland_version_header(version)

	git(['commit', '-a', '-m', f'To squash: Update versions {version}'])


	def apply_and_reexport_patches(version, patches, use_cherry_pick=False):
	if not patches:
	return

	print(f' Applying {len(patches)} Android patches')

	try:
	if use_cherry_pick:
	git(['cherry-pick'] + patches)
	else:
	git(['am'] + patches)
	except subprocess.CalledProcessError as e:
	if 'patch failed' not in e.stderr:
	raise
	# Print out the captured error mess
	sys.stderr.write(f'''
	Failure applying patches to Wayland {version} via:
	{e.cmd}

	Once the patches have been resolved, please re-export the patches with:

	git rm patches/*.diff
	git format-patch HEAD~{len(patches)}..HEAD --no-stat --no-signature \\
	--numbered --zero-commit --suffix=.diff --output-directory patches

	... and also add them to the final squashed commit.
	'''.strip())

	sys.stdout.write(e.stdout)
	sys.exit(e.stderr)

	patch_hashes = list(
	reversed(
	git(['log', f'-{len(patches)}',
	'--pretty=format:%H']).stdout.split()))

	# Clean out the existing patches
	git(['rm', 'patches/*.diff'])

	# Re-export the patches, omitting information that might change
	git([
	'format-patch', f'HEAD~{len(patches)}..HEAD', '--no-stat',
	'--no-signature', '--numbered', '--zero-commit', '--suffix=.diff',
	'--output-directory', 'patches'
	])

	# Add back all the exported patches
	git(['add', 'patches/*.diff'])

	# Create a commit for the exported patches if there are any differences.
	r = git(['diff-files', '--quiet', '--ignore-submodules'], check=False)
	if r.returncode:
	git(['commit', '-a', '-m', f'To squash: Update patches for {version}'])

	return patch_hashes


	def main():
	parser = argparse.ArgumentParser(description=(
	"Helper script for importing a snapshot of the Wayland sources."))
	parser.add_argument('version',
	nargs='?',
	default=None,
	help='The official version to import')
	parser.add_argument(
	'--no-validate-existing',
	dest='validate_existing',
	default=True,
	action='store_false',
	help='Whether to validate the current tree against upstream + patches')
	parser.add_argument('--no-squash',
	dest='squash',
	default=True,
	action='store_false',
	help='Whether to squash the import to a single commit')
	args = parser.parse_args()

	print(
	f'Preparing to importing Wayland core sources version {args.version}')

	assert_no_uncommitted_changes()

	official_source_git_url = metadata_read_git_url()
	current_version = metadata_read_current_version()

	setup_and_update_official_source_remote(official_source_git_url)

	# Get the list of Android added files to preserve
	preserve_files = determine_files_to_preserve(current_version)

	# Filter the list to get all patches that we will need to apply
	patch_files = sorted(path for path in preserve_files
	if path.startswith('patches/'))

	# Detect any add/add conflicts before we begin
	new_files = get_git_files(args.version or current_version)
	add_add_conflicts = preserve_files.intersection(new_files)
	if add_add_conflicts:
	sys.exit(f'''
	Error: The new version of Wayland adds files that are also added for Android:
	{add_add_conflicts}
	'''.strip())

	import_branch_name = f'import_{args.version}' if args.version else None

	if import_branch_name and git_is_ref(import_branch_name):
	sys.exit(f'''
	Error: Branch name {import_branch_name} already exists. Please delete or rename.
	'''.strip())

	initial_commit_hash = git_get_hash('HEAD')

	# Begin a branch for the version import, if a new version is being imported
	if import_branch_name:
	git(['checkout', '-b', import_branch_name])
	git([
	'commit', '--allow-empty', '-m',
	f'Update to Wayland {args.version}'
	])

	patch_hashes = None

	if args.validate_existing:
	print(f'Importing {current_version} to validate all current fixups')
	import_sources(current_version, preserve_files, update_metdata=False)
	patch_hashes = apply_and_reexport_patches(current_version, patch_files)

	r = git(
	['diff', '--quiet', '--ignore-submodules', initial_commit_hash],
	check=False)
	if r.returncode:
	sys.exit(f'''
	Failed to recreate the pre-import tree by importing the prior Wayland version
	{current_version} and applying the Android fixups.

	This is likely due to changes having been made to the Wayland sources without
	a corresponding patch file in patches/.

	To see the differences detected, run:

	git diff {initial_commit_hash}
	'''.strip())

	if args.version:
	print(f'Importing {args.version}')
	import_sources(args.version, preserve_files)
	apply_and_reexport_patches(
	args.version,
	(patch_hashes if patch_hashes is not None else patch_files),
	use_cherry_pick=(patch_hashes is not None))

	if args.squash:
	print('Squashing to one commit')
	git(['reset', '--soft', initial_commit_hash])
	git([
	'commit', '--allow-empty', '-m', f'''
	Update to Wayland {args.version}

	Automatic import using "./import_official_snapshot.py {args.version}"
	'''.strip()
	])


	if __name__ == '__main__':
	main()