| #!/usr/bin/python3 -B | |
| # Copyright 2017 The Android Open Source Project | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| """Generates the timezone data files used by Android.""" | |
| import argparse | |
| import glob | |
| import os | |
| import re | |
| import subprocess | |
| import sys | |
| import tarfile | |
| import tempfile | |
| sys.path.append('%s/external/icu/tools' % os.environ.get('ANDROID_BUILD_TOP')) | |
| import i18nutil | |
| import icuutil | |
| import tzdatautil | |
| # Unix epoch timestamp in seconds up to which transitions in tzdata will be pregenerated. | |
| pregeneration_upper_bound = 4102444800 # 1 Jan 2100 | |
| # Calculate the paths that are referred to by multiple functions. | |
| android_build_top = i18nutil.GetAndroidRootOrDie() | |
| timezone_dir = os.path.realpath('%s/system/timezone' % android_build_top) | |
| i18nutil.CheckDirExists(timezone_dir, 'system/timezone') | |
| android_host_out = i18nutil.GetAndroidHostOutOrDie() | |
| zone_compactor_dir = os.path.realpath('%s/system/timezone/input_tools/android' % android_build_top) | |
| i18nutil.CheckDirExists(zone_compactor_dir, 'system/timezone/input_tools/android') | |
| timezone_input_tools_dir = os.path.realpath('%s/input_tools' % timezone_dir) | |
| timezone_input_data_dir = os.path.realpath('%s/input_data' % timezone_dir) | |
| timezone_output_data_dir = '%s/output_data' % timezone_dir | |
| i18nutil.CheckDirExists(timezone_output_data_dir, 'output_data') | |
| tmp_dir = tempfile.mkdtemp('-tzdata') | |
| def GenerateZicInputFile(extracted_iana_data_dir): | |
| # Android APIs assume DST means "summer time" so we follow the rearguard format | |
| # introduced in 2018e. | |
| zic_input_file_name = 'rearguard.zi' | |
| # 'NDATA=' is used to remove unnecessary rules files. | |
| subprocess.check_call(['make', '-C', extracted_iana_data_dir, 'NDATA=', zic_input_file_name]) | |
| zic_input_file = '%s/%s' % (extracted_iana_data_dir, zic_input_file_name) | |
| if not os.path.exists(zic_input_file): | |
| print('Could not find %s' % zic_input_file) | |
| sys.exit(1) | |
| return zic_input_file | |
| def WriteSetupFile(zic_input_file): | |
| """Writes the list of zones that ZoneCompactor should process.""" | |
| links = [] | |
| zones = [] | |
| for line in open(zic_input_file): | |
| fields = line.split() | |
| if fields: | |
| line_type = fields[0] | |
| if line_type == 'Link': | |
| # Each "Link" line requires the creation of a link from an old tz ID to | |
| # a new tz ID, and implies the existence of a zone with the old tz ID. | |
| # | |
| # IANA terminology: | |
| # TARGET = the new tz ID, LINK-NAME = the old tz ID | |
| target = fields[1] | |
| link_name = fields[2] | |
| links.append('Link %s %s' % (target, link_name)) | |
| zones.append('Zone %s' % link_name) | |
| elif line_type == 'Zone': | |
| # Each "Zone" line indicates the existence of a tz ID. | |
| # | |
| # IANA terminology: | |
| # NAME is the tz ID, other fields like STDOFF, RULES, FORMAT,[UNTIL] are | |
| # ignored. | |
| name = fields[1] | |
| zones.append('Zone %s' % name) | |
| zone_compactor_setup_file = '%s/setup' % tmp_dir | |
| setup = open(zone_compactor_setup_file, 'w') | |
| # Ordering requirement from ZoneCompactor: Links must come first. | |
| for link in sorted(set(links)): | |
| setup.write('%s\n' % link) | |
| for zone in sorted(set(zones)): | |
| setup.write('%s\n' % zone) | |
| setup.close() | |
| return zone_compactor_setup_file | |
| def BuildIcuData(iana_data_tar_file): | |
| icu_build_dir = '%s/icu' % tmp_dir | |
| icuutil.PrepareIcuBuild(icu_build_dir) | |
| icuutil.MakeTzDataFiles(icu_build_dir, iana_data_tar_file) | |
| # Create ICU system image files. | |
| icuutil.MakeAndCopyIcuDataFiles(icu_build_dir) | |
| # Create the ICU's .res time zone files. | |
| icu_overlay_dir = '%s/icu_overlay' % timezone_output_data_dir | |
| icuutil.MakeAndCopyIcuTzFiles(icu_build_dir, icu_overlay_dir) | |
| # There are files in ICU which generation depends on ICU itself, | |
| # so multiple builds might be needed. | |
| icuutil.GenerateIcuDataFiles() | |
| # Copy ICU license file(s) | |
| icuutil.CopyLicenseFiles(icu_overlay_dir) | |
| def GetIanaVersion(iana_tar_file): | |
| iana_tar_filename = os.path.basename(iana_tar_file) | |
| iana_version = re.search('tz(?:data|code)(.+)\\.tar\\.gz', iana_tar_filename).group(1) | |
| return iana_version | |
| def ExtractTarFile(tar_file, dir): | |
| print('Extracting %s...' % tar_file) | |
| if not os.path.exists(dir): | |
| os.mkdir(dir) | |
| tar = tarfile.open(tar_file, 'r') | |
| tar.extractall(dir) | |
| def BuildZic(iana_tools_dir): | |
| iana_zic_code_tar_file = tzdatautil.GetIanaTarFile(iana_tools_dir, 'tzcode') | |
| iana_zic_code_version = GetIanaVersion(iana_zic_code_tar_file) | |
| iana_zic_data_tar_file = tzdatautil.GetIanaTarFile(iana_tools_dir, 'tzdata') | |
| iana_zic_data_version = GetIanaVersion(iana_zic_data_tar_file) | |
| print('Found IANA zic release %s/%s in %s/%s ...' \ | |
| % (iana_zic_code_version, iana_zic_data_version, iana_zic_code_tar_file, | |
| iana_zic_data_tar_file)) | |
| zic_build_dir = '%s/zic' % tmp_dir | |
| ExtractTarFile(iana_zic_code_tar_file, zic_build_dir) | |
| ExtractTarFile(iana_zic_data_tar_file, zic_build_dir) | |
| # zic | |
| print('Building zic...') | |
| # VERSION_DEPS= is to stop the build process looking for files that might not | |
| # be present across different versions. | |
| subprocess.check_call(['make', '-C', zic_build_dir, 'zic']) | |
| zic_binary_file = '%s/zic' % zic_build_dir | |
| if not os.path.exists(zic_binary_file): | |
| print('Could not find %s' % zic_binary_file) | |
| sys.exit(1) | |
| return zic_binary_file | |
| def BuildTzdata(zic_binary_file, extracted_iana_data_dir, iana_data_version): | |
| print('Generating zic input file...') | |
| zic_input_file = GenerateZicInputFile(extracted_iana_data_dir) | |
| print('Calling zic...') | |
| zic_output_dir = '%s/data' % tmp_dir | |
| os.mkdir(zic_output_dir) | |
| # -R specifies upper bound for generated transitions. | |
| zic_cmd = [zic_binary_file, '-b', 'slim', '-R', f'@{pregeneration_upper_bound}', '-d', zic_output_dir, zic_input_file] | |
| subprocess.check_call(zic_cmd) | |
| # ZoneCompactor | |
| zone_compactor_setup_file = WriteSetupFile(zic_input_file) | |
| print('Calling ZoneCompactor to update tzdata to %s...' % iana_data_version) | |
| tzdatautil.InvokeSoong(android_build_top, ['zone_compactor']) | |
| # Create args for ZoneCompactor | |
| header_string = 'tzdata%s' % iana_data_version | |
| print('Executing ZoneCompactor...') | |
| command = '%s/bin/zone_compactor' % android_host_out | |
| iana_output_data_dir = '%s/iana' % timezone_output_data_dir | |
| subprocess.check_call([command, zone_compactor_setup_file, zic_output_dir, iana_output_data_dir, | |
| header_string]) | |
| def BuildTzlookupAndTzIds(iana_data_dir): | |
| countryzones_source_file = '%s/android/countryzones.txt' % timezone_input_data_dir | |
| tzlookup_dest_file = '%s/android/tzlookup.xml' % timezone_output_data_dir | |
| tzids_dest_file = '%s/android/tzids.prototxt' % timezone_output_data_dir | |
| print('Calling TzLookupGenerator to create tzlookup.xml / tzids.prototxt...') | |
| tzdatautil.InvokeSoong(android_build_top, ['tzlookup_generator']) | |
| zone_tab_file = '%s/zone.tab' % iana_data_dir | |
| command = '%s/bin/tzlookup_generator' % android_host_out | |
| subprocess.check_call([command, countryzones_source_file, zone_tab_file, tzlookup_dest_file, | |
| tzids_dest_file]) | |
| def BuildTelephonylookup(): | |
| telephonylookup_source_file = '%s/android/telephonylookup.txt' % timezone_input_data_dir | |
| telephonylookup_dest_file = '%s/android/telephonylookup.xml' % timezone_output_data_dir | |
| print('Calling TelephonyLookupGenerator to create telephonylookup.xml...') | |
| tzdatautil.InvokeSoong(android_build_top, ['telephonylookup_generator']) | |
| command = '%s/bin/telephonylookup_generator' % android_host_out | |
| subprocess.check_call([command, telephonylookup_source_file, telephonylookup_dest_file]) | |
| def CreateTzVersion(iana_data_version, android_revision, output_version_file): | |
| create_tz_version_script = '%s/input_tools/version/create-tz_version.py' % timezone_dir | |
| subprocess.check_call([create_tz_version_script, | |
| '-iana_version', iana_data_version, | |
| '-revision', str(android_revision), | |
| '-output_version_file', output_version_file]) | |
| def UpdateTestFiles(): | |
| testing_data_dir = '%s/testing/data' % timezone_dir | |
| update_test_files_script = '%s/create-test-data.sh' % testing_data_dir | |
| subprocess.check_call([update_test_files_script], cwd=testing_data_dir) | |
| # Run from any directory, with no special setup required. | |
| # In the rare case when tzdata has to be updated, but under the same version, | |
| # pass "-revision" argument. | |
| # See http://www.iana.org/time-zones/ for more about the source of this data. | |
| def main(): | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument('-revision', type=int, default=1, | |
| help='Revision of current the IANA version, default = 1') | |
| args = parser.parse_args() | |
| android_revision = args.revision | |
| print('Source data file structure: %s' % timezone_input_data_dir) | |
| print('Source tools file structure: %s' % timezone_input_tools_dir) | |
| print('Intermediate / working dir: %s' % tmp_dir) | |
| print('Output data file structure: %s' % timezone_output_data_dir) | |
| iana_input_data_dir = '%s/iana' % timezone_input_data_dir | |
| iana_data_tar_file = tzdatautil.GetIanaTarFile(iana_input_data_dir, 'tzdata') | |
| iana_data_version = GetIanaVersion(iana_data_tar_file) | |
| print('IANA time zone data release %s in %s ...' % (iana_data_version, iana_data_tar_file)) | |
| icu_dir = icuutil.icuDir() | |
| print('Found icu in %s ...' % icu_dir) | |
| BuildIcuData(iana_data_tar_file) | |
| iana_tools_dir = '%s/iana' % timezone_input_tools_dir | |
| zic_binary_file = BuildZic(iana_tools_dir) | |
| iana_data_dir = '%s/iana_data' % tmp_dir | |
| ExtractTarFile(iana_data_tar_file, iana_data_dir) | |
| BuildTzdata(zic_binary_file, iana_data_dir, iana_data_version) | |
| BuildTzlookupAndTzIds(iana_data_dir) | |
| BuildTelephonylookup() | |
| # Create a version file from the output from prior stages. | |
| output_version_file = '%s/version/tz_version' % timezone_output_data_dir | |
| CreateTzVersion(iana_data_version, android_revision, output_version_file) | |
| # Update test versions of data files too. | |
| UpdateTestFiles() | |
| print('Look in %s and %s for new files' % (timezone_output_data_dir, icu_dir)) | |
| sys.exit(0) | |
| if __name__ == '__main__': | |
| main() |