| #!/usr/bin/env python3 |
| |
| import json |
| import os |
| import sys |
| |
| PRINT_ORIGINAL_FULL = False |
| |
| # This flags are augmented with flags added to the json files but not present in .gn or .gni files |
| IGNORED_FLAGS = [ |
| '-D_DEBUG', |
| '-Werror', |
| '-Xclang', |
| '-target-feature', |
| '+crc', |
| '+crypto', |
| ] |
| IGNORED_DEFINES = [ |
| 'HAVE_ARM64_CRC32C=1' |
| ] |
| DEFAULT_CFLAGS = [ |
| '-DHAVE_ARM64_CRC32C=0', |
| '-DUSE_AURA=1', |
| '-DUSE_GLIB=1', |
| '-DUSE_NSS_CERTS=1', |
| '-DUSE_UDEV', |
| '-DUSE_X11=1', |
| '-DWEBRTC_ANDROID_PLATFORM_BUILD=1', |
| '-DWEBRTC_APM_DEBUG_DUMP=0', |
| '-D_FILE_OFFSET_BITS=64', |
| '-D_GNU_SOURCE', |
| '-D_LARGEFILE64_SOURCE', |
| '-D_LARGEFILE_SOURCE', |
| '-Wno-global-constructors', |
| '-Wno-implicit-const-int-float-conversion', |
| '-Wno-missing-field-initializers', |
| '-Wno-unreachable-code-aggressive', |
| '-Wno-unreachable-code-break', |
| ] |
| |
| DEFAULT_CFLAGS_BY_ARCH = { |
| 'x86': ['-mavx2', '-mfma', '-msse2', '-msse3'], |
| 'x64': ['-mavx2', '-mfma', '-msse2', '-msse3'], |
| 'arm': ['-mthumb'], |
| 'arm64': [], |
| 'riscv64': [], |
| } |
| |
| FLAGS = ['cflags', 'cflags_c', 'cflags_cc', 'asmflags'] |
| FLAG_NAME_MAP = { |
| 'cflags': 'cflags', |
| 'asmflags': 'asflags', |
| 'cflags_cc': 'cppflags', |
| 'cflags_c': 'conlyflags', |
| } |
| |
| ARCH_NAME_MAP = {n: n for n in DEFAULT_CFLAGS_BY_ARCH.keys()} |
| ARCH_NAME_MAP['x64'] = 'x86_64' |
| |
| ARCHS = sorted(ARCH_NAME_MAP.keys()) |
| |
| def FormatList(l): |
| return json.dumps(sorted(list(l))) |
| |
| def IsInclude(name): |
| return name.endswith('.h') or name.endswith('.inc') |
| |
| def FilterIncludes(l): |
| return filter(lambda x: not IsInclude(x), l) |
| |
| def PrintOrigin(target): |
| print('/* From target:') |
| if PRINT_ORIGINAL_FULL: |
| print(json.dumps(target, sort_keys = True, indent = 4)) |
| else: |
| print(target['original_name']) |
| print('*/') |
| |
| def MakeRelatives(l): |
| return map(lambda x: x.split('//').pop(), l) |
| |
| def FormatName(name): |
| return 'webrtc_' + name.split('/').pop().replace(':', '__') |
| |
| def FormatNames(target): |
| target['original_name'] = target['name'] |
| target['name'] = FormatName(target['name']) |
| target['deps'] = sorted([FormatName(d) for d in target['deps']]) |
| return target |
| |
| def FilterFlags(flags, to_skip = set()): |
| skipped_opts = set(IGNORED_FLAGS).union(to_skip) |
| return [x for x in flags if not any([x.startswith(y) for y in skipped_opts])] |
| |
| def PrintHeader(): |
| print('package {') |
| print(' default_applicable_licenses: ["external_webrtc_license"],') |
| print('}') |
| print('') |
| print('// Added automatically by a large-scale-change that took the approach of') |
| print('// \'apply every license found to every target\'. While this makes sure we respect') |
| print('// every license restriction, it may not be entirely correct.') |
| print('//') |
| print('// e.g. GPL in an MIT project might only apply to the contrib/ directory.') |
| print('//') |
| print('// Please consider splitting the single license below into multiple licenses,') |
| print('// taking care not to lose any license_kind information, and overriding the') |
| print('// default license using the \'licenses: [...]\' property on targets as needed.') |
| print('//') |
| print('// For unused files, consider creating a \'fileGroup\' with "//visibility:private"') |
| print('// to attach the license to, and including a comment whether the files may be') |
| print('// used in the current project.') |
| print('//') |
| print('// large-scale-change included anything that looked like it might be a license') |
| print('// text as a license_text. e.g. LICENSE, NOTICE, COPYING etc.') |
| print('//') |
| print('// Please consider removing redundant or irrelevant files from \'license_text:\'.') |
| print('// See: http://go/android-license-faq') |
| print('') |
| print('///////////////////////////////////////////////////////////////////////////////') |
| print('// Do not edit this file directly, it\'s automatically generated by a script. //') |
| print('// Modify android_tools/generate_android_bp.py and run that instead. //') |
| print('///////////////////////////////////////////////////////////////////////////////') |
| print('') |
| print('license {') |
| print(' name: "external_webrtc_license",') |
| print(' visibility: [":__subpackages__"],') |
| print(' license_kinds: [') |
| print(' "SPDX-license-identifier-Apache-2.0",') |
| print(' "SPDX-license-identifier-BSD",') |
| print(' "SPDX-license-identifier-MIT",') |
| print(' "SPDX-license-identifier-Zlib",') |
| print(' "legacy_notice",') |
| print(' "legacy_unencumbered",') |
| print(' ],') |
| print(' license_text: [') |
| print(' "LICENSE",') |
| print(' "PATENTS",') |
| print(' "license_template.txt",') |
| print(' ],') |
| print('}') |
| |
| |
| |
| def GatherDefaultFlags(targets_by_arch): |
| # Iterate through all of the targets for each architecture collecting the flags that |
| # are the same for all targets in that architecture. Use a list instead of a set |
| # to maintain the flag ordering, which may be significant (e.g. -Wno-shadow has to |
| # come after -Wshadow). |
| arch_default_flags = {} |
| for arch, targets in targets_by_arch.items(): |
| arch_default_flags[arch] = {} |
| for target in targets.values(): |
| typ = target['type'] |
| if typ != 'static_library': |
| continue |
| for flag_type in FLAGS: |
| if not flag_type in arch_default_flags: |
| arch_default_flags[arch][flag_type] = target[flag_type] |
| else: |
| target_flags = set(target[flag_type]) |
| flags = arch_default_flags[arch][flag_type] |
| flags[:] = [ x for x in flags if x in target_flags ] |
| for flag_type, flags in arch_default_flags[arch].items(): |
| arch_default_flags[arch][flag_type] = FilterFlags(flags) |
| # Add in the hardcoded extra default cflags |
| arch_default_flags[arch]['cflags'] += DEFAULT_CFLAGS_BY_ARCH.get(arch, []) |
| |
| # Iterate through all of the architectures collecting the flags that are the same |
| # for all targets in all architectures. |
| default_flags = {} |
| for arch, flagset in arch_default_flags.items(): |
| for flag_type, arch_flags in flagset.items(): |
| if not flag_type in default_flags: |
| default_flags[flag_type] = arch_flags.copy() |
| else: |
| flags = default_flags[flag_type] |
| flags[:] = [ x for x in flags if x in arch_flags ] |
| # Add in the hardcoded extra default cflags |
| default_flags['cflags'] += DEFAULT_CFLAGS |
| |
| # Remove the global default flags from the per-architecture default flags |
| for arch, flagset in arch_default_flags.items(): |
| for flag_type in flagset.keys(): |
| flags = flagset[flag_type] |
| flags[:] = [ x for x in flags if x not in default_flags[flag_type] ] |
| |
| default_flags['arch'] = arch_default_flags |
| return default_flags |
| |
| def GenerateDefault(targets_by_arch): |
| in_default = GatherDefaultFlags(targets_by_arch) |
| print('cc_defaults {') |
| print(' name: "webrtc_defaults",') |
| print(' local_include_dirs: [') |
| print(' ".",') |
| print(' "webrtc",') |
| print(' "third_party/crc32c/src/include",') |
| print(' ],') |
| for typ in sorted(in_default.keys() - {'arch'}): |
| flags = in_default[typ] |
| if len(flags) > 0: |
| print(' {0}: ['.format(FLAG_NAME_MAP[typ])) |
| for flag in flags: |
| print(' "{0}",'.format(flag.replace('"', '\\"'))) |
| print(' ],') |
| print(' header_libs: [') |
| print(' "libwebrtc_absl_headers",') |
| print(' ],') |
| print(' static_libs: [') |
| print(' "libaom",') |
| print(' "libevent",') |
| print(' "libopus",') |
| print(' "libsrtp2",') |
| print(' "libvpx",') |
| print(' "libyuv",') |
| print(' "libpffft",') |
| print(' "rnnoise_rnn_vad",') |
| print(' ],') |
| print(' shared_libs: [') |
| print(' "libcrypto",') |
| print(' "libprotobuf-cpp-full",') |
| print(' "libprotobuf-cpp-lite",') |
| print(' "libssl",') |
| print(' ],') |
| print(' host_supported: true,') |
| print(' // vendor needed for libpreprocessing effects.') |
| print(' vendor: true,') |
| print(' target: {') |
| print(' darwin: {') |
| print(' enabled: false,') |
| print(' },') |
| print(' },') |
| print(' arch: {') |
| for a in ARCHS: |
| print(' {0}: {{'.format(ARCH_NAME_MAP[a])) |
| for typ in FLAGS: |
| flags = in_default['arch'].get(a, {}).get(typ, []) |
| if len(flags) > 0: |
| print(' {0}: ['.format(FLAG_NAME_MAP[typ])) |
| for flag in flags: |
| print(' "{0}",'.format(flag.replace('"', '\\"'))) |
| print(' ],') |
| print(' },') |
| print(' },') |
| print(' visibility: [') |
| print(' "//frameworks/av/media/libeffects/preprocessing:__subpackages__",') |
| print(' "//device/google/cuttlefish/host/frontend/webrtc:__subpackages__",') |
| print(' ],') |
| print('}') |
| |
| # The flags in the default entry can be safely removed from the targets |
| for arch, targets in targets_by_arch.items(): |
| for flag_type in FLAGS: |
| default_flags = set(in_default[flag_type]) | set(in_default['arch'][arch][flag_type]) |
| for target in targets.values(): |
| target[flag_type] = FilterFlags(target.get(flag_type, []), default_flags) |
| if len(target[flag_type]) == 0: |
| target.pop(flag_type) |
| |
| return in_default |
| |
| |
| def TransitiveDependencies(name, dep_type, targets): |
| target = targets[name] |
| field = 'transitive_' + dep_type |
| if field in target.keys(): |
| return target[field] |
| target[field] = {'global': set()} |
| for a in ARCHS: |
| target[field][a] = set() |
| if target['type'] == dep_type: |
| target[field]['global'].add(name) |
| for d in target.get('deps', []): |
| if targets[d]['type'] == dep_type: |
| target[field]['global'].add(d) |
| tDeps = TransitiveDependencies(d, dep_type, targets) |
| target[field]['global'] |= tDeps['global'] |
| for a in ARCHS: |
| target[field][a] |= tDeps[a] |
| if 'arch' in target: |
| for a, x in target['arch'].items(): |
| for d in x.get('deps', []): |
| tDeps = TransitiveDependencies(d, dep_type, targets) |
| target[field][a] |= tDeps['global'] | tDeps[a] |
| target[field][a] -= target[field]['global'] |
| |
| return target[field] |
| |
| def GenerateGroup(target): |
| # PrintOrigin(target) |
| pass |
| |
| def GenerateStaticLib(target, targets): |
| PrintOrigin(target) |
| name = target['name'] |
| print('cc_library_static {') |
| print(' name: "{0}",'.format(name)) |
| print(' defaults: ["webrtc_defaults"],') |
| sources = target.get('sources', []) |
| print(' srcs: {0},'.format(FormatList(sources))) |
| print(' host_supported: true,') |
| if 'asmflags' in target.keys(): |
| asmflags = target['asmflags'] |
| if len(asmflags) > 0: |
| print(' asflags: {0},'.format(FormatList(asmflags))) |
| if 'cflags' in target.keys(): |
| cflags = target['cflags'] |
| print(' cflags: {0},'.format(FormatList(cflags))) |
| if 'cflags_c' in target.keys(): |
| cflags_c = target['cflags_c'] |
| if len(cflags_c) > 0: |
| print(' conlyflags: {0},'.format(FormatList(cflags_c))) |
| if 'cflags_cc' in target.keys(): |
| cflags_cc = target['cflags_cc'] |
| if len(cflags_cc) > 0: |
| print(' cppflags: {0},'.format(FormatList(cflags_cc))) |
| if 'arch' in target: |
| print(' arch: {') |
| for arch_name in ARCHS: |
| if arch_name not in target['arch'].keys(): |
| continue |
| arch = target['arch'][arch_name] |
| print(' ' + ARCH_NAME_MAP[arch_name] + ': {') |
| if 'cflags' in arch.keys(): |
| cflags = arch['cflags'] |
| print(' cflags: {0},'.format(FormatList(cflags))) |
| if 'cflags_c' in arch.keys(): |
| cflags_c = arch['cflags_c'] |
| if len(cflags_c) > 0: |
| print(' conlyflags: {0},'.format(FormatList(cflags_c))) |
| if 'cflags_cc' in arch.keys(): |
| cflags_cc = arch['cflags_cc'] |
| if len(cflags_cc) > 0: |
| print(' cppflags: {0},'.format(FormatList(cflags_cc))) |
| if 'sources' in arch: |
| sources = arch['sources'] |
| print(' srcs: {0},'.format(FormatList(sources))) |
| if 'enabled' in arch: |
| print(' enabled: {0},'.format(arch['enabled'])) |
| print(' },') |
| print(' },') |
| print('}') |
| return name |
| |
| def DFS(seed, targets): |
| visited = set() |
| stack = [seed] |
| while len(stack) > 0: |
| nxt = stack.pop() |
| if nxt in visited: |
| continue |
| visited.add(nxt) |
| stack += targets[nxt]['deps'] |
| if 'arch' not in targets[nxt]: |
| continue |
| for arch in targets[nxt]['arch']: |
| if 'deps' in arch: |
| stack += arch['deps'] |
| return visited |
| |
| def Preprocess(project): |
| targets = {} |
| for name, target in project['targets'].items(): |
| target['name'] = name |
| targets[name] = target |
| if target['type'] == 'shared_library': |
| # Don't bother creating shared libraries |
| target['type'] = 'static_library' |
| if target['type'] == 'source_set': |
| # Convert source_sets to static libraires to avoid recompiling sources multiple times. |
| target['type'] = 'static_library' |
| if 'defines' in target: |
| target['cflags'] = target.get('cflags', []) + ['-D{0}'.format(d) for d in target['defines'] if d not in IGNORED_DEFINES] |
| target.pop('defines') |
| if 'sources' not in target: |
| continue |
| sources = list(MakeRelatives(FilterIncludes(target['sources']))) |
| if len(sources) > 0: |
| target['sources'] = sources |
| else: |
| target.pop('sources') |
| |
| # These dependencies are provided by aosp |
| ignored_targets = { |
| '//third_party/libaom:libaom', |
| '//third_party/libevent:libevent', |
| '//third_party/opus:opus', |
| '//third_party/libsrtp:libsrtp', |
| '//third_party/libvpx:libvpx', |
| '//third_party/libyuv:libyuv', |
| '//third_party/pffft:pffft', |
| '//third_party/rnnoise:rnn_vad', |
| '//third_party/boringssl:boringssl', |
| '//third_party/android_ndk:cpu_features', |
| '//buildtools/third_party/libunwind:libunwind', |
| '//buildtools/third_party/libc++:libc++', |
| } |
| for name, target in targets.items(): |
| # Skip all "action" targets |
| if target['type'] in {'action', 'action_foreach'}: |
| ignored_targets.add(name) |
| targets = {name: target for name, target in targets.items() if name not in ignored_targets} |
| |
| for target in targets.values(): |
| # Don't depend on ignored targets |
| target['deps'] = [d for d in target['deps'] if d not in ignored_targets] |
| |
| # Ignore empty static libraries |
| empty_libs = set() |
| for name, target in targets.items(): |
| if target['type'] == 'static_library' and 'sources' not in target and name != '//:webrtc': |
| empty_libs.add(name) |
| for empty_lib in empty_libs: |
| empty_lib_deps = targets[empty_lib].get('deps', []) |
| for target in targets.values(): |
| target['deps'] = FlattenEmptyLibs(target['deps'], empty_lib, empty_lib_deps) |
| for s in empty_libs: |
| targets.pop(s) |
| |
| # Select libwebrtc, libaudio_processing and its dependencies |
| selected = set() |
| selected |= DFS('//:webrtc', targets) |
| selected |= DFS('//modules/audio_processing:audio_processing', targets) |
| |
| return {FormatName(n): FormatNames(targets[n]) for n in selected} |
| |
| def _FlattenEmptyLibs(deps, empty_lib, empty_lib_deps): |
| for x in deps: |
| if x == empty_lib: |
| yield from empty_lib_deps |
| else: |
| yield x |
| |
| def FlattenEmptyLibs(deps, empty_lib, empty_lib_deps): |
| return list(_FlattenEmptyLibs(deps, empty_lib, empty_lib_deps)) |
| |
| def NonNoneFrom(l): |
| for a in l: |
| if a is not None: |
| return a |
| return None |
| |
| def MergeListField(target, f, target_by_arch): |
| set_by_arch = {} |
| for a, t in target_by_arch.items(): |
| if len(t) == 0: |
| # We only care about enabled archs |
| continue |
| set_by_arch[a] = set(t.get(f, [])) |
| |
| union = set() |
| for _, s in set_by_arch.items(): |
| union |= s |
| |
| common = union |
| for a, s in set_by_arch.items(): |
| common &= s |
| |
| not_common = {a: s - common for a,s in set_by_arch.items()} |
| |
| if len(common) > 0: |
| target[f] = list(common) |
| for a, s in not_common.items(): |
| if len(s) > 0: |
| target['arch'][a][f] = sorted(list(s)) |
| |
| def Merge(target_by_arch): |
| # The new target shouldn't have the transitive dependencies memoization fields |
| # or have the union of those fields from all 4 input targets. |
| target = {} |
| for f in ['original_name', 'name', 'type']: |
| target[f] = NonNoneFrom([t.get(f) for _,t in target_by_arch.items()]) |
| |
| target['arch'] = {} |
| for a, t in target_by_arch.items(): |
| target['arch'][a] = {} |
| if len(t) == 0: |
| target['arch'][a]['enabled'] = 'false' |
| |
| list_fields = ['sources', |
| 'deps', |
| 'cflags', |
| 'cflags_c', |
| 'cflags_cc', |
| 'asmflags'] |
| for lf in list_fields: |
| MergeListField(target, lf, target_by_arch) |
| |
| # Static libraries should be depended on at the root level and disabled for |
| # the corresponding architectures. |
| for arch in target['arch'].values(): |
| if 'deps' not in arch: |
| continue |
| deps = arch['deps'] |
| if 'deps' not in target: |
| target['deps'] = [] |
| target['deps'] += deps |
| arch.pop('deps') |
| if 'deps' in target: |
| target['deps'] = sorted(target['deps']) |
| |
| # Remove empty sets |
| for a in ARCHS: |
| if len(target['arch'][a]) == 0: |
| target['arch'].pop(a) |
| if len(target['arch']) == 0: |
| target.pop('arch') |
| |
| return target |
| |
| def DisabledArchs4Target(target): |
| ret = set() |
| for a in ARCHS: |
| if a not in target.get('arch', {}): |
| continue |
| if target['arch'][a].get('enabled', 'true') == 'false': |
| ret.add(a) |
| return ret |
| |
| |
| def HandleDisabledArchs(targets): |
| for n, t in targets.items(): |
| if 'arch' not in t: |
| continue |
| disabledArchs = DisabledArchs4Target(t) |
| if len(disabledArchs) == 0: |
| continue |
| # Fix targets that depend on this one |
| for t in targets.values(): |
| if DisabledArchs4Target(t) == disabledArchs: |
| # With the same disabled archs there is no need to move dependencies |
| continue |
| if 'deps' in t and n in t['deps']: |
| # Remove the dependency from the high level list |
| t['deps'] = sorted(set(t['deps']) - {n}) |
| if 'arch' not in t: |
| t['arch'] = {} |
| for a in ARCHS: |
| if a in disabledArchs: |
| continue |
| if a not in t['arch']: |
| t['arch'][a] = {} |
| if 'deps' not in t['arch'][a]: |
| t['arch'][a]['deps'] = [] |
| t['arch'][a]['deps'] += [n] |
| |
| def MergeAll(targets_by_arch): |
| names = set() |
| for t in targets_by_arch.values(): |
| names |= t.keys() |
| targets = {} |
| for name in names: |
| targets[name] = Merge({a: t.get(name, {}) for a,t in targets_by_arch.items()}) |
| |
| HandleDisabledArchs(targets) |
| |
| return targets |
| |
| def GatherAllFlags(obj): |
| if type(obj) != type({}): |
| # not a dictionary |
| return set() |
| ret = set() |
| for f in FLAGS: |
| ret |= set(obj.get(f, [])) |
| for v in obj.values(): |
| ret |= GatherAllFlags(v) |
| return ret |
| |
| def FilterFlagsInUse(flags, directory): |
| unused = [] |
| for f in flags: |
| nf = f |
| if nf.startswith("-D"): |
| nf = nf[2:] |
| i = nf.find('=') |
| if i > 0: |
| nf = nf[:i] |
| c = os.system(f"find {directory} -name '*.gn*' | xargs grep -q -s -e '{nf}'") |
| if c != 0: |
| # couldn't find the flag in *.gn or *.gni |
| unused.append(f) |
| return unused |
| |
| if len(sys.argv) != 2: |
| print('wrong number of arguments', file = sys.stderr) |
| exit(1) |
| |
| dir = sys.argv[1] |
| |
| targets_by_arch = {} |
| flags = set() |
| for arch in ARCHS: |
| path = "{0}/project_{1}.json".format(dir, arch) |
| json_file = open(path, 'r') |
| targets_by_arch[arch] = Preprocess(json.load(json_file)) |
| flags |= GatherAllFlags(targets_by_arch[arch]) |
| |
| unusedFlags = FilterFlagsInUse(flags, f"{dir}/..") |
| IGNORED_FLAGS = sorted(set(IGNORED_FLAGS) | set(unusedFlags)) |
| |
| PrintHeader() |
| |
| GenerateDefault(targets_by_arch) |
| |
| targets = MergeAll(targets_by_arch) |
| |
| print('\n\n') |
| |
| for name, target in sorted(targets.items()): |
| typ = target['type'] |
| if typ == 'static_library': |
| GenerateStaticLib(target, targets) |
| elif typ == 'group': |
| GenerateGroup(target) |
| else: |
| print('Unknown type: {0} ({1})'.format(typ, target['name']), file = sys.stderr) |
| exit(1) |
| print('\n\n') |
| |
| webrtc_libs = TransitiveDependencies(FormatName('//:webrtc'), 'static_library', targets) |
| print('cc_library_static {') |
| print(' name: "libwebrtc",') |
| print(' defaults: ["webrtc_defaults"],') |
| print(' export_include_dirs: ["."],') |
| print(' whole_static_libs: {0},'.format(FormatList(sorted(webrtc_libs['global']) + ['libpffft', 'rnnoise_rnn_vad']))) |
| print(' arch: {') |
| for a in ARCHS: |
| if len(webrtc_libs[a]) > 0: |
| print(' {0}: {{'.format(ARCH_NAME_MAP[a])) |
| print(' whole_static_libs: {0},'.format(FormatList(sorted(webrtc_libs[a])))) |
| print(' },') |
| print(' },') |
| print('}') |
| |
| print('\n\n') |
| |
| audio_proc_libs = TransitiveDependencies(FormatName('//modules/audio_processing:audio_processing'), 'static_library', targets) |
| print('cc_library_static {') |
| print(' name: "webrtc_audio_processing",') |
| print(' defaults: ["webrtc_defaults"],') |
| print(' export_include_dirs: [') |
| print(' ".",') |
| print(' "modules/include",') |
| print(' "modules/audio_processing/include",') |
| print(' ],') |
| print(' whole_static_libs: {0},'.format(FormatList(sorted(audio_proc_libs['global']) + ['libpffft', 'rnnoise_rnn_vad']))) |
| print(' arch: {') |
| for a in ARCHS: |
| if len(audio_proc_libs[a]) > 0: |
| print(' {0}: {{'.format(ARCH_NAME_MAP[a])) |
| print(' whole_static_libs: {0},'.format(FormatList(sorted(audio_proc_libs[a])))) |
| print(' },') |
| print(' },') |
| print('}') |
