tools/find_static_candidates.py - platform/build - Git at Google

 #!/usr/bin/env python3

 """Tool to find static libraries that maybe should be shared libraries and shared libraries that maybe should be static libraries.

 This tool only looks at the module-info.json for the current target.

 Example of "class" types for each of the modules in module-info.json
   "EXECUTABLES": 2307,
   "ETC": 9094,
   "NATIVE_TESTS": 10461,
   "APPS": 2885,
   "JAVA_LIBRARIES": 5205,
   "EXECUTABLES/JAVA_LIBRARIES": 119,
   "FAKE": 553,
   "SHARED_LIBRARIES/STATIC_LIBRARIES": 7591,
   "STATIC_LIBRARIES": 11535,
   "SHARED_LIBRARIES": 10852,
   "HEADER_LIBRARIES": 1897,
   "DYLIB_LIBRARIES": 1262,
   "RLIB_LIBRARIES": 3413,
   "ROBOLECTRIC": 39,
   "PACKAGING": 5,
   "PROC_MACRO_LIBRARIES": 36,
   "RENDERSCRIPT_BITCODE": 17,
   "DYLIB_LIBRARIES/RLIB_LIBRARIES": 8,
   "ETC/FAKE": 1

 None of the "SHARED_LIBRARIES/STATIC_LIBRARIES" are double counted in the
 modules with one class
 RLIB/

 All of these classes have shared_libs and/or static_libs
     "EXECUTABLES",
     "SHARED_LIBRARIES",
     "STATIC_LIBRARIES",
     "SHARED_LIBRARIES/STATIC_LIBRARIES", # cc_library
     "HEADER_LIBRARIES",
     "NATIVE_TESTS", # test modules
     "DYLIB_LIBRARIES", # rust
     "RLIB_LIBRARIES", # rust
     "ETC", # rust_bindgen
 """

 from collections import defaultdict

 import json, os, argparse

 ANDROID_PRODUCT_OUT = os.environ.get("ANDROID_PRODUCT_OUT")
 # If a shared library is used less than MAX_SHARED_INCLUSIONS times in a target,
 # then it will likely save memory by changing it to a static library
 # This move will also use less storage
 MAX_SHARED_INCLUSIONS = 2
 # If a static library is used more than MAX_STATIC_INCLUSIONS times in a target,
 # then it will likely save memory by changing it to a shared library
 # This move will also likely use less storage
 MIN_STATIC_INCLUSIONS = 3


 def parse_args():
   parser = argparse.ArgumentParser(
       description=(
           "Parse module-info.jso and display information about static and"
           " shared library dependencies."
       )
   )
   parser.add_argument(
       "--module", dest="module", help="Print the info for the module."
   )
   parser.add_argument(
       "--shared",
       dest="print_shared",
       action=argparse.BooleanOptionalAction,
       help=(
           "Print the list of libraries that are shared_libs for fewer than {}"
           " modules.".format(MAX_SHARED_INCLUSIONS)
       ),
   )
   parser.add_argument(
       "--static",
       dest="print_static",
       action=argparse.BooleanOptionalAction,
       help=(
           "Print the list of libraries that are static_libs for more than {}"
           " modules.".format(MIN_STATIC_INCLUSIONS)
       ),
   )
   parser.add_argument(
       "--recursive",
       dest="recursive",
       action=argparse.BooleanOptionalAction,
       default=True,
       help=(
           "Gather all dependencies of EXECUTABLES recursvily before calculating"
           " the stats. This eliminates duplicates from multiple libraries"
           " including the same dependencies in a single binary."
       ),
   )
   parser.add_argument(
       "--both",
       dest="both",
       action=argparse.BooleanOptionalAction,
       default=False,
       help=(
           "Print a list of libraries that are including libraries as both"
           " static and shared"
       ),
   )
   return parser.parse_args()


 class TransitiveHelper:

   def __init__(self):
     # keep a list of already expanded libraries so we don't end up in a cycle
     self.visited = defaultdict(lambda: defaultdict(set))

   # module is an object from the module-info dictionary
   # module_info is the dictionary from module-info.json
   # modify the module's shared_libs and static_libs with all of the transient
   # dependencies required from all of the explicit dependencies
   def flattenDeps(self, module, module_info):
     libs_snapshot = dict(shared_libs = set(module.get("shared_libs",{})), static_libs = set(module.get("static_libs",{})))

     for lib_class in ["shared_libs", "static_libs"]:
       for lib in libs_snapshot[lib_class]:
         if not lib or lib not in module_info or lib_class not in module:
           continue
         if lib in self.visited:
           module[lib_class].update(self.visited[lib][lib_class])
         else:
           res = self.flattenDeps(module_info[lib], module_info)
           module[lib_class].update(res.get(lib_class, {}))
           self.visited[lib][lib_class].update(res.get(lib_class, {}))

     return module

 def main():
   module_info = json.load(open(ANDROID_PRODUCT_OUT + "/module-info.json"))

   args = parse_args()

   if args.module:
     if args.module not in module_info:
       print("Module {} does not exist".format(args.module))
       exit(1)

   # turn all of the static_libs and shared_libs lists into sets to make them
   # easier to update
   for _, module in module_info.items():
     module["shared_libs"] = set(module.get("shared_libs", {}))
     module["static_libs"] = set(module.get("static_libs", {}))

   includedStatically = defaultdict(set)
   includedSharedly = defaultdict(set)
   includedBothly = defaultdict(set)
   transitive = TransitiveHelper()
   for name, module in module_info.items():
     if args.recursive:
       # in this recursive mode we only want to see what is included by the executables
       if "EXECUTABLES" not in module["class"]:
         continue
       module = transitive.flattenDeps(module, module_info)
       # filter out fuzzers by their dependency on clang
       if "static_libs" in module:
         if "libclang_rt.fuzzer" in module["static_libs"]:
           continue
     else:
       if "NATIVE_TESTS" in module["class"]:
         # We don't care about how tests are including libraries
         continue

     # count all of the shared and static libs included in this module
     if "shared_libs" in module:
       for lib in module["shared_libs"]:
         includedSharedly[lib].add(name)
     if "static_libs" in module:
       for lib in module["static_libs"]:
         includedStatically[lib].add(name)

     if "shared_libs" in module and  "static_libs" in module:
       intersection = set(module["shared_libs"]).intersection(
           module["static_libs"]
       )
       if intersection:
         includedBothly[name] = intersection

   if args.print_shared:
     print(
         "Shared libraries that are included by fewer than {} modules on a"
         " device:".format(MAX_SHARED_INCLUSIONS)
     )
     for name, libs in includedSharedly.items():
       if len(libs) < MAX_SHARED_INCLUSIONS:
         print("{}: {} included by: {}".format(name, len(libs), libs))

   if args.print_static:
     print(
         "Libraries that are included statically by more than {} modules on a"
         " device:".format(MIN_STATIC_INCLUSIONS)
     )
     for name, libs in includedStatically.items():
       if len(libs) > MIN_STATIC_INCLUSIONS:
         print("{}: {} included by: {}".format(name, len(libs), libs))

   if args.both:
     allIncludedBothly = set()
     for name, libs in includedBothly.items():
       allIncludedBothly.update(libs)

     print(
         "List of libraries used both statically and shared in the same"
         " processes:\n {}\n\n".format("\n".join(sorted(allIncludedBothly)))
     )
     print(
         "List of libraries used both statically and shared in any processes:\n {}".format("\n".join(sorted(includedStatically.keys() & includedSharedly.keys()))))

   if args.module:
     print(json.dumps(module_info[args.module], default=list, indent=2))
     print(
         "{} is included in shared_libs {} times by these modules: {}".format(
             args.module, len(includedSharedly[args.module]),
             includedSharedly[args.module]
         )
     )
     print(
         "{} is included in static_libs {} times by these modules: {}".format(
             args.module, len(includedStatically[args.module]),
             includedStatically[args.module]
         )
     )
     print("Shared libs included by this module that are used in fewer than {} processes:\n{}".format(
         MAX_SHARED_INCLUSIONS, [x for x in module_info[args.module]["shared_libs"] if len(includedSharedly[x]) < MAX_SHARED_INCLUSIONS]))


 if __name__ == "__main__":
   main()
	#!/usr/bin/env python3

	"""Tool to find static libraries that maybe should be shared libraries and shared libraries that maybe should be static libraries.

	This tool only looks at the module-info.json for the current target.

	Example of "class" types for each of the modules in module-info.json
	"EXECUTABLES": 2307,
	"ETC": 9094,
	"NATIVE_TESTS": 10461,
	"APPS": 2885,
	"JAVA_LIBRARIES": 5205,
	"EXECUTABLES/JAVA_LIBRARIES": 119,
	"FAKE": 553,
	"SHARED_LIBRARIES/STATIC_LIBRARIES": 7591,
	"STATIC_LIBRARIES": 11535,
	"SHARED_LIBRARIES": 10852,
	"HEADER_LIBRARIES": 1897,
	"DYLIB_LIBRARIES": 1262,
	"RLIB_LIBRARIES": 3413,
	"ROBOLECTRIC": 39,
	"PACKAGING": 5,
	"PROC_MACRO_LIBRARIES": 36,
	"RENDERSCRIPT_BITCODE": 17,
	"DYLIB_LIBRARIES/RLIB_LIBRARIES": 8,
	"ETC/FAKE": 1

	None of the "SHARED_LIBRARIES/STATIC_LIBRARIES" are double counted in the
	modules with one class
	RLIB/

	All of these classes have shared_libs and/or static_libs
	"EXECUTABLES",
	"SHARED_LIBRARIES",
	"STATIC_LIBRARIES",
	"SHARED_LIBRARIES/STATIC_LIBRARIES", # cc_library
	"HEADER_LIBRARIES",
	"NATIVE_TESTS", # test modules
	"DYLIB_LIBRARIES", # rust
	"RLIB_LIBRARIES", # rust
	"ETC", # rust_bindgen
	"""

	from collections import defaultdict

	import json, os, argparse

	ANDROID_PRODUCT_OUT = os.environ.get("ANDROID_PRODUCT_OUT")
	# If a shared library is used less than MAX_SHARED_INCLUSIONS times in a target,
	# then it will likely save memory by changing it to a static library
	# This move will also use less storage
	MAX_SHARED_INCLUSIONS = 2
	# If a static library is used more than MAX_STATIC_INCLUSIONS times in a target,
	# then it will likely save memory by changing it to a shared library
	# This move will also likely use less storage
	MIN_STATIC_INCLUSIONS = 3


	def parse_args():
	parser = argparse.ArgumentParser(
	description=(
	"Parse module-info.jso and display information about static and"
	" shared library dependencies."
	)
	)
	parser.add_argument(
	"--module", dest="module", help="Print the info for the module."
	)
	parser.add_argument(
	"--shared",
	dest="print_shared",
	action=argparse.BooleanOptionalAction,
	help=(
	"Print the list of libraries that are shared_libs for fewer than {}"
	" modules.".format(MAX_SHARED_INCLUSIONS)
	),
	)
	parser.add_argument(
	"--static",
	dest="print_static",
	action=argparse.BooleanOptionalAction,
	help=(
	"Print the list of libraries that are static_libs for more than {}"
	" modules.".format(MIN_STATIC_INCLUSIONS)
	),
	)
	parser.add_argument(
	"--recursive",
	dest="recursive",
	action=argparse.BooleanOptionalAction,
	default=True,
	help=(
	"Gather all dependencies of EXECUTABLES recursvily before calculating"
	" the stats. This eliminates duplicates from multiple libraries"
	" including the same dependencies in a single binary."
	),
	)
	parser.add_argument(
	"--both",
	dest="both",
	action=argparse.BooleanOptionalAction,
	default=False,
	help=(
	"Print a list of libraries that are including libraries as both"
	" static and shared"
	),
	)
	return parser.parse_args()


	class TransitiveHelper:

	def __init__(self):
	# keep a list of already expanded libraries so we don't end up in a cycle
	self.visited = defaultdict(lambda: defaultdict(set))

	# module is an object from the module-info dictionary
	# module_info is the dictionary from module-info.json
	# modify the module's shared_libs and static_libs with all of the transient
	# dependencies required from all of the explicit dependencies
	def flattenDeps(self, module, module_info):
	libs_snapshot = dict(shared_libs = set(module.get("shared_libs",{})), static_libs = set(module.get("static_libs",{})))

	for lib_class in ["shared_libs", "static_libs"]:
	for lib in libs_snapshot[lib_class]:
	if not lib or lib not in module_info or lib_class not in module:
	continue
	if lib in self.visited:
	module[lib_class].update(self.visited[lib][lib_class])
	else:
	res = self.flattenDeps(module_info[lib], module_info)
	module[lib_class].update(res.get(lib_class, {}))
	self.visited[lib][lib_class].update(res.get(lib_class, {}))

	return module

	def main():
	module_info = json.load(open(ANDROID_PRODUCT_OUT + "/module-info.json"))

	args = parse_args()

	if args.module:
	if args.module not in module_info:
	print("Module {} does not exist".format(args.module))
	exit(1)

	# turn all of the static_libs and shared_libs lists into sets to make them
	# easier to update
	for _, module in module_info.items():
	module["shared_libs"] = set(module.get("shared_libs", {}))
	module["static_libs"] = set(module.get("static_libs", {}))

	includedStatically = defaultdict(set)
	includedSharedly = defaultdict(set)
	includedBothly = defaultdict(set)
	transitive = TransitiveHelper()
	for name, module in module_info.items():
	if args.recursive:
	# in this recursive mode we only want to see what is included by the executables
	if "EXECUTABLES" not in module["class"]:
	continue
	module = transitive.flattenDeps(module, module_info)
	# filter out fuzzers by their dependency on clang
	if "static_libs" in module:
	if "libclang_rt.fuzzer" in module["static_libs"]:
	continue
	else:
	if "NATIVE_TESTS" in module["class"]:
	# We don't care about how tests are including libraries
	continue

	# count all of the shared and static libs included in this module
	if "shared_libs" in module:
	for lib in module["shared_libs"]:
	includedSharedly[lib].add(name)
	if "static_libs" in module:
	for lib in module["static_libs"]:
	includedStatically[lib].add(name)

	if "shared_libs" in module and "static_libs" in module:
	intersection = set(module["shared_libs"]).intersection(
	module["static_libs"]
	)
	if intersection:
	includedBothly[name] = intersection

	if args.print_shared:
	print(
	"Shared libraries that are included by fewer than {} modules on a"
	" device:".format(MAX_SHARED_INCLUSIONS)
	)
	for name, libs in includedSharedly.items():
	if len(libs) < MAX_SHARED_INCLUSIONS:
	print("{}: {} included by: {}".format(name, len(libs), libs))

	if args.print_static:
	print(
	"Libraries that are included statically by more than {} modules on a"
	" device:".format(MIN_STATIC_INCLUSIONS)
	)
	for name, libs in includedStatically.items():
	if len(libs) > MIN_STATIC_INCLUSIONS:
	print("{}: {} included by: {}".format(name, len(libs), libs))

	if args.both:
	allIncludedBothly = set()
	for name, libs in includedBothly.items():
	allIncludedBothly.update(libs)

	print(
	"List of libraries used both statically and shared in the same"
	" processes:\n {}\n\n".format("\n".join(sorted(allIncludedBothly)))
	)
	print(
	"List of libraries used both statically and shared in any processes:\n {}".format("\n".join(sorted(includedStatically.keys() & includedSharedly.keys()))))

	if args.module:
	print(json.dumps(module_info[args.module], default=list, indent=2))
	print(
	"{} is included in shared_libs {} times by these modules: {}".format(
	args.module, len(includedSharedly[args.module]),
	includedSharedly[args.module]
	)
	)
	print(
	"{} is included in static_libs {} times by these modules: {}".format(
	args.module, len(includedStatically[args.module]),
	includedStatically[args.module]
	)
	)
	print("Shared libs included by this module that are used in fewer than {} processes:\n{}".format(
	MAX_SHARED_INCLUSIONS, [x for x in module_info[args.module]["shared_libs"] if len(includedSharedly[x]) < MAX_SHARED_INCLUSIONS]))



	if __name__ == "__main__":
	main()