blob: 406c807727ab1bd337eb863db7c373dfa091d49f [file] [log] [blame]
# Copyright 2014 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Script to remove unused gconv charset modules from a build."""
from __future__ import print_function
import ahocorasick
import glob
import lddtree
import operator
import os
import stat
from chromite.lib import commandline
from chromite.lib import cros_build_lib
from chromite.lib import cros_logging as logging
from chromite.lib import osutils
# Path pattern to search for the gconv-modules file.
GCONV_MODULES_PATH = 'usr/*/gconv/gconv-modules'
# Sticky modules. These charsets modules are always included even if they
# aren't used. You can specify any charset name as supported by 'iconv_open',
# for example, 'LATIN1' or 'ISO-8859-1'.
# List of function names (symbols) known to use a charset as a parameter.
# glibc
# glib
class GconvModules(object):
"""Class to manipulate the gconv/gconv-modules file and referenced modules.
This class parses the contents of the gconv-modules file installed by glibc
which provides the definition of the charsets supported by iconv_open(3). It
allows to load the current gconv-modules file and rewrite it to include only
a subset of the supported modules, removing the other modules.
Each charset is involved on some transformation between that charset and an
internal representation. This transformation is defined on a .so file loaded
dynamically with dlopen(3) when the charset defined in this file is requested
to iconv_open(3).
See the comments on gconv-modules file for syntax details.
def __init__(self, gconv_modules_file):
"""Initialize the class.
gconv_modules_file: Path to gconv/gconv-modules file.
self._filename = gconv_modules_file
# An alias map of charsets. The key (fromcharset) is the alias name and
# the value (tocharset) is the real charset name. We also support a value
# that is an alias for another charset.
self._alias = {}
# The modules dict goes from charset to module names (the filenames without
# the .so extension). Since several transformations involving the same
# charset could be defined in different files, the values of this dict are
# a set of module names.
self._modules = {}
def Load(self):
"""Load the charsets from gconv-modules."""
for line in open(self._filename):
line = line.split('#', 1)[0].strip()
if not line: # Comment
lst = line.split()
if lst[0] == 'module':
_, fromset, toset, filename = lst[:4]
for charset in (fromset, toset):
charset = charset.rstrip('/')
mods = self._modules.get(charset, set())
self._modules[charset] = mods
elif lst[0] == 'alias':
_, fromset, toset = lst
fromset = fromset.rstrip('/')
toset = toset.rstrip('/')
# Warn if the same charset is defined as two different aliases.
if self._alias.get(fromset, toset) != toset:
logging.error('charset "%s" already defined as "%s".', fromset,
self._alias[fromset] = toset
cros_build_lib.Die('Unknown line: %s', line)
logging.debug('Found %d modules and %d alias in %s', len(self._modules),
len(self._alias), self._filename)
charsets = sorted(self._alias.keys() + self._modules.keys())
# Remove the 'INTERNAL' charset from the list, since it is not a charset
# but an internal representation used to convert to and from other charsets.
if 'INTERNAL' in charsets:
return charsets
def Rewrite(self, used_charsets, dry_run=False):
"""Rewrite gconv-modules file with only the used charsets.
used_charsets: A list of used charsets. This should be a subset of the
list returned by Load().
dry_run: Whether this function should not change any file.
# Compute the used modules.
used_modules = set()
for charset in used_charsets:
while charset in self._alias:
charset = self._alias[charset]
unused_modules = reduce(set.union, self._modules.values()) - used_modules
modules_dir = os.path.dirname(self._filename)
all_modules = set.union(used_modules, unused_modules)
# The list of charsets that depend on a given library. For example,
# libdeps[''] is the set of all the modules that require that
# library. These libraries live in the same directory as the modules.
libdeps = {}
for module in all_modules:
deps = lddtree.ParseELF(os.path.join(modules_dir, '' % module),
modules_dir, [])
if not 'needed' in deps:
for lib in deps['needed']:
# Ignore the libs without a path defined (outside the modules_dir).
if deps['libs'][lib]['path']:
libdeps[lib] = libdeps.get(lib, set()).union([module])
used_libdeps = set(lib for lib, deps in libdeps.iteritems()
if deps.intersection(used_modules))
unused_libdeps = set(libdeps).difference(used_libdeps)
logging.debug('Used modules: %s', ', '.join(sorted(used_modules)))
logging.debug('Used dependency libs: %s, '.join(sorted(used_libdeps)))
unused_size = 0
for module in sorted(unused_modules):
module_path = os.path.join(modules_dir, '' % module)
unused_size += os.lstat(module_path).st_size
logging.debug('rm %s', module_path)
if not dry_run:
unused_libdeps_size = 0
for lib in sorted(unused_libdeps):
lib_path = os.path.join(modules_dir, lib)
unused_libdeps_size += os.lstat(lib_path).st_size
logging.debug('rm %s', lib_path)
if not dry_run:
os.unlink(lib_path)'Done. Using %d gconv modules. Removed %d unused modules'
' (%.1f KiB) and %d unused dependencies (%.1f KiB)',
len(used_modules), len(unused_modules), unused_size / 1024.,
len(unused_libdeps), unused_libdeps_size / 1024.)
# Recompute the gconv-modules file with only the included gconv modules.
result = []
for line in open(self._filename):
lst = line.split('#', 1)[0].strip().split()
if not lst:
result.append(line) # Keep comments and copyright headers.
elif lst[0] == 'module':
_, _, _, filename = lst[:4]
if filename in used_modules:
result.append(line) # Used module
elif lst[0] == 'alias':
_, charset, _ = lst
charset = charset.rstrip('/')
while charset in self._alias:
charset = self._alias[charset]
if used_modules.intersection(self._modules[charset]):
result.append(line) # Alias to an used module
cros_build_lib.Die('Unknown line: %s', line)
if not dry_run:
osutils.WriteFile(self._filename, ''.join(result))
def MultipleStringMatch(patterns, corpus):
"""Search a list of strings in a corpus string.
patterns: A list of strings.
corpus: The text where to search for the strings.
A list of Booleans stating whether each pattern string was found in the
corpus or not.
tree = ahocorasick.KeywordTree()
for word in patterns:
result = [False] * len(patterns)
for i, j in tree.findall(corpus):
match = corpus[i:j]
result[patterns.index(match)] = True
return result
def GconvStrip(opts):
"""Process gconv-modules and remove unused modules.
opts: The command-line args passed to the script.
The exit code number indicating whether the process succeeded.
root_st = os.lstat(opts.root)
if not stat.S_ISDIR(root_st.st_mode):
cros_build_lib.Die('root (%s) must be a directory.' % opts.root)
# Detect the possible locations of the gconv-modules file.
gconv_modules_files = glob.glob(os.path.join(opts.root, GCONV_MODULES_PATH))
if not gconv_modules_files:
logging.warning('gconv-modules file not found.')
return 1
# Only one gconv-modules files should be present, either on /usr/lib or
# /usr/lib64, but not both.
if len(gconv_modules_files) > 1:
cros_build_lib.Die('Found several gconv-modules files.')
gconv_modules_file = gconv_modules_files[0]'Searching for unused gconv files defined in %s',
gmods = GconvModules(gconv_modules_file)
charsets = gmods.Load()
# Use scanelf to search for all the binary files on the rootfs that require
# or define the symbol iconv_open. We also include the binaries that define
# it since there could be internal calls to it from other functions.
files = set()
for symbol in GCONV_SYMBOLS:
cmd = ['scanelf', '--mount', '--quiet', '--recursive', '--format', '#s%F',
'--symbol', symbol, opts.root]
result = cros_build_lib.RunCommand(cmd, redirect_stdout=True,
symbol_files = result.output.splitlines()
logging.debug('Symbol %s found on %d files.', symbol, len(symbol_files))
# The charsets are represented as nul-terminated strings in the binary files,
# so we append the '\0' to each string. This prevents some false positives
# when the name of the charset is a substring of some other string. It doesn't
# prevent false positives when the charset name is the suffix of another
# string, for example a binary with the string "DON'T DO IT\0" will match the
# 'IT' charset. Empirical test on ChromeOS images suggests that only 4
# charsets could fall in category.
strings = [s + '\0' for s in charsets]'Will search for %d strings in %d files', len(strings),
# Charsets listed in STICKY_MOUDLES are initialized as used. Note that those
# strings should be listed in the gconv-modules file.
unknown_sticky_modules = set(STICKY_MODULES) - set(charsets)
if unknown_sticky_modules:
'The following charsets were explicitly requested in STICKY_MODULES '
'even though they don\'t exist: %s',
', '.join(unknown_sticky_modules))
global_used = [charset in STICKY_MODULES for charset in charsets]
for filename in files:
used_filename = MultipleStringMatch(strings,
osutils.ReadFile(filename, mode='rb'))
global_used = map(operator.or_, global_used, used_filename)
# Check the debug flag to avoid running an useless loop.
if opts.debug and any(used_filename):
logging.debug('File %s:', filename)
for i in range(len(used_filename)):
if used_filename[i]:
logging.debug(' - %s', strings[i])
used_charsets = [cs for cs, used in zip(charsets, global_used) if used]
gmods.Rewrite(used_charsets, opts.dry_run)
return 0
def ParseArgs(argv):
"""Return parsed commandline arguments."""
parser = commandline.ArgumentParser()
'--dry-run', action='store_true', default=False,
help='process but don\'t modify any file.')
'root', type='path',
help='path to the directory where the rootfs is mounted.')
opts = parser.parse_args(argv)
return opts
def main(argv):
"""Main function to start the script."""
opts = ParseArgs(argv)
logging.debug('Options are %s', opts)
return GconvStrip(opts)