usr/bin/3rd-party-merge

#!/usr/bin/python
#
# Copyright (C) 2015 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Merge packages into the target image."""

from __future__ import print_function

import argparse
import errno
import os
import re
import shutil
import sys


def makedirs(path):
    """Create all the dirs for |path| ignoring already existing errors."""
    try:
        os.makedirs(path)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise


def unlink(path):
    """Delete |path| ignoring missing paths."""
    try:
        os.unlink(path)
    except OSError as e:
        if e.errno != errno.ENOENT:
            raise


class Atom(object):
    """Hold details about a package atom (optionally in a vdb)."""

    _PREFIX_RE = re.compile(r'^([>=<~!]*)(.*)$')
    _PN_RE = r'(?P<package>[A-Za-z0-9_]+[A-Za-z0-9+_-]*?)'
    _PV_RE = (r'(?P<version>[0-9](\.[0-9]+)*(_(pre|p|beta|alpha|rc)[0-9]*)*)'
              r'(-r(?P<revision>[0-9]+))?')
    _PN_PV_RE = re.compile(r'^%s(-%s)?$' % (_PN_RE, _PV_RE))

    def __init__(self, atom, path=None):
        self.atom = atom
        self.path = path

        # Extract any version matching prefixes.
        m = self._PREFIX_RE.match(atom)
        self.prefix, atom = m.groups()

        # Split off trailing slot -- must be before category for subslots.
        if ':' in atom:
            atom, self._slot = atom.split(':', 1)
        else:
            self._slot = None

        # Split off leading category.
        if '/' in atom:
            self.category, atom = atom.split('/', 1)
        else:
            self.category = None

        # Split apart the name & version.
        m = self._PN_PV_RE.match(atom)
        self.pn = m.group('package')
        self.pv = m.group('version')
        self.rev = m.group('revision')

    def _load_vdb_entry(self, entry):
        if not self.path:
            return None
        path = os.path.join(self.path, entry)
        with open(path) as f:
            return f.read().strip()

    @property
    def slot(self):
        if self._slot is None and self.path:
            self._slot = self._load_vdb_entry('SLOT')
        return self._slot

    def load_contents(self, path_filter=None):
        """Read the CONTENTS file for |pkg| from |root|."""
        if not path_filter:
            path_filter = lambda x: True

        ret = []

        for line in self._load_vdb_entry('CONTENTS').splitlines():
            line = line.strip()
            if not line:
                continue
            typ, data = line.split(' ', 1)
            if typ == 'obj':
                path, _hash, _mtime = data.rsplit(' ', 2)
                if path_filter(path):
                    ret.append(('obj', path))
            elif typ == 'sym':
                source, target = data.split(' -> ', 1)
                target, _mtime = target.rsplit(' ', 1)
                if path_filter(source):
                    ret.append(('sym', source, target))
            elif typ == 'dir':
                pass
            else:
                raise Exception('Unhandled entry: %s' % line)

        return ret

    def match(self, atom):
        """See if |atom| matches us."""
        # TODO: Support the range matching like >=.
        if self.prefix not in (None, '', '=', '~'):
            raise ValueError('Only exact matches supported (not %s)'
                             % (self.prefix))

        # Package names must always exist & match.
        if atom.pn != self.pn:
            return False

        # Only require the category to match if both atoms specify it.
        if len(set((atom.category, self.category, None))) == 3:
            return False

        # Only require the slot to match if both atoms specify it.
        if self.slot and len(set((atom.slot, self.slot, None))) == 3:
            return False

        # Only require the version to match if both atoms specify it.
        if len(set((atom.pv, self.pv, None))) == 3:
            return False

        # Only require the revision to match if both atoms specify it.
        if (self.prefix is not '~' and
            len(set((atom.rev, self.rev, None)))) == 3:
            return False

        # If we're still here, there's nothing left to say :).
        return True

    def __str__(self):
        return ('Atom(%s/%s-%s-r%s:%s)' %
                (self.category, self.pn, self.pv, self.rev, self.slot))


class Vdb(object):
    """Hold details about a vdb (a database of installed packages)."""

    # https://projects.gentoo.org/pms/6/pms.html#x1-180003.1.1
    CATEGORY_RE = re.compile(r'^[A-Za-z0-9_]+[A-Za-z0-9+_.-]*$')

    # https://projects.gentoo.org/pms/6/pms.html#x1-200003.1.2
    PACKAGE_RE = re.compile(r'^[A-Za-z0-9_]+[A-Za-z0-9+_-]*-[0-9]+')

    def __init__(self, root):
        self.pkgs = []

        path = os.path.join(root, 'var', 'db', 'pkg')
        for cat in os.listdir(path):
            catdir = os.path.join(path, cat)
            if not os.path.isdir(catdir) or not self.CATEGORY_RE.match(cat):
                continue

            for pkg in os.listdir(catdir):
                pkgdir = os.path.join(catdir, pkg)
                if (os.path.isdir(pkgdir) and
                    os.path.isfile(os.path.join(pkgdir, 'CONTENTS')) and
                    self.PACKAGE_RE.match(pkg)):
                    self.pkgs.append(Atom('%s/%s' % (cat, pkg), pkgdir))

    def find_pkg(self, match_pkg):
        """Try to find the best match for |match_pkg|."""
        match_atom = Atom(match_pkg)
        for vdb_pkg in self.pkgs:
            if match_atom.match(vdb_pkg):
                return vdb_pkg


def ignore_files(path):
    """Figure out which files we actually care about."""
    if path.startswith('/usr/share/doc'):
        return False
    elif path.startswith('/usr/share/gtk-doc'):
        return False
    elif path.startswith('/usr/share/info'):
        return False
    elif path.startswith('/usr/share/man'):
        return False
    elif path.startswith('/usr/include'):
        return False
    elif path.startswith('/usr/bin') and path.endswith('-config'):
        return False
    elif path.startswith('/usr/lib'):
        if path.endswith('.a'):
            return False
        elif path.endswith('.pc'):
            return False

    return True


def merge(vdb, pkg, input_root, output_root, make_root, verbose=0):
    """Merge |pkg| from |input_root| to |output_root|."""
    print('Merging %s ' % pkg, end='')
    if verbose:
        print('from %s to %s ' % (input_root, output_root))

    # TODO: Grab a lock for |pkg|.

    # First get the file listing to merge.
    vdb_pkg = vdb.find_pkg(pkg)
    if not vdb_pkg:
        raise Exception('No packages found matching %s ' % pkg)
    print('matched %s ' % vdb_pkg, end='')
    contents = vdb_pkg.load_contents(path_filter=ignore_files)

    # Now actually merge them.
    for entry in contents:
        if entry[0] == 'obj':
            path = entry[1].lstrip('/')
            output = os.path.join(output_root, path)
            if verbose:
                print('>>> %s' % path)
            else:
                print('.', end='')
            makedirs(os.path.dirname(output))
            if os.path.exists(output):
                os.chmod(output, 0o666)
            shutil.copy2(os.path.join(input_root, path), output)
        elif entry[0] == 'sym':
            path = entry[1].lstrip('/')
            target = entry[2]
            output = os.path.join(output_root, path)
            if verbose:
                print('>>> %s -> %s' % (path, target))
            else:
                print('.', end='')
            unlink(output)
            makedirs(os.path.dirname(output))
            os.symlink(target, output)
        else:
            raise Exception('Unhandled entry: %r' % entry)

    make_target = os.path.join(make_root, pkg + '.emerge')
    makedirs(os.path.dirname(make_target))
    open(make_target, 'w')

    if not verbose:
        print('')


def get_parser():
    """Return a command line parser."""
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('-p', '--package', action='append', required=True,
                        dest='packages',
                        help='Merge these package(s)')
    parser.add_argument('--input-root', type=str, required=True,
                        help='Where to read the packages')
    parser.add_argument('--output-root', type=str, required=True,
                        help='Where to write the packages')
    parser.add_argument('--make-root', type=str, required=True,
                        help='Where make writes state files for rules')
    parser.add_argument('-v', '--verbose', action='count',
                        help='Make output more verbose')
    return parser


def main(argv):
    parser = get_parser()
    opts = parser.parse_args(argv)

    vdb = Vdb(opts.input_root)
    for pkg in opts.packages:
        merge(vdb, pkg, opts.input_root, opts.output_root, opts.make_root,
              verbose=opts.verbose)


if __name__ == '__main__':
    main(sys.argv[1:])