sdk/merger/merge.py

#!/usr/bin/env python
# Copyright 2018 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import argparse
import contextlib
import errno
import json
import os
import shutil
import sys
import tarfile
import tempfile


@contextlib.contextmanager
def _open_archive(archive, directory):
    '''Manages a directory in which an existing SDK is laid out.'''
    if directory:
        yield directory
    elif archive:
        temp_dir = tempfile.mkdtemp(prefix='fuchsia-merger')
        # Extract the tarball into the temporary directory.
        # This is vastly more efficient than accessing files one by one via
        # the tarfile API.
        with tarfile.open(archive) as archive_file:
            archive_file.extractall(temp_dir)
        try:
            yield temp_dir
        finally:
            shutil.rmtree(temp_dir, ignore_errors=True)
    else:
        raise Exception('Error: archive or directory must be set')


@contextlib.contextmanager
def _open_output(archive, directory):
    '''Manages the output of this script.'''
    if directory:
        # Remove any existing output.
        shutil.rmtree(directory, ignore_errors=True)
        yield directory
    elif archive:
        temp_dir = tempfile.mkdtemp(prefix='fuchsia-merger')
        try:
            yield temp_dir
            # Write the archive file.
            with tarfile.open(archive, "w:gz") as archive_file:
                archive_file.add(temp_dir, arcname='')
        finally:
            shutil.rmtree(temp_dir, ignore_errors=True)
    else:
        raise Exception('Error: archive or directory must be set')


def _get_manifest(sdk_dir):
    '''Returns the set of elements in the given SDK.'''
    with open(os.path.join(sdk_dir, 'meta', 'manifest.json'), 'r') as manifest:
        return json.load(manifest)


def _get_meta(element, sdk_dir):
    '''Returns the contents of the given element's manifest in a given SDK.'''
    with open(os.path.join(sdk_dir, element), 'r') as meta:
        return json.load(meta)


def _get_files(element_meta):
    '''Extracts the files associated with the given element.
    Returns a 2-tuple containing:
     - the set of arch-independent files;
     - the sets of arch-dependent files, indexed by architecture.
    '''
    type = element_meta['type']
    common_files = set()
    arch_files = {}
    if type == 'cc_prebuilt_library':
        common_files.update(element_meta['headers'])
        for arch, binaries in element_meta['binaries'].iteritems():
            contents = set()
            contents.add(binaries['link'])
            if 'dist' in binaries:
                contents.add(binaries['dist'])
            if 'debug' in binaries:
                contents.add(binaries['debug'])
            arch_files[arch] = contents
    elif type == 'cc_source_library':
        common_files.update(element_meta['headers'])
        common_files.update(element_meta['sources'])
    elif type == 'dart_library':
        common_files.update(element_meta['sources'])
    elif type == 'fidl_library':
        common_files.update(element_meta['sources'])
    elif type == 'host_tool':
        common_files.update(element_meta['files'])
    elif type == 'image':
        for arch, file in element_meta['file'].iteritems():
            arch_files[arch] = set([file])
    elif type == 'loadable_module':
        common_files.update(element_meta['resources'])
        arch_files.update(element_meta['binaries'])
    elif type == 'sysroot':
        for arch, version in element_meta['versions'].iteritems():
            contents = set()
            contents.update(version['headers'])
            contents.update(version['link_libs'])
            contents.update(version['dist_libs'])
            contents.update(version['debug_libs'])
            arch_files[arch] = contents
    elif type == 'documentation':
        common_files.update(element_meta['docs'])
    else:
        raise Exception('Unknown element type: ' + type)
    return (common_files, arch_files)


def _ensure_directory(path):
    '''Ensures that the directory hierarchy of the given path exists.'''
    target_dir = os.path.dirname(path)
    try:
        os.makedirs(target_dir)
    except OSError as exception:
        if exception.errno == errno.EEXIST and os.path.isdir(target_dir):
            pass
        else:
            raise


def _copy_file(file, source_dir, dest_dir):
    '''Copies a file to a given path, taking care of creating directories if
    needed.
    '''
    source = os.path.join(source_dir, file)
    destination = os.path.join(dest_dir, file)
    _ensure_directory(destination)
    shutil.copy2(source, destination)


def _copy_files(files, source_dir, dest_dir):
    '''Copies a set of files to a given directory.'''
    for file in files:
        _copy_file(file, source_dir, dest_dir)


def _copy_identical_files(set_one, source_dir_one, set_two, source_dir_two,
                          dest_dir):
    '''Verifies that two sets of files are absolutely identical and then copies
    them to the output directory.
    '''
    if set_one != set_two:
        return False
    # Not verifying that the contents of the files are the same, as builds are
    # not exactly stable at the moment.
    _copy_files(set_one, source_dir_one, dest_dir)
    return True


def _copy_element(element, source_dir, dest_dir):
    '''Copy an entire SDK element to a given directory.'''
    meta = _get_meta(element, source_dir)
    common_files, arch_files = _get_files(meta)
    files = common_files
    for more_files in arch_files.itervalues():
        files.update(more_files)
    _copy_files(files, source_dir, dest_dir)
    # Copy the metadata file as well.
    _copy_file(element, source_dir, dest_dir)


def _write_meta(element, source_dir_one, source_dir_two, dest_dir):
    '''Writes a meta file for the given element, resulting from the merge of the
    meta files for that element in the two given SDK directories.
    '''
    meta_one = _get_meta(element, source_dir_one)
    meta_two = _get_meta(element, source_dir_two)
    # TODO(DX-495): verify that the common parts of the metadata files are in
    # fact identical.
    type = meta_one['type']
    meta = {}
    if type == 'cc_prebuilt_library' or type == 'loadable_module':
        meta = meta_one
        meta['binaries'].update(meta_two['binaries'])
    elif type == 'image':
        meta = meta_one
        meta['file'].update(meta_two['file'])
    elif type == 'sysroot':
        meta = meta_one
        meta['versions'].update(meta_two['versions'])
    elif (type == 'cc_source_library' or type == 'dart_library' or
          type == 'fidl_library' or type == 'host_tool' or
          type == 'documentation'):
        # These elements are arch-independent, the metadata does not need any
        # update.
        meta = meta_one
    else:
        raise Exception('Unknown element type: ' + type)
    meta_path = os.path.join(dest_dir, element)
    _ensure_directory(meta_path)
    with open(meta_path, 'w') as meta_file:
        json.dump(meta, meta_file, indent=2, sort_keys=True)
    return True


def _write_manifest(source_dir_one, source_dir_two, dest_dir):
    '''Writes a manifest file resulting from the merge of the manifest files for
    the two given SDK directories.
    '''
    manifest_one = _get_manifest(source_dir_one)
    manifest_two = _get_manifest(source_dir_two)

    # Host architecture.
    if manifest_one['arch']['host'] != manifest_two['arch']['host']:
        print('Error: mismatching host architecture')
        return False
    manifest = {
        'arch': {
            'host': manifest_one['arch']['host'],
        }
    }

    # Target architectures.
    manifest['arch']['target'] = sorted(set(manifest_one['arch']['target']) |
                                        set(manifest_two['arch']['target']))

    # Parts.
    manifest['parts'] = sorted(set(manifest_one['parts']) |
                               set(manifest_two['parts']))

    manifest_path = os.path.join(dest_dir, 'meta', 'manifest.json')
    _ensure_directory(manifest_path)
    with open(manifest_path, 'w') as manifest_file:
        json.dump(manifest, manifest_file, indent=2, sort_keys=True)
    return True


def main():
    parser = argparse.ArgumentParser(
            description=('Merges the contents of two SDKs'))
    alpha_group = parser.add_mutually_exclusive_group(required=True)
    alpha_group.add_argument('--alpha-archive',
                             help='Path to the first SDK - as an archive',
                             default='')
    alpha_group.add_argument('--alpha-directory',
                             help='Path to the first SDK - as a directory',
                             default='')
    beta_group = parser.add_mutually_exclusive_group(required=True)
    beta_group.add_argument('--beta-archive',
                            help='Path to the second SDK - as an archive',
                            default='')
    beta_group.add_argument('--beta-directory',
                            help='Path to the second SDK - as a directory',
                            default='')
    output_group = parser.add_mutually_exclusive_group(required=True)
    output_group.add_argument('--output-archive',
                              help='Path to the merged SDK - as an archive',
                              default='')
    output_group.add_argument('--output-directory',
                              help='Path to the merged SDK - as a directory',
                              default='')
    args = parser.parse_args()

    has_errors = False

    with _open_archive(args.alpha_archive, args.alpha_directory) as alpha_dir, \
         _open_archive(args.beta_archive, args.beta_directory) as beta_dir, \
         _open_output(args.output_archive, args.output_directory) as out_dir:

        alpha_elements = set(_get_manifest(alpha_dir)['parts'])
        beta_elements = set(_get_manifest(beta_dir)['parts'])
        common_elements = alpha_elements & beta_elements

        # Copy elements that appear in a single SDK.
        for element in sorted(alpha_elements - common_elements):
            _copy_element(element, alpha_dir, out_dir)
        for element in (beta_elements - common_elements):
            _copy_element(element, beta_dir, out_dir)

        # Verify and merge elements which are common to both SDKs.
        for element in sorted(common_elements):
            alpha_meta = _get_meta(element, alpha_dir)
            beta_meta = _get_meta(element, beta_dir)
            alpha_common, alpha_arch = _get_files(alpha_meta)
            beta_common, beta_arch = _get_files(beta_meta)

            # Common files should not vary.
            if not _copy_identical_files(alpha_common, alpha_dir, beta_common,
                                         beta_dir, out_dir):
                print('Error: different common files for ' + element)
                has_errors = True
                continue

            # Arch-dependent files need to be merged in the metadata.
            all_arches = set(alpha_arch.keys()) | set(beta_arch.keys())
            for arch in all_arches:
                if arch in alpha_arch and arch in beta_arch:
                    if not _copy_identical_files(alpha_arch[arch], alpha_dir,
                                                 beta_arch[arch], beta_dir,
                                                 out_dir):
                        print('Error: different %s files for %s' % (arch,
                                                                   element))
                        has_errors = True
                        continue
                elif arch in alpha_arch:
                    _copy_files(alpha_arch[arch], alpha_dir, out_dir)
                elif arch in beta_arch:
                    _copy_files(beta_arch[arch], beta_dir, out_dir)

            if not _write_meta(element, alpha_dir, beta_dir, out_dir):
                print('Error: unable to merge meta for ' + element)
                has_errors = True

        if not _write_manifest(alpha_dir, beta_dir, out_dir):
            print('Error: could not write manifest file')
            has_errors = True

        # TODO(DX-495): verify that metadata files are valid.

    return 1 if has_errors else 0


if __name__ == '__main__':
    sys.exit(main())