Tools/fuchsia/extract_source_lists.py

#!/usr/bin/env python
#
# Copyright 2017 The Fuchsia Authors. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1.  Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
# 2.  Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


import sys
import os
import shutil
import subprocess

WEBKIT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
BUILD_DIR = os.path.join(WEBKIT_ROOT, 'WebKitBuild', 'Debug')

known_targets = [
  ('Source/JavaScriptCore', 'JavaScriptCore'),
  ('Source/WTF/wtf', 'wtf'),
  ('Source/WebCore', 'WebCore'),
  ('Source/WebKit', 'WebKit'),
  ('Tools/MiniBrowser', 'MiniBrowser'),
]

# Some files live inside the library directories but are intended for building into standalone
# executables, so we exclude them from the library source listings.
executable_source_files = [
    'LLIntOffsetsExtractor.cpp',
    'testWASM.cpp',
    'testair.cpp',
    'testb3.cpp',
    'jsc.cpp',
]
# These files reference undefined symbols and themselves define symbols that aren't used anywhere,
# so we exclude them from our build files.
bad_source_files = executable_source_files + [
    'DNSResolveQueue.cpp',
    'SynchronousLoaderClient.cpp',
]
source_lists = {}

def append_to_source_list(target, filename):
    if source_lists.has_key(target):
        source_lists[target].append(filename)
    else:
        source_lists[target] = [filename]

def process_compile(line):
    components = line.rstrip().split(' ')
    source = os.path.relpath(components[-1])
    if os.path.basename(source) in bad_source_files:
        # Skip files that are their own mains
        return
    if 'DerivedSources' in source:
        derived_path = os.path.relpath(source, os.path.join(BUILD_DIR, 'DerivedSources'))
        component = os.path.dirname(derived_path)
        filename = os.path.basename(derived_path)
        dest = os.path.join(WEBKIT_ROOT, 'DerivedSources', component, filename)
        target = component.replace('/', '_') + 'DerivedSources'
        append_to_source_list(target, os.path.relpath(dest, WEBKIT_ROOT))
        # TODO: handle generated files
        # print 'generated file %s' % source
        return
    known = False
    for target in known_targets:
        if source.startswith(target[0]):
            append_to_source_list(target[1], source)
            known = True
            break
    if not known:
        print('unknown target %s, line %s' % (source, line))

def buildlog():
    ninja = os.path.join(os.path.dirname(os.path.dirname(WEBKIT_ROOT)), 'buildtools', 'ninja')
    # Hack: Nuke the ninja log to force full rebuilds
    dot_ninja_log = os.path.join(BUILD_DIR, '.ninja_log')
    if os.path.exists(dot_ninja_log):
        os.remove(dot_ninja_log)
    return subprocess.check_output([ninja, '-v', '-n'], cwd=BUILD_DIR)


def main():
    # Grab the list of files to compile from the build log
    print('Processing file list')
    log = buildlog()
    for line in log:
        # Figure out what kind of statement this is
        if 'clang++ --sysroot' in line or 'clang --sysroot' in line:
            if not '-o bin/' in line:
                process_compile(line)
    for entry in source_lists:
        with open(os.path.join(WEBKIT_ROOT, entry + '.gni'), 'w') as gni:
            gni.write('''
%s_sources = [
''' % entry)
            for source in sorted(source_lists[entry]):
                gni.write('  "%s",\n' % source)
            gni.write('''
]
''')
    derived_source_dir = os.path.join(BUILD_DIR, 'DerivedSources')
    derived_source_header_guard_substring = derived_source_dir.replace('/', '_').upper()
    print('Copying derived sources')
     # Copy all source and header files from DerivedSources in the build directory to our location
     # in the source directory.
    for entry in os.walk(derived_source_dir):
        dirpath, dirnames, filenames = entry
        for filename in filenames:
            ext = os.path.splitext(filename)[1]
            if not ext in ['.h', '.cpp', '.c']:
                continue
            # LLIntAssembly.h is treated specially since it differs for release and debug builds -
            # see below.
            if filename == 'LLIntAssembly.h':
                continue
            source = os.path.join(dirpath, filename)
            dest = os.path.join(WEBKIT_ROOT, os.path.relpath(source, BUILD_DIR))
            if not os.path.exists(os.path.dirname(dest)):
                os.makedirs(os.path.dirname(dest))
            with open(source) as source_file:
                contents = source_file.read()
                # Turn all paths to derived sources into source-relative paths.
                replaced_contents = contents.replace(BUILD_DIR + '/DerivedSources', 'DerivedSources')
                # Turn all absolute paths in the source dir into root-relative paths.
                replaced_contents = replaced_contents.replace(WEBKIT_ROOT + '/', '')
                # Turn generated header guards referencing full paths into relative.
                replaced_contents = replaced_contents.replace(derived_source_header_guard_substring,
                        '_DERIVED_SOURCES')
                with open(dest, 'w') as dest_file:
                    dest_file.write(replaced_contents)

    # LLIntAssembly.h is different in debug vs release builds, unlike other generated files.
    # To keep the debug and release versions separate, copy the two versions into different
    # directories. BUILD.gn adds the appropriate directory to the include path.
    for mode in ['Debug', 'Release']:
        dest_dir = os.path.join(WEBKIT_ROOT, 'DerivedSources', 'JavaScriptCore', mode)
        if not os.path.exists(dest_dir):
            os.makedirs(dest_dir)
        source = os.path.join(WEBKIT_ROOT, 'WebKitBuild', mode, 'DerivedSources',
        'JavaScriptCore', 'LLIntAssembly.h')
        shutil.copy(source, dest_dir)

    print('Copying cmakeconfig.h')
    # Copy cmakeconfig.h somewhere more useful
    config_path = os.path.join(WEBKIT_ROOT, 'Source', 'config')
    if not os.path.exists(config_path):
        os.makedirs(config_path)
    shutil.copy(os.path.join(BUILD_DIR, 'cmakeconfig.h'), config_path)
    print('Done!')


if __name__ == '__main__':
    sys.exit(main())