scripts/gen_travis.py

#!/usr/bin/env python3

from itertools import combinations, chain
from enum import Enum, auto


LINUX = 'linux'
OSX = 'osx'
WINDOWS = 'windows'
FREEBSD = 'freebsd'


AMD64 = 'amd64'
ARM64 = 'arm64'
PPC64LE = 'ppc64le'


TRAVIS_TEMPLATE = """\
# This config file is generated by ./scripts/gen_travis.py.
# Do not edit by hand.

# We use 'minimal', because 'generic' makes Windows VMs hang at startup. Also
# the software provided by 'generic' is simply not needed for our tests.
# Differences are explained here:
# https://docs.travis-ci.com/user/languages/minimal-and-generic/
language: minimal
dist: focal

jobs:
  include:
{jobs}

before_install:
  - |-
    if test -f "./scripts/$TRAVIS_OS_NAME/before_install.sh"; then
      source ./scripts/$TRAVIS_OS_NAME/before_install.sh
    fi

before_script:
  - |-
    if test -f "./scripts/$TRAVIS_OS_NAME/before_script.sh"; then
      source ./scripts/$TRAVIS_OS_NAME/before_script.sh
    else
      scripts/gen_travis.py > travis_script && diff .travis.yml travis_script
      autoconf
      # If COMPILER_FLAGS are not empty, add them to CC and CXX
      ./configure ${{COMPILER_FLAGS:+ CC="$CC $COMPILER_FLAGS" \
CXX="$CXX $COMPILER_FLAGS"}} $CONFIGURE_FLAGS
      make -j3
      make -j3 tests
    fi

script:
  - |-
    if test -f "./scripts/$TRAVIS_OS_NAME/script.sh"; then
      source ./scripts/$TRAVIS_OS_NAME/script.sh
    else
      make check
    fi
"""


class Option(object):
    class Type:
        COMPILER = auto()
        COMPILER_FLAG = auto()
        CONFIGURE_FLAG = auto()
        MALLOC_CONF = auto()
        FEATURE = auto()

    def __init__(self, type, value):
        self.type = type
        self.value = value

    @staticmethod
    def as_compiler(value):
        return Option(Option.Type.COMPILER, value)

    @staticmethod
    def as_compiler_flag(value):
        return Option(Option.Type.COMPILER_FLAG, value)

    @staticmethod
    def as_configure_flag(value):
        return Option(Option.Type.CONFIGURE_FLAG, value)

    @staticmethod
    def as_malloc_conf(value):
        return Option(Option.Type.MALLOC_CONF, value)

    @staticmethod
    def as_feature(value):
        return Option(Option.Type.FEATURE, value)

    def __eq__(self, obj):
        return (isinstance(obj, Option) and obj.type == self.type
                and obj.value == self.value)


# The 'default' configuration is gcc, on linux, with no compiler or configure
# flags.  We also test with clang, -m32, --enable-debug, --enable-prof,
# --disable-stats, and --with-malloc-conf=tcache:false.  To avoid abusing
# travis though, we don't test all 2**7 = 128 possible combinations of these;
# instead, we only test combinations of up to 2 'unusual' settings, under the
# hope that bugs involving interactions of such settings are rare.
MAX_UNUSUAL_OPTIONS = 2


GCC = Option.as_compiler('CC=gcc CXX=g++')
CLANG = Option.as_compiler('CC=clang CXX=clang++')
CL = Option.as_compiler('CC=cl.exe CXX=cl.exe')


compilers_unusual = [CLANG,]


CROSS_COMPILE_32BIT = Option.as_feature('CROSS_COMPILE_32BIT')
feature_unusuals = [CROSS_COMPILE_32BIT]


configure_flag_unusuals = [Option.as_configure_flag(opt) for opt in (
    '--enable-debug',
    '--enable-prof',
    '--disable-stats',
    '--disable-libdl',
    '--enable-opt-safety-checks',
    '--with-lg-page=16',
)]


malloc_conf_unusuals = [Option.as_malloc_conf(opt) for opt in (
    'tcache:false',
    'dss:primary',
    'percpu_arena:percpu',
    'background_thread:true',
)]


all_unusuals = (compilers_unusual + feature_unusuals
    + configure_flag_unusuals + malloc_conf_unusuals)


def get_extra_cflags(os, compiler):
    if os == FREEBSD:
        return []

    if os == WINDOWS:
        # For non-CL compilers under Windows (for now it's only MinGW-GCC),
        # -fcommon needs to be specified to correctly handle multiple
        # 'malloc_conf' symbols and such, which are declared weak under Linux.
        # Weak symbols don't work with MinGW-GCC.
        if compiler != CL.value:
            return ['-fcommon']
        else:
            return []

    # We get some spurious errors when -Warray-bounds is enabled.
    extra_cflags = ['-Werror', '-Wno-array-bounds']
    if compiler == CLANG.value or os == OSX:
        extra_cflags += [
            '-Wno-unknown-warning-option',
            '-Wno-ignored-attributes'
        ]
    if os == OSX:
        extra_cflags += [
            '-Wno-deprecated-declarations',
        ]
    return extra_cflags


# Formats a job from a combination of flags
def format_job(os, arch, combination):
    compilers = [x.value for x in combination if x.type == Option.Type.COMPILER]
    assert(len(compilers) <= 1)
    compiler_flags = [x.value for x in combination if x.type == Option.Type.COMPILER_FLAG]
    configure_flags = [x.value for x in combination if x.type == Option.Type.CONFIGURE_FLAG]
    malloc_conf = [x.value for x in combination if x.type == Option.Type.MALLOC_CONF]
    features = [x.value for x in combination if x.type == Option.Type.FEATURE]

    if len(malloc_conf) > 0:
        configure_flags.append('--with-malloc-conf=' + ','.join(malloc_conf))

    if not compilers:
        compiler = GCC.value
    else:
        compiler = compilers[0]

    extra_environment_vars = ''
    cross_compile = CROSS_COMPILE_32BIT.value in features
    if os == LINUX and cross_compile:
        compiler_flags.append('-m32')

    features_str = ' '.join([' {}=yes'.format(feature) for feature in features])

    stringify = lambda arr, name: ' {}="{}"'.format(name, ' '.join(arr)) if arr else ''
    env_string = '{}{}{}{}{}{}'.format(
            compiler,
            features_str,
            stringify(compiler_flags, 'COMPILER_FLAGS'),
            stringify(configure_flags, 'CONFIGURE_FLAGS'),
            stringify(get_extra_cflags(os, compiler), 'EXTRA_CFLAGS'),
            extra_environment_vars)

    job = '    - os: {}\n'.format(os)
    job += '      arch: {}\n'.format(arch)
    job += '      env: {}'.format(env_string)
    return job


def generate_unusual_combinations(unusuals, max_unusual_opts):
    """
    Generates different combinations of non-standard compilers, compiler flags,
    configure flags and malloc_conf settings.

    @param max_unusual_opts: Limit of unusual options per combination.
    """
    return chain.from_iterable(
            [combinations(unusuals, i) for i in range(max_unusual_opts + 1)])


def included(combination, exclude):
    """
    Checks if the combination of options should be included in the Travis
    testing matrix.

    @param exclude: A list of options to be avoided.
    """
    return not any(excluded in combination for excluded in exclude)


def generate_jobs(os, arch, exclude, max_unusual_opts, unusuals=all_unusuals):
    jobs = []
    for combination in generate_unusual_combinations(unusuals, max_unusual_opts):
        if included(combination, exclude):
            jobs.append(format_job(os, arch, combination))
    return '\n'.join(jobs)


def generate_linux(arch):
    os = LINUX

    # Only generate 2 unusual options for AMD64 to reduce matrix size
    max_unusual_opts = MAX_UNUSUAL_OPTIONS if arch == AMD64 else 1

    exclude = []
    if arch == PPC64LE:
        # Avoid 32 bit builds and clang on PowerPC
        exclude = (CROSS_COMPILE_32BIT, CLANG,)

    return generate_jobs(os, arch, exclude, max_unusual_opts)


def generate_macos(arch):
    os = OSX

    max_unusual_opts = 1

    exclude = ([Option.as_malloc_conf(opt) for opt in (
            'dss:primary',
            'percpu_arena:percpu',
            'background_thread:true')] +
        [Option.as_configure_flag('--enable-prof')] +
        [CLANG,])

    return generate_jobs(os, arch, exclude, max_unusual_opts)


def generate_windows(arch):
    os = WINDOWS

    max_unusual_opts = 3
    unusuals = (
        Option.as_configure_flag('--enable-debug'),
        CL,
        CROSS_COMPILE_32BIT,
    )
    return generate_jobs(os, arch, (), max_unusual_opts, unusuals)


def generate_freebsd(arch):
    os = FREEBSD

    max_unusual_opts = 4
    unusuals = (
        Option.as_configure_flag('--enable-debug'),
        Option.as_configure_flag('--enable-prof --enable-prof-libunwind'),
        Option.as_configure_flag('--with-lg-page=16 --with-malloc-conf=tcache:false'),
        CROSS_COMPILE_32BIT,
    )
    return generate_jobs(os, arch, (), max_unusual_opts, unusuals)



def get_manual_jobs():
    return """\
    # Development build
    - os: linux
      env: CC=gcc CXX=g++ CONFIGURE_FLAGS="--enable-debug \
--disable-cache-oblivious --enable-stats --enable-log --enable-prof" \
EXTRA_CFLAGS="-Werror -Wno-array-bounds"
    # --enable-expermental-smallocx:
    - os: linux
      env: CC=gcc CXX=g++ CONFIGURE_FLAGS="--enable-debug \
--enable-experimental-smallocx --enable-stats --enable-prof" \
EXTRA_CFLAGS="-Werror -Wno-array-bounds"
"""


def main():
    jobs = '\n'.join((
        generate_windows(AMD64),

        generate_freebsd(AMD64),

        generate_linux(AMD64),
        generate_linux(PPC64LE),

        generate_macos(AMD64),

        get_manual_jobs(),
    ))

    print(TRAVIS_TEMPLATE.format(jobs=jobs))


if __name__ == '__main__':
    main()