blob: 0f5488d120f13443199935445d8946d1537776c9 [file] [log] [blame]
#!/bin/bash
# Copyright 2023 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
SCRIPT_DIR="$(dirname "$0")"
die () {
echo >&2 "ERROR: $*"
exit 1
}
_VERBOSE=
run () {
if [[ -n "${_VERBOSE}" ]]; then
echo "@COMMAND: $*"
fi
"$@"
}
# Determine Fuchsia host_tag for prebuilt binaries.
UNAME_OS="$(uname -s)"
UNAME_ARCH="$(uname -m)"
case "${UNAME_OS}" in
Linux)
HOST_OS=linux
;;
Darwin)
HOST_OS=mac
;;
*)
die "Unknown host operation system: ${UNAME_OS}"
;;
esac
case "${UNAME_ARCH}" in
amd64|x86_64)
HOST_ARCH=x64
;;
arm64|aarch64)
HOST_ARCH=arm64
;;
*)
die "Unknown host CPU architecture: ${UNAME_ARCH}"
;;
esac
DEFAULT_JEMALLOC_GIT_URL=https://fuchsia.googlesource.com/third_party/github.com/jemalloc/jemalloc.git
DEFAULT_JEMALLOC_TAG=5.3.0
HOST_TAG="${HOST_OS}-${HOST_ARCH}"
ALLOCATOR=
ALLOCATOR_LINK_ONLY=
BUILD_DIR=
CLANG_BINPREFIX=
CMAKE=cmake
DEBUG=
HELP=
INCREMENTAL=
INSTALL_PATH=
LTO=
NINJA=
NO_LTO=
NO_CLANG=
NO_TESTS=
STATIC=
SYSROOT=
TARGET_FLAGS=
USE_LIBCXX=
USE_MINGW64=
USE_WINE=
WINDOWS_SSH=${NINJA_WINDOWS_SSH:-}
for OPT; do
case "${OPT}" in
--help)
HELP=true
;;
--cmake=*)
CMAKE="${OPT#--cmake=}"
;;
--debug)
DEBUG=true
;;
--static)
STATIC=true
;;
-v|--verbose)
_VERBOSE=1
;;
-i|--incremental)
INCREMENTAL=true
;;
--no-clang)
NO_CLANG=true
;;
--build-dir=*)
BUILD_DIR="${OPT#--build-dir=}"
;;
--clang-binprefix=*)
CLANG_BINPREFIX="${OPT#--clang-binprefix=}"
;;
--sysroot=*)
SYSROOT="${OPT#--sysroot=}"
;;
--fuchsia-dir=*)
FUCHSIA_DIR="${OPT#--fuchsia-dir=}"
CLANG_BINPREFIX="${FUCHSIA_DIR}/prebuilt/third_party/clang/${HOST_TAG}/bin"
if [[ "${HOST_OS}" == "linux" ]]; then
SYSROOT="${FUCHSIA_DIR}/prebuilt/third_party/sysroot/linux"
fi
if [[ "${HOST_OS}" == "mac" ]]; then
# NOTE: Required to link binaries properly, otherwise on MacOS
# the prebuilt toolchain will try to link the system libc++.a
# which is incompatible with the libc++ headers that come with
# the prebuilt toolchain.
USE_LIBCXX=true
fi
NINJA="${FUCHSIA_DIR}/prebuilt/third_party/ninja/${HOST_TAG}/ninja"
CMAKE="${FUCHSIA_DIR}/prebuilt/third_party/cmake/${HOST_TAG}/bin/cmake"
;;
--target=*)
TARGET_FLAGS="${OPT}"
;;
--ninja=*)
NINJA="${OPT##--ninja=}"
;;
--no-tests)
NO_TESTS=true
;;
--no-lto)
NO_LTO=true
;;
--allocator=*)
ALLOCATOR="${OPT#--allocator=}"
ALLOCATOR_LINK_ONLY=
;;
--allocator-link-only=*)
ALLOCATOR="${OPT#--allocator-link-only=}"
ALLOCATOR_LINK_ONLY=true
;;
--jemalloc-git-url=*)
JEMALLOC_GIT_URL="${OPT#--jemalloc-git-url=}"
;;
--jemalloc-tag=*)
JEMALLOC_TAG="${OPT#--jemalloc-tag=}"
;;
--use-libc++)
USE_LIBCXX=true
;;
--install-to=*)
INSTALL_PATH="${OPT#--install-to=}"
;;
--mingw64)
USE_MINGW64=true
;;
--wine)
USE_WINE=true
WINDOWS_SSH=
;;
--windows-ssh=*)
WINDOWS_SSH=${OPT##--windows-ssh=}
USE_WINE=
;;
-*)
die "Unknown option $OPT, see --help."
;;
*)
die "This script does not take parameters [$OPT]. See --help."
;;
esac
done
if [[ -n "$HELP" ]]; then
PROGNAME="$(basename "$0")"
cat <<EOF
Usage: ${PROGNAME} [options]
Rebuild an optimized Ninja binary locally, using settings similar to the ones
use by the Fuchsia LUCI recipe (which generates the prebuilt used by the
Fuchsia build). This allows better performance comparisons. This means:
- Enabling link-time optimization
(except if '--no-lto' is used).
- Linking against the rpmalloc or mimalloc library
(when '--allocator=<name>' or '--allocator-link-only=<name>' is used).
- Use the Fuchsia prebuilt Clang toolchain for compilation
(when '--fuchsia-dir=FUCHSIA_DIR' is specified).
The script will also rebuild the test suite and run it to validate the
generated binary. Use '--no-tests'to disable this.
Use '--install-to=PATH' to copy the generated Ninja binary to a specific
location. It will be under 'build-cmake/ninja/build/ninja' otherwise.
Use '--no-clang' to use the default C++ compiler, otherwise the script will
use 'clang++' instead, unless '--clang-binprefix=PREFIX' is used.
Use '--ninja=PATH' to use Ninja as the CMake build generator (default is Make,
which is considerably slower).
Use '--fuchsia-dir=FUCHSIA_DIR' to use the prebuilt Clang toolchain, prebuilt
sysroot and Ninja tool.
Valid options:
--help Print this message.
--cmake=PATH Specify CMake executable path to use.
--ninja=PATH Specify Ninja executable path, to use as a CMake
generator. This results in faster build than the
default Make generator.
--no-lto Disable Link Time Optimization (LTO).
--no-tests Do not run tests.
--incremental, -i Perform incremental build.
--debug Generate debug version.
--install-to=FILE Copy the resulting binary to FILE.
--no-clang Do not use Clang, only the system 'c++' and 'ar'.
--clang-binprefix=DIR Specify directory with Clang toolchain binaries.
--sysroot=DIR Specify sysroot directory.
--fuchsia-dir=DIR Specify Fuchsia directory where to find Clang and sysroot prebuilts.
--target=ARCH Specify clang target triple for cross-compilation..
--build-dir=DIR Specify directory (default is ${BUILD_DIR}).
--static Generate static executable. Note that this can be
slower due to lack of SIMD-optimized memchr() and
related functions.
--mingw64 Use mingw64 on Linux to cross-compile Windows binaries,
--wine Use Wine on Linux to run the tests. Requires
--mingw64.
--windows-ssh=<hostname>
--windows-ssh=<hostname>:<port>
Run Windows binaries on a remote Windows machine
with ssh. Requires --mingw64. Note: setting the
value in the NINJA_WINDOWS_SSH environment variable
works too.
--allocator=rpmalloc
--allocator=mimalloc
--allocator=jemalloc
Build and link an alternative malloc library, which
in many cases results in a 10% faster binary. Note
that the allocator static library is cached in
a user-specific directory to be reused later
with --allocator-link-only (see below).
--allocator-link-only=<name>
Same as --allocator=<name> but do not rebuild the
allocator, and instead reuse a cached version from
a previous --allocator=<name> call.
--jemalloc-git-url=URL
jemalloc source git URL
[$DEFAULT_JEMALLOC_GIT_URL]
--jemalloc-tag=TAG
jemalloc git tag [$DEFAULT_JEMALLOC_TAG]
--use-libc++ Use libc++ (incompatible with --no-clang).
EOF
exit 0
fi
# Directory where to store cached elements (e.g. allocator libraries, gtest, etc)
if [[ -z "${XDG_CACHE_HOME}" ]]; then
XDG_CACHE_HOME=${HOME}/.cache
fi
CACHE_DIR="${XDG_CACHE_HOME}/build-ninja-cache"
# Location of binaries specific to a given build type.
# For now use a single directory, later, separate based on build options to
# speed up incremental builds when switching between build types.
BUILD_CACHE_DIR="${CACHE_DIR}/build"
# Validate --allocator value is any,
case "${ALLOCATOR}" in
jemalloc|rpmalloc|mimalloc|"") # Note: "" selects for ALLOCATOR not being set.
;;
*)
die "Invalid --allocator value ${ALLOCATOR}, must be one of: jemalloc, rpmalloc, mimalloc"
;;
esac
CMAKE_ARGS=()
if [[ -n "${NINJA}" ]]; then
CMAKE_ARGS+=(-GNinja -DCMAKE_MAKE_PROGRAM="${NINJA}")
fi
if [[ -n "${DEBUG}" ]]; then
CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Debug")
else
CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Release")
fi
if [[ -n "${USE_MINGW64}" ]]; then
if [[ -n "${ALLOCATOR}" ]]; then
die "The --mingw64 option is not compatible with the --allocator option"
fi
MINGW64_TOOLCHAIN="${SCRIPT_DIR}/mingw64-toolchain.cmake"
CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=${MINGW64_TOOLCHAIN}")
if [[ -z "${BUILD_DIR}" ]]; then
BUILD_DIR=build-mingw64
fi
if [[ -z "${USE_WINE}" && -z "${WINDOWS_SSH}" ]]; then
echo >&2 "WARNING: Disabling tests due to lack of --wine or --windows-ssh option."
NO_TESTS=true
fi
else
if [[ -n "${USE_WINE}" ]]; then
echo >&2 "ERROR: --wine requires --mingw64"
exit 1
fi
if [[ -n "${WINDOWS_SSH}" ]]; then
echo >&2 "ERROR: --windows-ssh requires --mingw64"
exit 1
fi
fi
if [[ -z "${BUILD_DIR}" ]]; then
BUILD_DIR=build-cmake
fi
if [[ -n "${CLANG_BINPREFIX}" ]]; then
# Ensure CLANG_BINPREFIX has a trailing separator to make
# expressions like "${CLANG_BINPREFIX}clang" work properly.
CLANG_BINPREFIX="${CLANG_BINPREFIX%/}/"
fi
if [[ -n "${NO_CLANG}" ]]; then
CC=$(which cc)
CXX=$(which c++)
AR=$(which ar)
else
CC="${CLANG_BINPREFIX}clang"
CXX="${CLANG_BINPREFIX}clang++"
AR="${CLANG_BINPREFIX}llvm-ar"
fi
if [[ ! -f "${AR}" ]]; then
unset AR
fi
CFLAGS="$TARGET_FLAGS"
CXXFLAGS="$TARGET_FLAGS"
LDFLAGS="$TARGET_FLAGS"
CXXFLAGS="$CXXFLAGS -fno-exceptions -fno-rtti"
if [[ -n "${STATIC}" ]]; then
CXXFLAGS="$CXXFLAGS -static"
LDFLAGS="$LDFLAGS -static"
fi
LIBCXX_FLAGS="-static-libstdc++"
if [[ -z "$USE_MINGW64" ]]; then
LIBCXX_FLAGS="$LIBCXX_FLAGS -pthread"
fi
if [[ -n "${USE_MINGW64}" && -z "${NO_LTO}" ]]; then
echo >&2 "WARNING: Disabling LTO for mingw64 toolchain since it produces broken binaries!"
NO_LTO=true
fi
if [[ -n "${NO_LTO}" ]]; then
CFLAGS="$CFLAGS -fno-lto"
CXXFLAGS="$CXXFLAGS -fno-lto"
LDFLAGS="$LDFLAGS -fno-lto"
fi
if [[ -n "${USE_LIBCXX}" ]]; then
LIBCXX_A="$("${CLANG_BINPREFIX}clang++" --print-file-name=libc++.a)"
if [[ "${LIBCXX_A}" == "libc++.a" ]]; then
# NOTE: --print-file-name=libc++.a does not work on MacOS for some reason.
LIBCXX_A="${CLANG_BINPREFIX}../lib/libc++.a"
if [[ ! -f "${LIBCXX_A}" ]]; then
die "Could not find libc++.a with your Clang toolchain!"
fi
fi
CXXFLAGS="$CXXFLAGS -stdlib=libc++"
LIBCXX_FLAGS="-stdlib=libc++ $LIBCXX_A"
if [[ -z "$USE_MINGW64" ]]; then
LIBCXX_FLAGS="$LIBCXX_FLAGS -lpthread -ldl"
fi
fi
# Ensure we generate smaller executables on Linux
if [[ "$OSTYPE" =~ linux* ]]; then
CFLAGS="$CFLAGS -fPIE"
fi
export CC CXX CFLAGS CXXFLAGS LDFLAGS
if [[ -n "${SYSROOT}" ]]; then
CMAKE_ARGS+=("-DCMAKE_SYSROOT=$SYSROOT")
fi
if [[ "${ALLOCATOR}" == "rpmalloc" ]]; then
RPMALLOC_LIB="${BUILD_CACHE_DIR}/librpmallocwrap.a"
if [[ -z "${ALLOCATOR_LINK_ONLY}" ]]; then
if [[ -z "${NINJA}" ]]; then
die "Ninja is required to build rpmalloc, please use --ninja or --fuchsia-dir!"
fi
echo "Rebuilding rpmalloc library from scratch"
RPMALLOC_GIT_URL="https://fuchsia.googlesource.com/third_party/github.com/mjansson/rpmalloc"
RPMALLOC_BRANCH='+upstream/develop'
RPMALLOC_REVISION='b097fd0916500439721a114bb9cd8d14bd998683'
RPMALLOC_ARCH=x86-64
RPMALLOC_OS=linux
if [[ "$RPMALLOC_OS" == "macos" ]]; then
RPMALLOC_LIBPATH=lib/$RPMALLOC_OS/release/librpmallocwrap.a
else
RPMALLOC_LIBPATH=lib/$RPMALLOC_OS/release/$RPMALLOC_ARCH/librpmallocwrap.a
fi
TMPDIR="$(mktemp -d /tmp/build-rpmalloc.XXXXX)"
(
run cd "$TMPDIR"
run git init
run git fetch --tags --quiet "$RPMALLOC_GIT_URL" "$RPMALLOC_BRANCH"
run git checkout "$RPMALLOC_REVISION"
# Clang will now complain when `-funit-at-a-time` is being used.
run sed -i -e "s|-Wno-disabled-macro-expansion'|-Wno-disabled-macro-expansion', '-Wno-ignored-optimization-argument'|g" build/ninja/clang.py
# Remove -Weverything
run sed -i -e "s|-Weverything|-Wno-pedantic|g" build/ninja/clang.py
export CC CXX AR CFLAGS CXXFLAGS LDFLAGS
run ./configure.py -c release -a $RPMALLOC_ARCH --lto
run ${NINJA} $RPMALLOC_LIBPATH
)
run mkdir -p "$(dirname "$RPMALLOC_LIB")"
run cp -f "${TMPDIR}/${RPMALLOC_LIBPATH}" "${RPMALLOC_LIB}"
else
if [[ ! -f "$RPMALLOC_LIB" ]]; then
die "rpmalloc library is missing, use --allocator=rpmalloc instead: $RPMALLOC_LIB"
fi
fi
echo "Using rpmalloc library: $RPMALLOC_LIB"
fi
if [[ "${ALLOCATOR}" == "mimalloc" ]]; then
MIMALLOC_OBJ="${BUILD_CACHE_DIR}/mimalloc.o"
if [[ -z "${ALLOCATOR_LINK_ONLY}" ]]; then
echo "Rebuilding mimalloc library from scratch"
MIMALLOC_GIT_URL="https://github.com/microsoft/mimalloc.git"
MIMALLOC_BRANCH='master'
MIMALLOC_REVISION='5ac9e36'
MIALLOC_CMAKE_OPTIONS=(\
-DCMAKE_BUILD_TYPE=Release \
-DMI_OVERRIDE=ON \
-DMI_BUILD_SHARED=OFF \
-DMI_BUILD_STATIC=OFF \
-DMI_BUILD_OBJECT=ON \
-DMI_BUILD_TESTS=OFF \
)
if [[ -n "${NINJA}" ]]; then
MIALLOC_CMAKE_OPTIONS+=(-GNinja -DCMAKE_MAKE_PROGRAM="${NINJA}")
fi
TMPDIR=$(mktemp -d /tmp/build-mimalloc.XXXXX)
(
run cd "$TMPDIR"
run git init
run git fetch --tags --quiet "$MIMALLOC_GIT_URL" "$MIMALLOC_BRANCH"
run git checkout "$MIMALLOC_REVISION"
run export CC CXX AR CFLAGS CXXFLAGS LDFLAGS
run cmake -B build-cmake "${MIALLOC_CMAKE_OPTIONS[@]}" .
run cmake --build build-cmake --parallel
)
run mkdir -p "$(dirname "$MIMALLOC_OBJ")"
run cp -f "$TMPDIR/build-cmake/mimalloc.o" "$MIMALLOC_OBJ"
else
if [[ ! -f "$MIMALLOC_OBJ" ]]; then
die "mimalloc library is missing, use --allocator=mimalloc: $MIMALLOC_OBJ"
fi
fi
echo "Using mimalloc library: $MIMALLOC_OBJ"
LDFLAGS="$MIMALLOC_OBJ $LDFLAGS"
fi
if [[ "${ALLOCATOR}" == "jemalloc" ]]; then
JEMALLOC_LIB="${BUILD_CACHE_DIR}/libjemalloc.a"
if [[ -z "${ALLOCATOR_LINK_ONLY}" ]]; then
echo "Rebuilding jemalloc from scratch (this may take a minute or more)."
JEMALLOC_SRC="$(mktemp -d /tmp/build-jemalloc.XXXXXX)"
JEMALLOC_LOG="${TMPDIR:-/tmp}/jemalloc-build-$$.log"
echo "Log file at: ${JEMALLOC_LOG}"
if [[ -z "${JEMALLOC_TAG}" ]]; then
JEMALLOC_TAG="${DEFAULT_JEMALLOC_TAG}"
fi
if [[ -z "$JEMALLOC_GIT_URL" ]]; then
JEMALLOC_GIT_URL="${DEFAULT_JEMALLOC_GIT_URL}"
fi
JEMALLOC_CFLAGS="$CFLAGS -Wno-error"
JEMALLOC_LDFLAGS="$LDFLAGS"
if [[ -n "$LTO" ]]; then
JEMALLOC_CFLAGS="$JEMALLOC_CFLAGS -flto"
JEMALLOC_LDFLAGS="$JEMALLOC_LDFLAGS -flto"
fi
(
run cd "${JEMALLOC_SRC}"
run git init
run git fetch "${JEMALLOC_GIT_URL}" refs/tags/"${JEMALLOC_TAG}" --depth=1
run git checkout FETCH_HEAD
CFLAGS="$JEMALLOC_CFLAGS" \
CXXFLAGS="$JEMALLOC_CFLAGS" \
LDFLAGS="$JEMALLOC_LDFLAGS" \
run ./autogen.sh \
--disable-shared \
--enable-static \
--disable-libdl \
--disable-syscall \
--disable-stats
run make -j"$(nproc)" "lib/libjemalloc.a"
) > "${JEMALLOC_LOG}" 2>&1
if [[ "$?" != 0 ]]; then
die "jemalloc compilation failed, please look at: $JEMALLOC_LOG"
fi
run cp "${JEMALLOC_SRC}/lib/libjemalloc.a" "${JEMALLOC_LIB}"
else
if [[ ! -f "$JEMALLOC_LIB" ]]; then
die "jemalloc library is missing, use --allocator=jemalloc: $JEMALLOC_LIB"
fi
fi
echo "Using jemalloc library: $JEMALLOC_LIB"
LDFLAGS="$JEMALLOC_LIB $LDFLAGS"
fi
LIBCXX_LDFLAGS="${LIBCXX_FLAGS}"
LDFLAGS="$LDFLAGS $LIBCXX_LDFLAGS"
# To link Ninja with rpmalloc, a top-level CMakeLists.txt, that includes the
# Ninja one and adds a few directives, is needed.
SRC_DIR=$(pwd 2>/dev/null)
TEST_DIR="${BUILD_DIR}"
run mkdir -p "${BUILD_DIR}"
if [[ -z "${INCREMENTAL}" ]]; then
run rm -rf "${BUILD_DIR}"/*
fi
BUILD_BUILD_DIR="${BUILD_DIR}"
BUILD_SRC_DIR="${SRC_DIR}"
if [[ -n "${RPMALLOC_LIB}" ]]; then
BUILD_BUILD_DIR="${BUILD_DIR}/build"
BUILD_SRC_DIR="${BUILD_DIR}"
TEST_DIR="${BUILD_BUILD_DIR}/ninja"
if [[ -z "${INCREMENTAL}" ]]; then
cat > "${BUILD_DIR}"/CMakeLists.txt <<EOF
cmake_minimum_required(VERSION 3.15)
project(ninja-rpmalloc)
include(CTest)
add_subdirectory("${SRC_DIR}" ninja)
target_link_libraries(ninja PRIVATE "${RPMALLOC_LIB}" -lpthread -ldl)
EOF
fi
fi
if [[ -z "$INCREMENTAL" ]]; then
run "${CMAKE}" -S"${BUILD_SRC_DIR}" -B"${BUILD_BUILD_DIR}" "${CMAKE_ARGS[@]}"
fi
run "${CMAKE}" --build "${BUILD_BUILD_DIR}" --parallel
if [[ -z "${NO_TESTS}" ]]; then
if [[ -n "${USE_WINE}" ]]; then
run "${SCRIPT_DIR}/mingw64-wine.sh" "${TEST_DIR}"/ninja_test.exe
elif [[ -n "${WINDOWS_SSH}" ]]; then
# Copy program binaries to Windows host with scp, then run them
# with ssh. Print the corresponding command for verification.
IFS=: read -ra _WINDOWS_ARGS <<< "${WINDOWS_SSH}"
_WINDOWS_HOST="${_WINDOWS_ARGS[0]}"
_WINDOWS_PORT="${_WINDOWS_ARGS[1]}"
_WINDOWS_SSH_ARGS=(-t)
_WINDOWS_SCP_ARGS=()
if [[ -n "${_WINDOWS_PORT}" ]]; then
_WINDOWS_SSH_ARGS+=(-p "${_WINDOWS_PORT}")
_WINDOWS_SCP_ARGS+=(-P "${_WINDOWS_PORT}")
fi
# NOTE: Pass all executables because their names change all the time
_SRC_FILES=("${BUILD_BUILD_DIR}"/*.exe)
run scp "${_WINDOWS_SCP_ARGS[@]}" "${_SRC_FILES[@]}" "${_WINDOWS_HOST}":
run ssh "${_WINDOWS_SSH_ARGS[@]}" "${_WINDOWS_HOST}" .\\ninja_test.exe
else
# Run the Ninja tests locally.
# They are run in the test directory, but Python tests are collected from
# the source tree, omitting those from CMake dependencies such
# as ./<build_dir>/_deps/googletest-src/.
#
# The main problem with them occurs when there are several build directories
# in the same top-level dir (e.g. `build-cmake/` and `build-mingw64/`
# because pytest wouldi, by default, collect tests from both directories on
# the next build, resulting in a very weird Python module import failure!
mapfile -t PYTHON_TESTS <<< "$(find "${SRC_DIR}" -name '*_test.py' | grep -v _deps/)"
(run cd "${TEST_DIR}" && run ./ninja_test && run pytest "${PYTHON_TESTS[@]}")
fi
fi
if [[ -n "${INSTALL_PATH}" ]]; then
run cp "${TEST_DIR}"/ninja "${INSTALL_PATH}"
fi