ladybird/Toolchain/BuildIt.sh

#!/usr/bin/env bash
set -e
# This file will need to be run in bash, for now.


# === CONFIGURATION AND SETUP ===

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

echo "$DIR"

ARCH=${ARCH:-"i686"}
TARGET="$ARCH-pc-serenity"
PREFIX="$DIR/Local"
BUILD=$(realpath "$DIR/../Build")
SYSROOT="$BUILD/Root"

MAKE="make"
MD5SUM="md5sum"
NPROC="nproc"

# Each cache entry is 70 MB. 5 entries are 350 MiB.
# It seems that Travis starts having trouble around a total
# cache size of 9 GiB, so I think this is a good amount.
KEEP_CACHE_COUNT=5

if command -v ginstall &>/dev/null; then
    INSTALL=ginstall
else
    INSTALL=install
fi

if [ "$(uname -s)" = "OpenBSD" ]; then
    MAKE=gmake
    MD5SUM="md5 -q"
    NPROC="sysctl -n hw.ncpuonline"
    export CC=egcc
    export CXX=eg++
    export with_gmp=/usr/local
    export LDFLAGS=-Wl,-z,notext
elif [ "$(uname -s)" = "FreeBSD" ]; then
    MAKE=gmake
    MD5SUM="md5 -q"
    NPROC="sysctl -n hw.ncpu"
fi

git_patch=
while [ "$1" != "" ]; do
    case $1 in
        --dev )           git_patch=1
                          ;;
    esac
    shift
done

echo PREFIX is "$PREFIX"
echo SYSROOT is "$SYSROOT"

mkdir -p "$DIR/Tarballs"

BINUTILS_VERSION="2.35.1"
BINUTILS_MD5SUM="bca600eea3b8fc33ad3265c9c1eee8d4"
BINUTILS_NAME="binutils-$BINUTILS_VERSION"
BINUTILS_PKG="${BINUTILS_NAME}.tar.gz"
BINUTILS_BASE_URL="http://ftp.gnu.org/gnu/binutils"

GCC_VERSION="10.2.0"
GCC_MD5SUM="941a8674ea2eeb33f5c30ecf08124874"
GCC_NAME="gcc-$GCC_VERSION"
GCC_PKG="${GCC_NAME}.tar.gz"
GCC_BASE_URL="http://ftp.gnu.org/gnu/gcc"


# === CHECK CACHE AND REUSE ===

pushd "$DIR"
    if [ "${TRY_USE_LOCAL_TOOLCHAIN}" = "y" ] ; then
        echo "Checking cached toolchain:"
        # TODO: This is still overly pessimistic.
        DEPS_CONFIG="\
            uname=$(uname),TARGET=${TARGET},
            BuildItHash=$($MD5SUM "$(basename "$0")"),
            MAKE=${MAKE},MD5SUM=${MD5SUM},NPROC=${NPROC},
            CC=${CC},CXX=${CXX},with_gmp=${with_gmp},LDFLAGS=${LDFLAGS},
            BINUTILS_VERSION=${BINUTILS_VERSION},BINUTILS_MD5SUM=${BINUTILS_MD5SUM},
            GCC_VERSION=${GCC_VERSION},GCC_MD5SUM=${GCC_MD5SUM}"
        DEPS_HASH=$("$DIR/ComputeDependenciesHash.sh" "$MD5SUM" <<<"${DEPS_CONFIG}")
        DEPS_HASH_EXIT_CODE="$?"
        CACHED_TOOLCHAIN_ARCHIVE="Cache/ToolchainLocal_${DEPS_HASH}.tar.gz"
        if [ "$DEPS_HASH_EXIT_CODE" -ne 0 ] ; then
            # Make it stand out more
            echo
            echo
            echo
            echo
            echo "Dependency hashing failed"
            echo "Will rebuild toolchain from scratch, and NOT SAVE THE RESULT."
            echo "Someone should look into this, but for now it'll work, albeit inefficient."
            echo
            echo
            echo
            echo
            # Should be empty anyway, but just to make sure:
            DEPS_HASH=""
        elif [ -r "${CACHED_TOOLCHAIN_ARCHIVE}" ] ; then
            echo "Cache at ${CACHED_TOOLCHAIN_ARCHIVE} exists!"
            echo "Extracting toolchain from cache:"
            if tar xzf "${CACHED_TOOLCHAIN_ARCHIVE}" ; then
                echo "Done 'building' the toolchain."
                exit 0
            else
                echo "Could not extract cached toolchain archive, removing it and rebuilding from scratch."
                rm -f "${CACHED_TOOLCHAIN_ARCHIVE}"
            fi
        else
            echo "Cache at ${CACHED_TOOLCHAIN_ARCHIVE} does not exist."
            echo "Will rebuild toolchain from scratch, and save the result."
            echo "But first, getting rid of old, outdated caches. Current caches:"
            pushd "Cache/"
                ls -l
                # Travis preserves timestamps. Don't ask me why, but it does.
                # We can exploit this to get an easy approximation of recent-ness.
                # Our purging algorithm is simple: keep only the newest X entries.
                # Note that `find` doesn't easily support ordering by date,
                # and we control the filenames anyway.
                # shellcheck disable=SC2012
                ls -t | tail "-n+${KEEP_CACHE_COUNT}" | xargs -r rm -v
                echo "After deletion:"
                ls -l
            popd
        fi
    fi
popd


# === DOWNLOAD AND PATCH ===

pushd "$DIR/Tarballs"
    md5="$($MD5SUM $BINUTILS_PKG | cut -f1 -d' ')"
    echo "bu md5='$md5'"
    if [ ! -e $BINUTILS_PKG ] || [ "$md5" != ${BINUTILS_MD5SUM} ] ; then
        rm -f $BINUTILS_PKG
        curl -LO "$BINUTILS_BASE_URL/$BINUTILS_PKG"
    else
        echo "Skipped downloading binutils"
    fi

    md5="$($MD5SUM ${GCC_PKG} | cut -f1 -d' ')"
    echo "gc md5='$md5'"
    if [ ! -e $GCC_PKG ] || [ "$md5" != ${GCC_MD5SUM} ] ; then
        rm -f $GCC_PKG
        curl -LO "$GCC_BASE_URL/$GCC_NAME/$GCC_PKG"
    else
        echo "Skipped downloading gcc"
    fi

    if [ ! -d ${BINUTILS_NAME} ]; then
        echo "Extracting binutils..."
        tar -xzf ${BINUTILS_PKG}

        pushd ${BINUTILS_NAME}
            if [ "$git_patch" = "1" ]; then
                git init > /dev/null
                git add . > /dev/null
                git commit -am "BASE" > /dev/null
                git apply "$DIR"/Patches/binutils.patch > /dev/null
            else
                patch -p1 < "$DIR"/Patches/binutils.patch > /dev/null
            fi
        popd
    else
        echo "Skipped extracting binutils"
    fi

    if [ ! -d $GCC_NAME ]; then
        echo "Extracting gcc..."
        tar -xzf $GCC_PKG
        pushd $GCC_NAME
            if [ "$git_patch" = "1" ]; then
                git init > /dev/null
                git add . > /dev/null
                git commit -am "BASE" > /dev/null
                git apply "$DIR"/Patches/gcc.patch > /dev/null
            else
                patch -p1 < "$DIR/Patches/gcc.patch" > /dev/null
            fi
        popd
    else
        echo "Skipped extracting gcc"
    fi

    if [ "$(uname)" = "Darwin" ]; then
        pushd "gcc-${GCC_VERSION}"
        ./contrib/download_prerequisites
        popd
    fi

popd


# === COMPILE AND INSTALL ===

mkdir -p "$PREFIX"
mkdir -p "$DIR/Build/binutils"
mkdir -p "$DIR/Build/gcc"

if [ -z "$MAKEJOBS" ]; then
    MAKEJOBS=$($NPROC)
fi

pushd "$DIR/Build/"
    unset PKG_CONFIG_LIBDIR # Just in case

    pushd binutils
        echo "XXX configure binutils"
        "$DIR"/Tarballs/$BINUTILS_NAME/configure --prefix="$PREFIX" \
                                                 --target="$TARGET" \
                                                 --with-sysroot="$SYSROOT" \
                                                 --enable-shared \
                                                 --disable-nls \
                                                 ${TRY_USE_LOCAL_TOOLCHAIN:+"--quiet"} || exit 1
        if [ "$(uname)" = "Darwin" ]; then
            # under macOS generated makefiles are not resolving the "intl"
            # dependency properly to allow linking its own copy of
            # libintl when building with --enable-shared.
            "$MAKE" -j "$MAKEJOBS" || true
            pushd intl
            "$MAKE" all-yes
            popd
        fi
        echo "XXX build binutils"
        "$MAKE" -j "$MAKEJOBS" || exit 1
        "$MAKE" install || exit 1
    popd

    pushd gcc
        if [ "$(uname -s)" = "OpenBSD" ]; then
            perl -pi -e 's/-no-pie/-nopie/g' "$DIR/Tarballs/gcc-$GCC_VERSION/gcc/configure"
        fi

        echo "XXX configure gcc and libgcc"
        "$DIR/Tarballs/gcc-$GCC_VERSION/configure" --prefix="$PREFIX" \
                                            --target="$TARGET" \
                                            --with-sysroot="$SYSROOT" \
                                            --disable-nls \
                                            --with-newlib \
                                            --enable-shared \
                                            --enable-languages=c,c++ \
                                            ${TRY_USE_LOCAL_TOOLCHAIN:+"--quiet"} || exit 1

        echo "XXX build gcc and libgcc"
        "$MAKE" -j "$MAKEJOBS" all-gcc all-target-libgcc || exit 1
        echo "XXX install gcc and libgcc"
        "$MAKE" install-gcc install-target-libgcc || exit 1

        echo "XXX serenity libc and libm headers"
        mkdir -p "$BUILD"
        pushd "$BUILD"
            mkdir -p Root/usr/include/
            SRC_ROOT=$(realpath "$DIR"/..)
            FILES=$(find "$SRC_ROOT"/Libraries/LibC "$SRC_ROOT"/Libraries/LibM -name '*.h' -print)
            for header in $FILES; do
                target=$(echo "$header" | sed -e "s@$SRC_ROOT/Libraries/LibC@@" -e "s@$SRC_ROOT/Libraries/LibM@@")
                $INSTALL -D "$header" "Root/usr/include/$target"
            done
            unset SRC_ROOT
        popd

        echo "XXX build libstdc++"
        "$MAKE" -j "$MAKEJOBS" all-target-libstdc++-v3 || exit 1
        echo "XXX install libstdc++"
        "$MAKE" install-target-libstdc++-v3 || exit 1

        if [ "$(uname -s)" = "OpenBSD" ]; then
            cd "$DIR/Local/libexec/gcc/i686-pc-serenity/$GCC_VERSION" && ln -sf liblto_plugin.so.0.0 liblto_plugin.so
        fi

    popd
popd


# == SAVE TO CACHE ==

pushd "$DIR"
    if [ "${TRY_USE_LOCAL_TOOLCHAIN}" = "y" ] ; then
        echo "Caching toolchain:"

        if [ -z "${DEPS_HASH}" ] ; then
            echo "NOT SAVED, because hashing failed."
            echo "It's computed in the beginning; see there for the error message."
        elif [ -e "Cache/ToolchainLocal_${DEPS_HASH}.tar.gz" ] ; then
            # Note: This checks for *existence*.  Initially we checked for
            # *readability*. If Travis borks permissions, there's not much we can do.
            echo "Cache exists but was not used?!"
            echo "Not touching cache then."
        else
            mkdir -p Cache/
            # We *most definitely* don't need debug symbols in the linker/compiler.
            # This cuts the uncompressed size from 1.2 GiB per Toolchain down to about 190 MiB.
            echo "Before: $(du -sh Local)"
            find Local/ -type f -executable ! -name '*.la' ! -name '*.sh' ! -name 'mk*' -exec strip {} +
            echo "After: $(du -sh Local)"
            tar czf "Cache/ToolchainLocal_${DEPS_HASH}.tar.gz" Local/
        fi
    fi
popd