diff --git a/Toolchain/BuildIt.sh b/Toolchain/BuildIt.sh index 28470271840..7b2a6b0aef1 100755 --- a/Toolchain/BuildIt.sh +++ b/Toolchain/BuildIt.sh @@ -19,11 +19,6 @@ MAKE="make" MD5SUM="md5sum" NPROC="nproc" -# Each cache entry is 70 MB. 5 entries are 350 MiB. -# It seems that Travis starts having trouble around a total -# cache size of 9 GiB, so I think this is a good amount. -KEEP_CACHE_COUNT=5 - if command -v ginstall &>/dev/null; then INSTALL=ginstall else @@ -77,60 +72,41 @@ GCC_BASE_URL="http://ftp.gnu.org/gnu/gcc" pushd "$DIR" if [ "${TRY_USE_LOCAL_TOOLCHAIN}" = "y" ] ; then - echo "Checking cached toolchain:" - # TODO: This is still overly pessimistic. - DEPS_CONFIG="\ - uname=$(uname),TARGET=${TARGET}, - BuildItHash=$($MD5SUM "$(basename "$0")"), - MAKE=${MAKE},MD5SUM=${MD5SUM},NPROC=${NPROC}, - CC=${CC},CXX=${CXX},with_gmp=${with_gmp},LDFLAGS=${LDFLAGS}, - BINUTILS_VERSION=${BINUTILS_VERSION},BINUTILS_MD5SUM=${BINUTILS_MD5SUM}, - GCC_VERSION=${GCC_VERSION},GCC_MD5SUM=${GCC_MD5SUM}" - DEPS_HASH=$("$DIR/ComputeDependenciesHash.sh" "$MD5SUM" <<<"${DEPS_CONFIG}") - DEPS_HASH_EXIT_CODE="$?" - CACHED_TOOLCHAIN_ARCHIVE="Cache/ToolchainLocal_${DEPS_HASH}.tar.gz" - if [ "$DEPS_HASH_EXIT_CODE" -ne 0 ] ; then - # Make it stand out more - echo - echo - echo - echo - echo "Dependency hashing failed" - echo "Will rebuild toolchain from scratch, and NOT SAVE THE RESULT." - echo "Someone should look into this, but for now it'll work, albeit inefficient." - echo - echo - echo - echo - # Should be empty anyway, but just to make sure: - DEPS_HASH="" - elif [ -r "${CACHED_TOOLCHAIN_ARCHIVE}" ] ; then + # The actual logic had to be moved to .github/workflows/cmake.yml. + # Github Actions guarantees that Toolchain/Cache/ is empty on a cache + # miss, and non-empty on a cache hit. + # The following logic is correct *only* because of that. + + mkdir -p Cache + echo "Cache (before):" + ls -l Cache + CACHED_TOOLCHAIN_ARCHIVE="Cache/ToolchainBinariesGithubActions.tar.gz" + if [ -r "${CACHED_TOOLCHAIN_ARCHIVE}" ] ; then echo "Cache at ${CACHED_TOOLCHAIN_ARCHIVE} exists!" echo "Extracting toolchain from cache:" if tar xzf "${CACHED_TOOLCHAIN_ARCHIVE}" ; then echo "Done 'building' the toolchain." + echo "Cache unchanged." exit 0 else - echo "Could not extract cached toolchain archive, removing it and rebuilding from scratch." + echo + echo + echo + echo "Could not extract cached toolchain archive." + echo "This means the cache is broken and *should be removed*!" + echo "As Github Actions cannot update a cache, this will unnecessarily" + echo "slow down all future builds for this hash, until someone" + echo "resets the cache." + echo + echo + echo rm -f "${CACHED_TOOLCHAIN_ARCHIVE}" fi else echo "Cache at ${CACHED_TOOLCHAIN_ARCHIVE} does not exist." echo "Will rebuild toolchain from scratch, and save the result." - echo "But first, getting rid of old, outdated caches. Current caches:" - pushd "Cache/" - ls -l - # Travis preserves timestamps. Don't ask me why, but it does. - # We can exploit this to get an easy approximation of recent-ness. - # Our purging algorithm is simple: keep only the newest X entries. - # Note that `find` doesn't easily support ordering by date, - # and we control the filenames anyway. - # shellcheck disable=SC2012 - ls -t | tail "-n+${KEEP_CACHE_COUNT}" | xargs -r rm -v - echo "After deletion:" - ls -l - popd fi + echo "::group::Actually building Toolchain" fi popd @@ -285,24 +261,20 @@ popd pushd "$DIR" if [ "${TRY_USE_LOCAL_TOOLCHAIN}" = "y" ] ; then - echo "Caching toolchain:" + echo "::endgroup::" + echo "Building cache tar:" - if [ -z "${DEPS_HASH}" ] ; then - echo "NOT SAVED, because hashing failed." - echo "It's computed in the beginning; see there for the error message." - elif [ -e "Cache/ToolchainLocal_${DEPS_HASH}.tar.gz" ] ; then - # Note: This checks for *existence*. Initially we checked for - # *readability*. If Travis borks permissions, there's not much we can do. - echo "Cache exists but was not used?!" - echo "Not touching cache then." - else - mkdir -p Cache/ - # We *most definitely* don't need debug symbols in the linker/compiler. - # This cuts the uncompressed size from 1.2 GiB per Toolchain down to about 190 MiB. - echo "Before: $(du -sh Local)" - find Local/ -type f -executable ! -name '*.la' ! -name '*.sh' ! -name 'mk*' -exec strip {} + - echo "After: $(du -sh Local)" - tar czf "Cache/ToolchainLocal_${DEPS_HASH}.tar.gz" Local/ - fi + rm -f "${CACHED_TOOLCHAIN_ARCHIVE}" # Just in case + + # We *most definitely* don't need debug symbols in the linker/compiler. + # This cuts the uncompressed size from 1.2 GiB per Toolchain down to about 190 MiB. + echo "Stripping executables ..." + echo "Before: $(du -sh Local)" + find Local/ -type f -executable ! -name '*.la' ! -name '*.sh' ! -name 'mk*' -exec strip {} + + echo "After: $(du -sh Local)" + tar czf "${CACHED_TOOLCHAIN_ARCHIVE}" Local/ + + echo "Cache (after):" + ls -l Cache fi popd diff --git a/Toolchain/ComputeDependenciesHash.sh b/Toolchain/ComputeDependenciesHash.sh deleted file mode 100755 index 3a4ba36889a..00000000000 --- a/Toolchain/ComputeDependenciesHash.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -# This file will need to be run in bash, for now. - -if [ $# -lt 1 ] ; then - echo "USAGE: echo \"YOURCONFIG\" | $0 " >&2 - echo "Example: echo \"uname=Linux,TARGET=i686-pc-serenity\" | $0 md5sum" >&2 - echo "Example: echo \"uname=OpenBSD,TARGET=i686-pc-serenity\" | $0 md5 -q" >&2 - exit 1 -fi - -DIR=$( cd "$( dirname "$0" )" && pwd ) -cd "${DIR}/.." -if [ ! -r LICENSE ] ; then - echo "$0: Got confused by the directories, giving up." >&2 - exit 1 -fi - -# Ensure cleanup -DEPLIST_FILE=$(mktemp /tmp/serenity_deps_XXXXXXXX.lst) -function finish { - rm -f "${DEPLIST_FILE}" -} -trap finish EXIT - -# First, capture the caller's input. -echo "$0: Configuration:" >&2 -cat /dev/stdin | tee /dev/stderr > "${DEPLIST_FILE}" -# "$@" is the md5sum invocation. -"$@" Toolchain/ComputeDependenciesHash.sh | tee /dev/stderr >> "${DEPLIST_FILE}" - -# libstdc++ depends on the *headers* of libc, so we pessimistically assume it depends -# on *all* of them. -# This list of files can be cut down considerably: -# strace -ff -e trace=file "make install-target-libstdc++-v3" 2>&1 >/dev/null | perl -ne 's/^[^"]+"(([^\\"]|\\[\\"nt])*)".*/$1/ && print' | sort -u | grep -P 'serenity/Build/Root/usr/include/.*\.h$' -# However, we don't want to risk breaking the build when we upgrade gcc in the future. -# -# If you want to further cut down the Toolchain rebuilds on Travis, -# one way would be to reduce this list somehow. -cd Libraries/LibC/ -find -name '*.h' | sort | xargs "$@" | tee /dev/stderr >> "${DEPLIST_FILE}" - -# The piping might hide non-zero exit-codes, -# but thankfully only the first command can reasonably fail. -echo "$0: Toolchain hash:" >&2 -"$@" "${DEPLIST_FILE}" | cut -f1 -d' ' | tee /dev/stderr - -echo "$0: Great success!" >&2