From a82a17eee8dc418c36c734d2d00333fa46bbb1f5 Mon Sep 17 00:00:00 2001 From: Ben Wiederhake Date: Tue, 3 Nov 2020 20:43:23 +0100 Subject: [PATCH] Toolchain: Trust cache blindly This is necessary because cache reusability will be determined by Github Actions. Note that we only cache if explicitly asked to do so, which only happens on Github Actions. --- Toolchain/BuildIt.sh | 102 ++++++++++----------------- Toolchain/ComputeDependenciesHash.sh | 48 ------------- 2 files changed, 37 insertions(+), 113 deletions(-) delete mode 100755 Toolchain/ComputeDependenciesHash.sh diff --git a/Toolchain/BuildIt.sh b/Toolchain/BuildIt.sh index 28470271840..7b2a6b0aef1 100755 --- a/Toolchain/BuildIt.sh +++ b/Toolchain/BuildIt.sh @@ -19,11 +19,6 @@ MAKE="make" MD5SUM="md5sum" NPROC="nproc" -# Each cache entry is 70 MB. 5 entries are 350 MiB. -# It seems that Travis starts having trouble around a total -# cache size of 9 GiB, so I think this is a good amount. -KEEP_CACHE_COUNT=5 - if command -v ginstall &>/dev/null; then INSTALL=ginstall else @@ -77,60 +72,41 @@ GCC_BASE_URL="http://ftp.gnu.org/gnu/gcc" pushd "$DIR" if [ "${TRY_USE_LOCAL_TOOLCHAIN}" = "y" ] ; then - echo "Checking cached toolchain:" - # TODO: This is still overly pessimistic. - DEPS_CONFIG="\ - uname=$(uname),TARGET=${TARGET}, - BuildItHash=$($MD5SUM "$(basename "$0")"), - MAKE=${MAKE},MD5SUM=${MD5SUM},NPROC=${NPROC}, - CC=${CC},CXX=${CXX},with_gmp=${with_gmp},LDFLAGS=${LDFLAGS}, - BINUTILS_VERSION=${BINUTILS_VERSION},BINUTILS_MD5SUM=${BINUTILS_MD5SUM}, - GCC_VERSION=${GCC_VERSION},GCC_MD5SUM=${GCC_MD5SUM}" - DEPS_HASH=$("$DIR/ComputeDependenciesHash.sh" "$MD5SUM" <<<"${DEPS_CONFIG}") - DEPS_HASH_EXIT_CODE="$?" - CACHED_TOOLCHAIN_ARCHIVE="Cache/ToolchainLocal_${DEPS_HASH}.tar.gz" - if [ "$DEPS_HASH_EXIT_CODE" -ne 0 ] ; then - # Make it stand out more - echo - echo - echo - echo - echo "Dependency hashing failed" - echo "Will rebuild toolchain from scratch, and NOT SAVE THE RESULT." - echo "Someone should look into this, but for now it'll work, albeit inefficient." - echo - echo - echo - echo - # Should be empty anyway, but just to make sure: - DEPS_HASH="" - elif [ -r "${CACHED_TOOLCHAIN_ARCHIVE}" ] ; then + # The actual logic had to be moved to .github/workflows/cmake.yml. + # Github Actions guarantees that Toolchain/Cache/ is empty on a cache + # miss, and non-empty on a cache hit. + # The following logic is correct *only* because of that. + + mkdir -p Cache + echo "Cache (before):" + ls -l Cache + CACHED_TOOLCHAIN_ARCHIVE="Cache/ToolchainBinariesGithubActions.tar.gz" + if [ -r "${CACHED_TOOLCHAIN_ARCHIVE}" ] ; then echo "Cache at ${CACHED_TOOLCHAIN_ARCHIVE} exists!" echo "Extracting toolchain from cache:" if tar xzf "${CACHED_TOOLCHAIN_ARCHIVE}" ; then echo "Done 'building' the toolchain." + echo "Cache unchanged." exit 0 else - echo "Could not extract cached toolchain archive, removing it and rebuilding from scratch." + echo + echo + echo + echo "Could not extract cached toolchain archive." + echo "This means the cache is broken and *should be removed*!" + echo "As Github Actions cannot update a cache, this will unnecessarily" + echo "slow down all future builds for this hash, until someone" + echo "resets the cache." + echo + echo + echo rm -f "${CACHED_TOOLCHAIN_ARCHIVE}" fi else echo "Cache at ${CACHED_TOOLCHAIN_ARCHIVE} does not exist." echo "Will rebuild toolchain from scratch, and save the result." - echo "But first, getting rid of old, outdated caches. Current caches:" - pushd "Cache/" - ls -l - # Travis preserves timestamps. Don't ask me why, but it does. - # We can exploit this to get an easy approximation of recent-ness. - # Our purging algorithm is simple: keep only the newest X entries. - # Note that `find` doesn't easily support ordering by date, - # and we control the filenames anyway. - # shellcheck disable=SC2012 - ls -t | tail "-n+${KEEP_CACHE_COUNT}" | xargs -r rm -v - echo "After deletion:" - ls -l - popd fi + echo "::group::Actually building Toolchain" fi popd @@ -285,24 +261,20 @@ popd pushd "$DIR" if [ "${TRY_USE_LOCAL_TOOLCHAIN}" = "y" ] ; then - echo "Caching toolchain:" + echo "::endgroup::" + echo "Building cache tar:" - if [ -z "${DEPS_HASH}" ] ; then - echo "NOT SAVED, because hashing failed." - echo "It's computed in the beginning; see there for the error message." - elif [ -e "Cache/ToolchainLocal_${DEPS_HASH}.tar.gz" ] ; then - # Note: This checks for *existence*. Initially we checked for - # *readability*. If Travis borks permissions, there's not much we can do. - echo "Cache exists but was not used?!" - echo "Not touching cache then." - else - mkdir -p Cache/ - # We *most definitely* don't need debug symbols in the linker/compiler. - # This cuts the uncompressed size from 1.2 GiB per Toolchain down to about 190 MiB. - echo "Before: $(du -sh Local)" - find Local/ -type f -executable ! -name '*.la' ! -name '*.sh' ! -name 'mk*' -exec strip {} + - echo "After: $(du -sh Local)" - tar czf "Cache/ToolchainLocal_${DEPS_HASH}.tar.gz" Local/ - fi + rm -f "${CACHED_TOOLCHAIN_ARCHIVE}" # Just in case + + # We *most definitely* don't need debug symbols in the linker/compiler. + # This cuts the uncompressed size from 1.2 GiB per Toolchain down to about 190 MiB. + echo "Stripping executables ..." + echo "Before: $(du -sh Local)" + find Local/ -type f -executable ! -name '*.la' ! -name '*.sh' ! -name 'mk*' -exec strip {} + + echo "After: $(du -sh Local)" + tar czf "${CACHED_TOOLCHAIN_ARCHIVE}" Local/ + + echo "Cache (after):" + ls -l Cache fi popd diff --git a/Toolchain/ComputeDependenciesHash.sh b/Toolchain/ComputeDependenciesHash.sh deleted file mode 100755 index 3a4ba36889a..00000000000 --- a/Toolchain/ComputeDependenciesHash.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -# This file will need to be run in bash, for now. - -if [ $# -lt 1 ] ; then - echo "USAGE: echo \"YOURCONFIG\" | $0 " >&2 - echo "Example: echo \"uname=Linux,TARGET=i686-pc-serenity\" | $0 md5sum" >&2 - echo "Example: echo \"uname=OpenBSD,TARGET=i686-pc-serenity\" | $0 md5 -q" >&2 - exit 1 -fi - -DIR=$( cd "$( dirname "$0" )" && pwd ) -cd "${DIR}/.." -if [ ! -r LICENSE ] ; then - echo "$0: Got confused by the directories, giving up." >&2 - exit 1 -fi - -# Ensure cleanup -DEPLIST_FILE=$(mktemp /tmp/serenity_deps_XXXXXXXX.lst) -function finish { - rm -f "${DEPLIST_FILE}" -} -trap finish EXIT - -# First, capture the caller's input. -echo "$0: Configuration:" >&2 -cat /dev/stdin | tee /dev/stderr > "${DEPLIST_FILE}" -# "$@" is the md5sum invocation. -"$@" Toolchain/ComputeDependenciesHash.sh | tee /dev/stderr >> "${DEPLIST_FILE}" - -# libstdc++ depends on the *headers* of libc, so we pessimistically assume it depends -# on *all* of them. -# This list of files can be cut down considerably: -# strace -ff -e trace=file "make install-target-libstdc++-v3" 2>&1 >/dev/null | perl -ne 's/^[^"]+"(([^\\"]|\\[\\"nt])*)".*/$1/ && print' | sort -u | grep -P 'serenity/Build/Root/usr/include/.*\.h$' -# However, we don't want to risk breaking the build when we upgrade gcc in the future. -# -# If you want to further cut down the Toolchain rebuilds on Travis, -# one way would be to reduce this list somehow. -cd Libraries/LibC/ -find -name '*.h' | sort | xargs "$@" | tee /dev/stderr >> "${DEPLIST_FILE}" - -# The piping might hide non-zero exit-codes, -# but thankfully only the first command can reasonably fail. -echo "$0: Toolchain hash:" >&2 -"$@" "${DEPLIST_FILE}" | cut -f1 -d' ' | tee /dev/stderr - -echo "$0: Great success!" >&2