Enable caching/incremental builds for Windows builds on travis.

Problem: msbuild uses timestamps to determine what needs to be rebuilt, and git doesn't track files' last modified time, so everything is always fully rebuilt.
Solution: This commit adds an sqlite database that tracks C/C++ files and their md5 hashes. If a file's hash hasn't changed, then it doesn't need to be rebuilt, so its last modified time is backdated 20 years. The 20 years is an arbitrarily chosen amount of time and has no special significance.
This commit is contained in:
pentarctagon 2020-01-03 09:20:36 -06:00 committed by Pentarctagon
parent d1f2d9add2
commit 6e27a84071
4 changed files with 48 additions and 3 deletions

View file

@ -9,6 +9,8 @@ cache:
directories:
- $HOME/build-cache
- $HOME/flatpak-cache
- $TRAVIS_BUILD_DIR/projectfiles/VC14/Debug
- $TRAVIS_BUILD_DIR/projectfiles/VC14/Release
git:
depth: 5

View file

@ -28,6 +28,7 @@ if [ "$TRAVIS_OS_NAME" = "osx" ]; then
fi
elif [ "$TRAVIS_OS_NAME" = "windows" ]; then
start=`pwd`
choco install sqlite
choco install python --version=3.6.8
cd /c/Python36
ln -s python.exe python3.exe
@ -38,12 +39,14 @@ elif [ "$TRAVIS_OS_NAME" = "windows" ]; then
mv external-VC15 external
cd $start
export PATH="/c/Python36:"$PATH":/c/Program Files (x86)/Microsoft Visual Studio/2017/BuildTools/MSBuild/15.0/Bin/amd64:$start/../external/dll"
if [ "$(which python3)" == "" ] || [ ! -d "../external" ]; then
if [ "$(which python3)" == "" ] || [ "$(which sqlite3)" == "" ] || [ ! -d "../external" ]; then
echo "Failed to retrieve dependencies!"
exit 1
else
echo "Dependencies retrieved and installed!"
fi
./utils/travis/windows-file-hasher.sh "projectfiles/VC14/$OPT/filehashes.sqlite"
else
if [ "$NLS" != "true" ]; then
echo "po/" >> .dockerignore

View file

@ -42,11 +42,19 @@ if [ "$TRAVIS_OS_NAME" = "osx" ]; then
elif [ "$TRAVIS_OS_NAME" = "windows" ]; then
powershell "MSBuild.exe projectfiles/VC14/wesnoth.sln -p:PlatformToolset=v141 -p:Configuration=$OPT"
BUILD_RET=$?
if [ "$BUILD_RET" != "0" ]; then
sqlite3 "projectfiles/VC14/$OPT/filehashes.sqlite" "update FILES set MD5 = OLD_MD5, OLD_MD5 = '-' where OLD_MD5 != '-'"
else
sqlite3 "projectfiles/VC14/$OPT/filehashes.sqlite" "update FILES set OLD_MD5 = '-' where OLD_MD5 != '-'"
fi
if [ "$OPT" == "Release" ] && [ "$BUILD_RET" == "0" ]; then
./run_wml_tests -g -v -c -t "$WML_TEST_TIME"
else
exit $BUILD_RET
BUILD_RET=$?
fi
exit $BUILD_RET
else
# additional permissions required due to flatpak's use of bubblewrap
docker run --cap-add=ALL --privileged \

View file

@ -0,0 +1,32 @@
#!/bin/bash
DBFILE="$1"
if [ "$DBFILE" == "" ]; then
echo "No database file provided!"
exit 1
fi
sqlite3 $DBFILE "create table if not exists FILES(NAME VARCHAR(1000) PRIMARY KEY, MD5 VARCHAR(32), OLD_MD5 VARCHAR(32) DEFAULT '-')"
for file in $(find . -type f | grep '\.c$\|\.cpp$\|\.h$\|\.hpp$\|\.ipp$\|\.tpp$\|\.ii' | cut -c 3-); do
newhash=`md5sum "$file" | cut -d" " -f1`
oldhash=`sqlite3 "$DBFILE" "select MD5 from FILES where NAME = '"$file"'"`
if [ "$oldhash" == "" ]; then
sqlite3 "$DBFILE" "insert into FILES(NAME, MD5) values('$file', '$newhash')"
printf '%-15s %-60s has %-32s\n' "new file:" "$file" "$newhash"
elif [ "$oldhash" != "$newhash" ]; then
sqlite3 "$DBFILE" "update FILES set OLD_MD5 = MD5, MD5 = '$newhash' where NAME = '$file'"
printf '%-15s %-60s had %-32s %-13s %-32s\n' "changed file:" "$file" "$oldhash" "and now has" "$newhash"
else
printf '%-15s %-60s had %-32s %-13s %-32s\n' "unchanged file:" "$file" "$oldhash" "and still has" "$newhash"
touch -d "20 years ago" "$file"
fi
done
for file in `sqlite3 "$DBFILE" "select NAME from FILES"`; do
file=`echo "$file" | tr -d '\r'`
if [ ! -f "$file" ]; then
printf '%-15s %-60s\n' "File removed:" "$file"
sqlite3 "$DBFILE" "delete from FILES where NAME = '$file'"
fi
done