Enable caching/incremental builds for Windows builds on travis.

Problem: msbuild uses timestamps to determine what needs to be rebuilt, and git doesn't track files' last modified time, so everything is always fully rebuilt.
Solution: This commit adds an sqlite database that tracks C/C++ files and their md5 hashes. If a file's hash hasn't changed, then it doesn't need to be rebuilt, so its last modified time is backdated 20 years. The 20 years is an arbitrarily chosen amount of time and has no special significance.
This commit is contained in:
pentarctagon 2020-01-03 09:20:36 -06:00 committed by Pentarctagon
parent d1f2d9add2
commit 6e27a84071
4 changed files with 48 additions and 3 deletions

View file

@ -9,6 +9,8 @@ cache:
directories: directories:
- $HOME/build-cache - $HOME/build-cache
- $HOME/flatpak-cache - $HOME/flatpak-cache
- $TRAVIS_BUILD_DIR/projectfiles/VC14/Debug
- $TRAVIS_BUILD_DIR/projectfiles/VC14/Release
git: git:
depth: 5 depth: 5

View file

@ -28,6 +28,7 @@ if [ "$TRAVIS_OS_NAME" = "osx" ]; then
fi fi
elif [ "$TRAVIS_OS_NAME" = "windows" ]; then elif [ "$TRAVIS_OS_NAME" = "windows" ]; then
start=`pwd` start=`pwd`
choco install sqlite
choco install python --version=3.6.8 choco install python --version=3.6.8
cd /c/Python36 cd /c/Python36
ln -s python.exe python3.exe ln -s python.exe python3.exe
@ -38,12 +39,14 @@ elif [ "$TRAVIS_OS_NAME" = "windows" ]; then
mv external-VC15 external mv external-VC15 external
cd $start cd $start
export PATH="/c/Python36:"$PATH":/c/Program Files (x86)/Microsoft Visual Studio/2017/BuildTools/MSBuild/15.0/Bin/amd64:$start/../external/dll" export PATH="/c/Python36:"$PATH":/c/Program Files (x86)/Microsoft Visual Studio/2017/BuildTools/MSBuild/15.0/Bin/amd64:$start/../external/dll"
if [ "$(which python3)" == "" ] || [ ! -d "../external" ]; then if [ "$(which python3)" == "" ] || [ "$(which sqlite3)" == "" ] || [ ! -d "../external" ]; then
echo "Failed to retrieve dependencies!" echo "Failed to retrieve dependencies!"
exit 1 exit 1
else else
echo "Dependencies retrieved and installed!" echo "Dependencies retrieved and installed!"
fi fi
./utils/travis/windows-file-hasher.sh "projectfiles/VC14/$OPT/filehashes.sqlite"
else else
if [ "$NLS" != "true" ]; then if [ "$NLS" != "true" ]; then
echo "po/" >> .dockerignore echo "po/" >> .dockerignore

View file

@ -42,11 +42,19 @@ if [ "$TRAVIS_OS_NAME" = "osx" ]; then
elif [ "$TRAVIS_OS_NAME" = "windows" ]; then elif [ "$TRAVIS_OS_NAME" = "windows" ]; then
powershell "MSBuild.exe projectfiles/VC14/wesnoth.sln -p:PlatformToolset=v141 -p:Configuration=$OPT" powershell "MSBuild.exe projectfiles/VC14/wesnoth.sln -p:PlatformToolset=v141 -p:Configuration=$OPT"
BUILD_RET=$? BUILD_RET=$?
if [ "$BUILD_RET" != "0" ]; then
sqlite3 "projectfiles/VC14/$OPT/filehashes.sqlite" "update FILES set MD5 = OLD_MD5, OLD_MD5 = '-' where OLD_MD5 != '-'"
else
sqlite3 "projectfiles/VC14/$OPT/filehashes.sqlite" "update FILES set OLD_MD5 = '-' where OLD_MD5 != '-'"
fi
if [ "$OPT" == "Release" ] && [ "$BUILD_RET" == "0" ]; then if [ "$OPT" == "Release" ] && [ "$BUILD_RET" == "0" ]; then
./run_wml_tests -g -v -c -t "$WML_TEST_TIME" ./run_wml_tests -g -v -c -t "$WML_TEST_TIME"
else BUILD_RET=$?
exit $BUILD_RET
fi fi
exit $BUILD_RET
else else
# additional permissions required due to flatpak's use of bubblewrap # additional permissions required due to flatpak's use of bubblewrap
docker run --cap-add=ALL --privileged \ docker run --cap-add=ALL --privileged \

View file

@ -0,0 +1,32 @@
#!/bin/bash
DBFILE="$1"
if [ "$DBFILE" == "" ]; then
echo "No database file provided!"
exit 1
fi
sqlite3 $DBFILE "create table if not exists FILES(NAME VARCHAR(1000) PRIMARY KEY, MD5 VARCHAR(32), OLD_MD5 VARCHAR(32) DEFAULT '-')"
for file in $(find . -type f | grep '\.c$\|\.cpp$\|\.h$\|\.hpp$\|\.ipp$\|\.tpp$\|\.ii' | cut -c 3-); do
newhash=`md5sum "$file" | cut -d" " -f1`
oldhash=`sqlite3 "$DBFILE" "select MD5 from FILES where NAME = '"$file"'"`
if [ "$oldhash" == "" ]; then
sqlite3 "$DBFILE" "insert into FILES(NAME, MD5) values('$file', '$newhash')"
printf '%-15s %-60s has %-32s\n' "new file:" "$file" "$newhash"
elif [ "$oldhash" != "$newhash" ]; then
sqlite3 "$DBFILE" "update FILES set OLD_MD5 = MD5, MD5 = '$newhash' where NAME = '$file'"
printf '%-15s %-60s had %-32s %-13s %-32s\n' "changed file:" "$file" "$oldhash" "and now has" "$newhash"
else
printf '%-15s %-60s had %-32s %-13s %-32s\n' "unchanged file:" "$file" "$oldhash" "and still has" "$newhash"
touch -d "20 years ago" "$file"
fi
done
for file in `sqlite3 "$DBFILE" "select NAME from FILES"`; do
file=`echo "$file" | tr -d '\r'`
if [ ! -f "$file" ]; then
printf '%-15s %-60s\n' "File removed:" "$file"
sqlite3 "$DBFILE" "delete from FILES where NAME = '$file'"
fi
done