Merge pull request #20552 from Microsoft/jjh/dockerfilereliability

Windows CI: Making dockerfile WAAAAAY faster

Dockerfile.windows

@@ -19,14 +19,7 @@
 # Important notes:
 # ---------------
 #
-# Multiple commands in a single powershell RUN command are deliberately not done. This is
-# because PS doesn't have a concept quite like set -e in bash. It would be possible to use 
-# try-catch script blocks, but that would make this file unreadable. The problem is that
-# if there are two commands eg "RUN powershell -command fail; succeed", as far as docker
-# would be concerned, the return code from the overall RUN is succeed. This doesn't apply to
-# RUN which uses cmd as the command interpreter such as "RUN fail; succeed".
-#
-# 'sleep 5' is a deliberate workaround for a current problem on containers in Windows 
+# 'Start-Sleep' is a deliberate workaround for a current problem on containers in Windows 
 # Server 2016. It ensures that the network is up and available for when the command is
 # network related. This bug is being tracked internally at Microsoft and exists in TP4.
 # Generally sleep 1 or 2 is probably enough, but making it 5 to make the build file
@@ -39,55 +32,70 @@
 # Don't try to use a volume for passing the source through. The cygwin posix utilities will
 # balk at reparse points. Again, see the example at the top of this file on how use a volume
 # to get the built binary out of the container.
+#
+# The steps are minimised dramatically to improve performance (TP4 is slow on commit)
 
 FROM windowsservercore
 
 # Environment variable notes:
-#  - GOLANG_VERSION should be updated to be consistent with the Linux dockerfile.
+#  - GOLANG_VERSION must consistent with 'Dockerfile' used by Linux'.
 #  - FROM_DOCKERFILE is used for detection of building within a container.
 ENV GOLANG_VERSION=1.5.3 \
-    GIT_VERSION=2.7.0 \
+    GIT_LOCATION=https://github.com/git-for-windows/git/releases/download/v2.7.1.windows.2/Git-2.7.1.2-64-bit.exe \
     RSRC_COMMIT=ba14da1f827188454a4591717fff29999010887f \
     GOPATH=C:/go;C:/go/src/github.com/docker/docker/vendor \
     FROM_DOCKERFILE=1
 
-# Make sure we're in temp for the downloads
-WORKDIR c:/windows/temp
-
-# Download everything else we need to install
-# We want a 64-bit make.exe, not 16 or 32-bit. This was hard to find, so documenting the links
-#  - http://sourceforge.net/p/mingw-w64/wiki2/Make/ -->
-#  - http://sourceforge.net/projects/mingw-w64/files/External%20binary%20packages%20%28Win64%20hosted%29/ -->
-#  - http://sourceforge.net/projects/mingw-w64/files/External binary packages %28Win64 hosted%29/make/
-RUN powershell -command sleep 5; Invoke-WebRequest -UserAgent 'DockerCI' -outfile make.zip http://downloads.sourceforge.net/project/mingw-w64/External%20binary%20packages%20%28Win64%20hosted%29/make/make-3.82.90-20111115.zip
-RUN powershell -command sleep 5; Invoke-WebRequest -UserAgent 'DockerCI' -outfile gcc.zip http://downloads.sourceforge.net/project/tdm-gcc/TDM-GCC%205%20series/5.1.0-tdm64-1/gcc-5.1.0-tdm64-1-core.zip 
-RUN powershell -command sleep 5; Invoke-WebRequest -UserAgent 'DockerCI' -outfile runtime.zip http://downloads.sourceforge.net/project/tdm-gcc/MinGW-w64%20runtime/GCC%205%20series/mingw64runtime-v4-git20150618-gcc5-tdm64-1.zip
-RUN powershell -command sleep 5; Invoke-WebRequest -UserAgent 'DockerCI' -outfile binutils.zip http://downloads.sourceforge.net/project/tdm-gcc/GNU%20binutils/binutils-2.25-tdm64-1.zip
-RUN powershell -command sleep 5; Invoke-WebRequest -UserAgent 'DockerCI' -outfile 7zsetup.exe http://www.7-zip.org/a/7z1514-x64.exe 
-RUN powershell -command sleep 5; Invoke-WebRequest -UserAgent 'DockerCI' -outfile lzma.7z http://www.7-zip.org/a/lzma1514.7z
-RUN powershell -command sleep 5; Invoke-WebRequest -UserAgent 'DockerCI' -outfile gitsetup.exe https://github.com/git-for-windows/git/releases/download/v%GIT_VERSION%.windows.1/Git-%GIT_VERSION%-64-bit.exe
-RUN powershell -command sleep 5; Invoke-WebRequest -UserAgent 'DockerCI' -outfile go.msi https://storage.googleapis.com/golang/go%GOLANG_VERSION%.windows-amd64.msi
-
-# Path
-RUN setx /M Path "c:\git\cmd;c:\git\bin;c:\git\usr\bin;%Path%;c:\gcc\bin;c:\7zip"
-
-# Install and expand the bits we downloaded. 
-# Note: The git, 7z and go.msi installers execute asynchronously. 
-RUN powershell -command start-process .\gitsetup.exe -ArgumentList '/VERYSILENT /SUPPRESSMSGBOXES /CLOSEAPPLICATIONS /DIR=c:\git' -Wait
-RUN powershell -command start-process .\7zsetup -ArgumentList '/S /D=c:/7zip' -Wait
-RUN powershell -command start-process .\go.msi -ArgumentList '/quiet' -Wait
-RUN powershell -command Expand-Archive gcc.zip \gcc -Force 
-RUN powershell -command Expand-Archive runtime.zip \gcc -Force 
-RUN powershell -command Expand-Archive binutils.zip \gcc -Force
-RUN powershell -command 7z e lzma.7z bin/lzma.exe
-RUN powershell -command 7z x make.zip  make-3.82.90-20111115/bin_amd64/make.exe 
-RUN powershell -command mv make-3.82.90-20111115/bin_amd64/make.exe /gcc/bin/ 
-
-# RSRC for manifest and icon	
-RUN powershell -command sleep 5 ; git clone https://github.com/akavel/rsrc.git c:\go\src\github.com\akavel\rsrc 
-RUN cd c:/go/src/github.com/akavel/rsrc && git checkout -q %RSRC_COMMIT% && go install -v
+WORKDIR c:/
 
+# Everything downloaded/installed in one go (better performance, esp on TP4)
+RUN \
+ setx /M Path "c:\git\cmd;c:\git\bin;c:\git\usr\bin;%Path%;c:\gcc\bin;c:\go\bin" && \
+ setx GOROOT "c:\go" && \
+ powershell -command \
+  $ErrorActionPreference = 'Stop'; \
+  Start-Sleep -Seconds 5; \
+  Function Download-File([string] $source, [string] $target) { \
+   $wc = New-Object net.webclient; $wc.Downloadfile($source, $target) \
+  } \
+  \
+  Write-Host INFO: Downloading git...; \		
+  Download-File %GIT_LOCATION% gitsetup.exe; \
+  \
+  Write-Host INFO: Downloading go...; \
+  Download-File https://storage.googleapis.com/golang/go%GOLANG_VERSION%.windows-amd64.msi go.msi; \
+  \
+  Write-Host INFO: Downloading compiler 1 of 3...; \
+  Download-File https://raw.githubusercontent.com/jhowardmsft/docker-tdmgcc/master/gcc.zip gcc.zip; \
+  \
+  Write-Host INFO: Downloading compiler 2 of 3...; \
+  Download-File https://raw.githubusercontent.com/jhowardmsft/docker-tdmgcc/master/runtime.zip runtime.zip; \
+  \
+  Write-Host INFO: Downloading compiler 3 of 3...; \
+  Download-File https://raw.githubusercontent.com/jhowardmsft/docker-tdmgcc/master/binutils.zip binutils.zip; \
+  \
+  Write-Host INFO: Installing git...; \
+  Start-Process gitsetup.exe -ArgumentList '/VERYSILENT /SUPPRESSMSGBOXES /CLOSEAPPLICATIONS /DIR=c:\git\' -Wait; \
+  \
+  Write-Host INFO: Installing go..."; \
+  Start-Process msiexec -ArgumentList '-i go.msi -quiet' -Wait; \
+  \
+  Write-Host INFO: Unzipping compiler...; \
+  c:\git\usr\bin\unzip.exe -q -o gcc.zip -d /c/gcc; \
+  c:\git\usr\bin\unzip.exe -q -o runtime.zip -d /c/gcc; \
+  c:\git\usr\bin\unzip.exe -q -o binutils.zip -d /c/gcc"; \
+  \
+  Write-Host INFO: Removing interim files; \
+  Remove-Item *.zip; \
+  Remove-Item go.msi; \
+  Remove-Item gitsetup.exe; \
+  \
+  Write-Host INFO: Cloning and installing RSRC; \
+  c:\git\bin\git.exe clone https://github.com/akavel/rsrc.git c:\go\src\github.com\akavel\rsrc; \
+  cd \go\src\github.com\akavel\rsrc; c:\git\bin\git.exe checkout -q %RSRC_COMMIT%; c:\go\bin\go.exe install -v; \
+  \
+  Write-Host INFO: Completed
+  
 # Prepare for building
-WORKDIR c:/
 COPY . /go/src/github.com/docker/docker