Windows CI: Make sure that CI fails on any error

- If unit tests fails
- If intergration tests fails

Signed-off-by: Olli Janatuinen <olli.janatuinen@gmail.com>

Dockerfile.windows

@@ -166,7 +166,7 @@ FROM microsoft/windowsservercore
 SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
 
 ARG GO_VERSION=1.13.15
-ARG GOTESTSUM_COMMIT=v0.3.5
+ARG GOTESTSUM_COMMIT=v0.5.3
 
 # Environment variable notes:
 #  - GO_VERSION must be consistent with 'Dockerfile' used by Linux.

Jenkinsfile

@@ -1014,11 +1014,12 @@ pipeline {
                             junit testResults: 'bundles/junit-report-*.xml', allowEmptyResults: true
                             catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') {
                                 powershell '''
+                                cd $env:WORKSPACE
                                 $bundleName="windowsRS1-integration"
                                 Write-Host -ForegroundColor Green "Creating ${bundleName}-bundles.zip"
 
                                 # archiveArtifacts does not support env-vars to , so save the artifacts in a fixed location
-                                Compress-Archive -Path "${env:TEMP}/CIDUT.out", "${env:TEMP}/CIDUT.err", "${env:TEMP}/testresults/unittests/junit-report-unit-tests.xml" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip"
+                                Compress-Archive -Path "bundles/CIDUT.out", "bundles/CIDUT.err", "bundles/junit-report-*.xml" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip"
                                 '''
 
                                 archiveArtifacts artifacts: '*-bundles.zip', allowEmptyArchive: true
@@ -1075,11 +1076,12 @@ pipeline {
                             junit testResults: 'bundles/junit-report-*.xml', allowEmptyResults: true
                             catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') {
                                 powershell '''
+                                cd $env:WORKSPACE
                                 $bundleName="windowsRS5-integration"
                                 Write-Host -ForegroundColor Green "Creating ${bundleName}-bundles.zip"
 
                                 # archiveArtifacts does not support env-vars to , so save the artifacts in a fixed location
-                                Compress-Archive -Path "${env:TEMP}/CIDUT.out", "${env:TEMP}/CIDUT.err", "${env:TEMP}/junit-report-*.xml" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip"
+                                Compress-Archive -Path "bundles/CIDUT.out", "bundles/CIDUT.err", "bundles/junit-report-*.xml" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip"
                                 '''
 
                                 archiveArtifacts artifacts: '*-bundles.zip', allowEmptyArchive: true

hack/ci/windows.ps1

@@ -789,26 +789,19 @@ Try {
             $TestRunExitCode = $LastExitCode
             $ErrorActionPreference = "Stop"
 
-            # Saving for artifacts......
+            # Saving where jenkins will take a look at.....
+            New-Item -Force -ItemType Directory bundles | Out-Null
             $unitTestsContPath="$ContainerNameForUnitTests`:c`:\gopath\src\github.com\docker\docker\bundles"
             $JunitExpectedContFilePath = "$unitTestsContPath\junit-report-unit-tests.xml"
-            docker cp $JunitExpectedContFilePath "$TEMPORIG"
+            docker cp $JunitExpectedContFilePath "bundles"
             if (-not($LastExitCode -eq 0)) {
-                Throw "ERROR: Failed to docker cp the unit tests report ($JunitExpectedContFilePath) to $TEMPORIG"
+                Throw "ERROR: Failed to docker cp the unit tests report ($JunitExpectedContFilePath) to bundles"
             }
 
-            if (Test-Path "$TEMPORIG\junit-report-unit-tests.xml") {
-                Write-Host -ForegroundColor Magenta "INFO: Unit tests results($TEMPORIG\junit-report-unit-tests.xml) exist. pwd=$pwd"
+            if (Test-Path "bundles\junit-report-unit-tests.xml") {
+                Write-Host -ForegroundColor Magenta "INFO: Unit tests results(bundles\junit-report-unit-tests.xml) exist. pwd=$pwd"
             } else {
-                Write-Host -ForegroundColor Magenta "ERROR: Unit tests results($TEMPORIG\junit-report-unit-tests.xml) do not exist. pwd=$pwd"
-            }
-            
-            # Saving where jenkins will take a look at.....
-            $bundlesDir = "bundles"
-            New-Item -Force -ItemType Directory $bundlesDir | Out-Null
-            docker cp $JunitExpectedContFilePath "$bundlesDir"
-            if (-not($LastExitCode -eq 0)) {
-                Throw "ERROR: Failed to docker cp the unit tests report ($JunitExpectedContFilePath) to $bundlesDir"
+                Write-Host -ForegroundColor Magenta "ERROR: Unit tests results(bundles\junit-report-unit-tests.xml) do not exist. pwd=$pwd"
             }
 
             if (-not($TestRunExitCode -eq 0)) {
@@ -855,8 +848,7 @@ Try {
             #https://blogs.technet.microsoft.com/heyscriptingguy/2011/09/20/solve-problems-with-external-command-lines-in-powershell/ is useful to see tokenising
             $jsonFilePath = "..\\bundles\\go-test-report-intcli-tests.json"
             $xmlFilePath = "..\\bundles\\junit-report-intcli-tests.xml"
-            $c = "gotestsum --format=standard-quiet --jsonfile=$jsonFilePath --junitfile=$xmlFilePath -- "
-            $c += "`"-test.v`" "
+            $c = "gotestsum --format=standard-verbose --jsonfile=$jsonFilePath --junitfile=$xmlFilePath -- "
             if ($null -ne $env:INTEGRATION_TEST_NAME) { # Makes is quicker for debugging to be able to run only a subset of the integration tests
                 $c += "`"-test.run`" "
                 $c += "`"$env:INTEGRATION_TEST_NAME`" "
@@ -886,14 +878,9 @@ Try {
 
                 $ErrorActionPreference = "SilentlyContinue"
                 Write-Host -ForegroundColor Cyan "INFO: Integration API tests being run from the host:"
-                if (!($env:INTEGRATION_TESTFLAGS)) {
-                    $env:INTEGRATION_TESTFLAGS = "-test.v"
-                }
                 $start=(Get-Date); Invoke-Expression ".\hack\make.ps1 -TestIntegration"; $Duration=New-Timespan -Start $start -End (Get-Date)
                 $IntTestsRunResult = $LastExitCode
                 $ErrorActionPreference = "Stop"
-                # Copy all the test results to TEMPORIG for archival
-                Copy-Item -Path "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker\bundles\junit-report*xml" -Destination $TEMPORIG
                 if (-not($IntTestsRunResult -eq 0)) {
                     Throw "ERROR: Integration API tests failed at $(Get-Date). Duration`:$Duration"
                 }
@@ -904,7 +891,6 @@ Try {
                 Set-Location "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker\integration-cli"
                 # Explicit to not use measure-command otherwise don't get output as it goes
                 $start=(Get-Date); Invoke-Expression $c; $Duration=New-Timespan -Start $start -End (Get-Date)
-                Copy-Item -Path $xmlFilePath -Destination $TEMPORIG
             }
             $ErrorActionPreference = "Stop"
             if (-not($LastExitCode -eq 0)) {
@@ -1052,10 +1038,10 @@ Finally {
 
     # Save the daemon under test log
     if ($daemonStarted -eq 1) {
-        Write-Host -ForegroundColor Green "INFO: Saving daemon under test log ($env:TEMP\dut.out) to $TEMPORIG\CIDUT.out"
-        Copy-Item  "$env:TEMP\dut.out" "$TEMPORIG\CIDUT.out" -Force -ErrorAction SilentlyContinue
-        Write-Host -ForegroundColor Green "INFO: Saving daemon under test log ($env:TEMP\dut.err) to $TEMPORIG\CIDUT.err"
-        Copy-Item  "$env:TEMP\dut.err" "$TEMPORIG\CIDUT.err" -Force -ErrorAction SilentlyContinue
+        Write-Host -ForegroundColor Green "INFO: Saving daemon under test log ($env:TEMP\dut.out) to bundles\CIDUT.out"
+        Copy-Item  "$env:TEMP\dut.out" "bundles\CIDUT.out" -Force -ErrorAction SilentlyContinue
+        Write-Host -ForegroundColor Green "INFO: Saving daemon under test log ($env:TEMP\dut.err) to bundles\CIDUT.err"
+        Copy-Item  "$env:TEMP\dut.err" "bundles\CIDUT.err" -Force -ErrorAction SilentlyContinue
     }
 
     Set-Location "$env:SOURCES_DRIVE\$env:SOURCES_SUBDIR" -ErrorAction SilentlyContinue

hack/dockerfile/install/gotestsum.installer

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-: ${GOTESTSUM_COMMIT:=v0.3.5}
+: ${GOTESTSUM_COMMIT:=v0.5.3}
 
 install_gotestsum() (
 	set -e

hack/make.ps1

@@ -320,10 +320,19 @@ Function Run-UnitTests() {
     $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/man"
     $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/integration"
     $pkgList = $pkgList -replace "`r`n", " "
-    $goTestCommand = "$GOTESTSUM_LOCATION\gotestsum.exe --format=standard-quiet --jsonfile=bundles\go-test-report-unit-tests.json --junitfile=bundles\junit-report-unit-tests.xml -- " + $raceParm + " -cover -ldflags -w -a """ + "-test.timeout=10m" + """ $pkgList"
-    Write-Host "INFO: Invoking unit tests run with $goTestCommand"
-    Invoke-Expression $goTestCommand
-    if ($LASTEXITCODE -ne 0) { Throw "Unit tests failed" }
+
+    $goTestArg = "--format=standard-verbose --jsonfile=bundles\go-test-report-unit-tests.json --junitfile=bundles\junit-report-unit-tests.xml -- " + $raceParm + " -cover -ldflags -w -a """ + "-test.timeout=10m" + """ $pkgList"
+    Write-Host "INFO: Invoking unit tests run with $GOTESTSUM_LOCATION\gotestsum.exe $goTestArg"
+    $pinfo = New-Object System.Diagnostics.ProcessStartInfo
+    $pinfo.FileName = "$GOTESTSUM_LOCATION\gotestsum.exe"
+    $pinfo.WorkingDirectory = "$($PWD.Path)"
+    $pinfo.UseShellExecute = $false
+    $pinfo.Arguments = $goTestArg
+    $p = New-Object System.Diagnostics.Process
+    $p.StartInfo = $pinfo
+    $p.Start() | Out-Null
+    $p.WaitForExit()
+    if ($p.ExitCode -ne 0) { Throw "Unit tests failed" }
 }
 
 # Run the integration tests
@@ -353,19 +362,13 @@ Function Run-IntegrationTests() {
         $pinfo = New-Object System.Diagnostics.ProcessStartInfo
         $pinfo.FileName = "gotestsum.exe"
         $pinfo.WorkingDirectory = "$($PWD.Path)"
-        $pinfo.RedirectStandardError = $true
         $pinfo.UseShellExecute = $false
-        $pinfo.Arguments = "--format=standard-quiet --jsonfile=$jsonFilePath --junitfile=$xmlFilePath -- $env:INTEGRATION_TESTFLAGS"
+        $pinfo.Arguments = "--format=standard-verbose --jsonfile=$jsonFilePath --junitfile=$xmlFilePath -- $env:INTEGRATION_TESTFLAGS"
         $p = New-Object System.Diagnostics.Process
         $p.StartInfo = $pinfo
         $p.Start() | Out-Null
         $p.WaitForExit()
-        $err = $p.StandardError.ReadToEnd()
-        if (($LASTEXITCODE -ne 0) -and ($err -notlike "*warning: no tests to run*")) {
-            Throw "Integration tests failed: $err"
-        } else {
-            Write-Host "$err"
-        }
+        if ($p.ExitCode -ne 0) { Throw "Integration tests failed" }
     }
 }
 
@@ -506,7 +509,7 @@ Catch [Exception] {
     Write-Host -ForegroundColor Red  "     \/        \/             \/     \/ "
     Write-Host
 
-    Throw $_
+    exit 1
 }
 Finally {
     Pop-Location # As we pushed to the root of the repo as the very first thing