Build and use gotestsum for running all tests

1. Dockerfile.Windows modified to build gotestsum.exe

2. Use gotestsum.exe in invoking the execution of:

   (a) Unit tests (run in containers),
   (b) Integration tests (run outside containers)
   (c) Integration-cli (run outside containers)

No changes made to other categories of tests (e.g.
LCOW).

3. Copy .xml files produced by gotestsum in
   appropriate paths where Jenkins can ingest them

4. Modify Jenkinsfile to mark results output as
   being jUnit "type" as well as to archive the
   .xml test result files as artifacts.

Signed-off-by: Vikram bir Singh <vikrambir.singh@docker.com>
Signed-off-by: Sebastiaan van Stijn <github@gone.nl>

Dockerfile.windows

@@ -166,6 +166,7 @@ FROM microsoft/windowsservercore
 SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
 
 ARG GO_VERSION=1.13.4
+ARG GOTESTSUM_COMMIT=v0.3.5
 
 # Environment variable notes:
 #  - GO_VERSION must be consistent with 'Dockerfile' used by Linux.
@@ -174,7 +175,8 @@ ENV GO_VERSION=${GO_VERSION} `
     GIT_VERSION=2.11.1 `
     GOPATH=C:\gopath `
     GO111MODULE=off `
-    FROM_DOCKERFILE=1
+    FROM_DOCKERFILE=1 `
+    GOTESTSUM_COMMIT=${GOTESTSUM_COMMIT}
 
 RUN `
   Function Test-Nano() { `
@@ -249,13 +251,46 @@ RUN `
   Remove-Item C:\binutils.zip; `
   Remove-Item C:\gitsetup.zip; `
   `
-  Write-Host INFO: Creating source directory...; `
-  New-Item -ItemType Directory -Path ${GOPATH}\src\github.com\docker\docker | Out-Null; `
+  # Ensure all directories exist that we will require below....
+  $srcDir = """$Env:GOPATH`\src\github.com\docker\docker\bundles"""; `
+  Write-Host INFO: Ensuring existence of directory $srcDir...; `
+  New-Item -Force -ItemType Directory -Path $srcDir | Out-Null; `
   `
   Write-Host INFO: Configuring git core.autocrlf...; `
-  C:\git\cmd\git config --global core.autocrlf true; `
+  C:\git\cmd\git config --global core.autocrlf true;
+
+RUN `
+  Function Build-GoTestSum() { `
+    Write-Host "INFO: Building gotestsum version $Env:GOTESTSUM_COMMIT in $Env:GOPATH"; `
+    $optsForGet = @('"get"', '"-d"', '"gotest.tools/gotestsum"'); `
+    &go $optsForGet; `
+    $savedExitCode = $LASTEXITCODE; `
+    if ($savedExitCode -ne 0) {  `
+      Throw '"Failed getting gotestsum sources..."'  `
+    }; `
+    Write-Host "INFO:     Sources obtained for gotestsum..."; `
+    $GotestsumPath=Join-Path -Path $Env:GOPATH -ChildPath "src\gotest.tools\gotestsum"; `
+    Push-Location $GotestsumPath; `
+    $optsForCheckout = @('"checkout"', '"-q"', """$Env:GOTESTSUM_COMMIT"""); `
+    &git $optsForCheckout; `
+    $savedExitCode = $LASTEXITCODE; `
+    if ($savedExitCode -eq 0) { `
+      Write-Host "INFO:     Checkout done for gotestsum..."; `
+      $optsForBuild = @('"build"', '"-buildmode=exe"'); `
+      &go $optsForBuild; `
+      $savedExitCode = $LASTEXITCODE; `
+    } else { `
+      Throw '"gotestsum checkout failed..."'; `
+    } `
+    Pop-Location; `
+    `
+    if ($savedExitCode -ne 0) {  `
+      Throw '"gotestsum build failed..."'; `
+    } `
+    Write-Host "INFO:     Build done for gotestsum..."; `
+  } `
   `
-  Write-Host INFO: Completed
+  Build-GoTestSum
 
 # Make PowerShell the default entrypoint
 ENTRYPOINT ["powershell.exe"]

Jenkinsfile

@@ -928,13 +928,14 @@ pipeline {
                     }
                     post {
                         always {
+                            junit testResults: 'bundles/junit-report-*.xml', allowEmptyResults: true
                             catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') {
                                 powershell '''
                                 $bundleName="windowsRS1-integration"
                                 Write-Host -ForegroundColor Green "Creating ${bundleName}-bundles.zip"
 
                                 # archiveArtifacts does not support env-vars to , so save the artifacts in a fixed location
-                                Compress-Archive -Path "${env:TEMP}/CIDUT.out", "${env:TEMP}/CIDUT.err" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip"
+                                Compress-Archive -Path "${env:TEMP}/CIDUT.out", "${env:TEMP}/CIDUT.err", "${env:TEMP}/testresults/unittests/junit-report-unit-tests.xml" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip"
                                 '''
 
                                 archiveArtifacts artifacts: '*-bundles.zip', allowEmptyArchive: true
@@ -988,13 +989,14 @@ pipeline {
                     }
                     post {
                         always {
+                            junit testResults: 'bundles/junit-report-*.xml', allowEmptyResults: true
                             catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') {
                                 powershell '''
                                 $bundleName="windowsRS5-integration"
                                 Write-Host -ForegroundColor Green "Creating ${bundleName}-bundles.zip"
 
                                 # archiveArtifacts does not support env-vars to , so save the artifacts in a fixed location
-                                Compress-Archive -Path "${env:TEMP}/CIDUT.out", "${env:TEMP}/CIDUT.err" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip"
+                                Compress-Archive -Path "${env:TEMP}/CIDUT.out", "${env:TEMP}/CIDUT.err", "${env:TEMP}/junit-report-*.xml" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip"
                                 '''
 
                                 archiveArtifacts artifacts: '*-bundles.zip', allowEmptyArchive: true

hack/ci/windows.ps1

@@ -447,6 +447,8 @@ Try {
     $errorActionPreference='Stop'
     New-Item -ItemType Directory "$env:TEMP" -ErrorAction SilentlyContinue | Out-Null
     New-Item -ItemType Directory "$env:TEMP\userprofile" -ErrorAction SilentlyContinue  | Out-Null
+    New-Item -ItemType Directory "$env:TEMP\testresults" -ErrorAction SilentlyContinue  | Out-Null
+    New-Item -ItemType Directory "$env:TEMP\testresults\unittests" -ErrorAction SilentlyContinue  | Out-Null
     New-Item -ItemType Directory "$env:TEMP\localappdata" -ErrorAction SilentlyContinue | Out-Null
     New-Item -ItemType Directory "$env:TEMP\binary" -ErrorAction SilentlyContinue | Out-Null
     New-Item -ItemType Directory "$env:TEMP\installer" -ErrorAction SilentlyContinue | Out-Null
@@ -525,6 +527,16 @@ Try {
         if (-not($LastExitCode -eq 0)) {
             Throw "ERROR: Failed to docker cp the daemon binary (dockerd.exe) to $env:TEMP\binary"
         }
+
+        $GotestsumBinRelLocationInContainer="src\gotest.tools\gotestsum\gotestsum.exe"
+        if (-not($LastExitCode -eq 0)) {
+            Throw "ERROR: Failed to docker cp the gotestsum binary (gotestsum.exe) to $env:TEMP\binary"
+        }
+        docker cp "$COMMITHASH`:c`:\gopath\$GotestsumBinRelLocationInContainer" $env:TEMP\binary\
+        if (-not (Test-Path "$env:TEMP\binary\gotestsum.exe")) {
+            Throw "ERROR: gotestsum.exe not found...." `
+        }
+
         $ErrorActionPreference = "Stop"
 
         # Copy the built dockerd.exe to dockerd-$COMMITHASH.exe so that easily spotted in task manager.
@@ -774,11 +786,36 @@ Try {
     # Run the unit tests inside a container unless SKIP_UNIT_TESTS is defined
     if (($null -eq $env:LCOW_MODE) -and ($null -eq $env:LCOW_BASIC_MODE)) {
         if ($null -eq $env:SKIP_UNIT_TESTS) {
+            $ContainerNameForUnitTests = $COMMITHASH + "_UnitTests"
             Write-Host -ForegroundColor Cyan "INFO: Running unit tests at $(Get-Date)..."
             $ErrorActionPreference = "SilentlyContinue"
-            $Duration=$(Measure-Command {docker run -e DOCKER_GITCOMMIT=$COMMITHASH$CommitUnsupported docker hack\make.ps1 -TestUnit | Out-Host })
+            $Duration=$(Measure-Command {docker run --name $ContainerNameForUnitTests -e DOCKER_GITCOMMIT=$COMMITHASH$CommitUnsupported docker hack\make.ps1 -TestUnit | Out-Host })
+            $TestRunExitCode = $LastExitCode
             $ErrorActionPreference = "Stop"
+
+            # Saving for artifacts......
+            $unitTestsContPath="$ContainerNameForUnitTests`:c`:\gopath\src\github.com\docker\docker\bundles"
+            $JunitExpectedContFilePath = "$unitTestsContPath\junit-report-unit-tests.xml"
+            docker cp $JunitExpectedContFilePath "$TEMPORIG"
+            if (-not($LastExitCode -eq 0)) {
+                Throw "ERROR: Failed to docker cp the unit tests report ($JunitExpectedContFilePath) to $TEMPORIG"
+            }
+
+            if (Test-Path "$TEMPORIG\junit-report-unit-tests.xml") {
+                Write-Host -ForegroundColor Magenta "INFO: Unit tests results($TEMPORIG\junit-report-unit-tests.xml) exist. pwd=$pwd"
+            } else {
+                Write-Host -ForegroundColor Magenta "ERROR: Unit tests results($TEMPORIG\junit-report-unit-tests.xml) do not exist. pwd=$pwd"
+            }
+            
+            # Saving where jenkins will take a look at.....
+            $bundlesDir = "bundles"
+            New-Item -Force -ItemType Directory $bundlesDir | Out-Null
+            docker cp $JunitExpectedContFilePath "$bundlesDir"
             if (-not($LastExitCode -eq 0)) {
+                Throw "ERROR: Failed to docker cp the unit tests report ($JunitExpectedContFilePath) to $bundlesDir"
+            }
+
+            if (-not($TestRunExitCode -eq 0)) {
                 Throw "ERROR: Unit tests failed"
             }
             Write-Host  -ForegroundColor Green "INFO: Unit tests ended at $(Get-Date). Duration`:$Duration"
@@ -830,7 +867,9 @@ Try {
             $env:OrigDOCKER_HOST="$env:DOCKER_HOST"
     
             #https://blogs.technet.microsoft.com/heyscriptingguy/2011/09/20/solve-problems-with-external-command-lines-in-powershell/ is useful to see tokenising
-            $c = "go test "
+            $jsonFilePath = "..\\bundles\\go-test-report-intcli-tests.json"
+            $xmlFilePath = "..\\bundles\\junit-report-intcli-tests.xml"
+            $c = "gotestsum --format=standard-quiet --jsonfile=$jsonFilePath --junitfile=$xmlFilePath -- "
             $c += "`"-test.v`" "
             if ($null -ne $env:INTEGRATION_TEST_NAME) { # Makes is quicker for debugging to be able to run only a subset of the integration tests
                 $c += "`"-test.run`" "
@@ -864,10 +903,12 @@ Try {
                 if (!($env:INTEGRATION_TESTFLAGS)) {
                     $env:INTEGRATION_TESTFLAGS = "-test.v"
                 }
-                Set-Location "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker"
                 $start=(Get-Date); Invoke-Expression ".\hack\make.ps1 -TestIntegration"; $Duration=New-Timespan -Start $start -End (Get-Date)
+                $IntTestsRunResult = $LastExitCode
                 $ErrorActionPreference = "Stop"
-                if (-not($LastExitCode -eq 0)) {
+                # Copy all the test results to TEMPORIG for archival
+                Copy-Item -Path "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker\bundles\junit-report*xml" -Destination $TEMPORIG
+                if (-not($IntTestsRunResult -eq 0)) {
                     Throw "ERROR: Integration API tests failed at $(Get-Date). Duration`:$Duration"
                 }
 
@@ -877,6 +918,7 @@ Try {
                 Set-Location "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker\integration-cli"
                 # Explicit to not use measure-command otherwise don't get output as it goes
                 $start=(Get-Date); Invoke-Expression $c; $Duration=New-Timespan -Start $start -End (Get-Date)
+                Copy-Item -Path $xmlFilePath -Destination $TEMPORIG
             }
             $ErrorActionPreference = "Stop"
             if (-not($LastExitCode -eq 0)) {

hack/make.ps1

@@ -94,6 +94,7 @@ param(
 $ErrorActionPreference = "Stop"
 $ProgressPreference = "SilentlyContinue"
 $pushed=$False  # To restore the directory if we have temporarily pushed to one.
+Set-Variable GOTESTSUM_LOCATION -option Constant -value "$env:GOPATH/src/gotest.tools/gotestsum"
 
 # Utility function to get the commit ID of the repository
 Function Get-GitCommit() {
@@ -319,31 +320,42 @@ Function Run-UnitTests() {
     $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/man"
     $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/integration"
     $pkgList = $pkgList -replace "`r`n", " "
-    $goTestCommand = "go test" + $raceParm + " -cover -ldflags -w -tags """ + "autogen daemon" + """ -a """ + "-test.timeout=10m" + """ $pkgList"
+    $goTestCommand = "$GOTESTSUM_LOCATION\gotestsum.exe --format=standard-quiet --jsonfile=bundles\go-test-report-unit-tests.json --junitfile=bundles\junit-report-unit-tests.xml -- " + $raceParm + " -cover -ldflags -w -a """ + "-test.timeout=10m" + """ $pkgList"
+    Write-Host "INFO: Invoking unit tests run with $goTestCommand"
     Invoke-Expression $goTestCommand
     if ($LASTEXITCODE -ne 0) { Throw "Unit tests failed" }
 }
 
 # Run the integration tests
 Function Run-IntegrationTests() {
-    $env:DOCKER_INTEGRATION_DAEMON_DEST = $root + "\bundles\tmp"
+    $escRoot = [Regex]::Escape($root)
+    $env:DOCKER_INTEGRATION_DAEMON_DEST = $bundlesDir + "\tmp"
     $dirs = go list -test -f '{{- if ne .ForTest `"`" -}}{{- .Dir -}}{{- end -}}' .\integration\...
-    $integration_api_dirs = @()
     ForEach($dir in $dirs) {
-        $integration_api_dirs += $dir
-        Write-Host "Building test suite binary $dir"
-        go test -c -o "$dir\test.exe" $dir
-    }
-
-    ForEach($dir in $integration_api_dirs) {
+        # Normalize directory name for using in the test results files.
+        $normDir = $dir.Trim()
+        $normDir = $normDir -replace $escRoot, ""
+        $normDir = $normDir -replace "\\", "-"
+        $normDir = $normDir -replace "\/", "-"
+        $normDir = $normDir -replace "\.", "-"
+        if ($normDir.StartsWith("-"))
+        {
+            $normDir = $normDir.TrimStart("-")
+        }
+        if ($normDir.EndsWith("-"))
+        {
+            $normDir = $normDir.TrimEnd("-")
+        }
+        $jsonFilePath = $bundlesDir + "\go-test-report-int-tests-$normDir" + ".json"
+        $xmlFilePath = $bundlesDir + "\junit-report-int-tests-$normDir" + ".xml"
         Set-Location $dir
         Write-Host "Running $($PWD.Path)"
         $pinfo = New-Object System.Diagnostics.ProcessStartInfo
-        $pinfo.FileName = "$($PWD.Path)\test.exe"
+        $pinfo.FileName = "gotestsum.exe"
         $pinfo.WorkingDirectory = "$($PWD.Path)"
         $pinfo.RedirectStandardError = $true
         $pinfo.UseShellExecute = $false
-        $pinfo.Arguments = $env:INTEGRATION_TESTFLAGS
+        $pinfo.Arguments = "--format=standard-quiet --jsonfile=$jsonFilePath --junitfile=$xmlFilePath -- $env:INTEGRATION_TESTFLAGS"
         $p = New-Object System.Diagnostics.Process
         $p.StartInfo = $pinfo
         $p.Start() | Out-Null
@@ -351,6 +363,8 @@ Function Run-IntegrationTests() {
         $err = $p.StandardError.ReadToEnd()
         if (($LASTEXITCODE -ne 0) -and ($err -notlike "*warning: no tests to run*")) {
             Throw "Integration tests failed: $err"
+        } else {
+            Write-Host "$err"
         }
     }
 }
@@ -363,6 +377,11 @@ Try {
     $root = $(Split-Path $MyInvocation.MyCommand.Definition -Parent | Split-Path -Parent)
     Push-Location $root
 
+    # Ensure the bundles directory exists
+    $bundlesDir = $root + "\bundles"
+    Set-Variable bundlesDir -option ReadOnly
+    New-Item -Force $bundlesDir -ItemType Directory | Out-Null
+
     # Handle the "-All" shortcut to turn on all things we can handle.
     # Note we expressly only include the items which can run in a container - the validations tests cannot
     # as they require the .git directory which is excluded from the image by .dockerignore
@@ -424,8 +443,6 @@ Try {
 
     # Build the binaries
     if ($Client -or $Daemon) {
-        # Create the bundles directory if it doesn't exist
-        if (-not (Test-Path ".\bundles")) { New-Item ".\bundles" -ItemType Directory | Out-Null }
 
         # Perform the actual build
         if ($Daemon) { Execute-Build "daemon" "daemon" "dockerd" }