From 8d2e1ee5467b7b0d8c917260adf7b08bd09ff476 Mon Sep 17 00:00:00 2001 From: Vikram bir Singh Date: Tue, 8 Oct 2019 01:14:51 +0000 Subject: [PATCH] Build and use gotestsum for running all tests 1. Dockerfile.Windows modified to build gotestsum.exe 2. Use gotestsum.exe in invoking the execution of: (a) Unit tests (run in containers), (b) Integration tests (run outside containers) (c) Integration-cli (run outside containers) No changes made to other categories of tests (e.g. LCOW). 3. Copy .xml files produced by gotestsum in appropriate paths where Jenkins can ingest them 4. Modify Jenkinsfile to mark results output as being jUnit "type" as well as to archive the .xml test result files as artifacts. Signed-off-by: Vikram bir Singh Signed-off-by: Sebastiaan van Stijn --- Dockerfile.windows | 45 +++++++++++++++++++++++++++++++++++----- Jenkinsfile | 6 ++++-- hack/ci/windows.ps1 | 50 +++++++++++++++++++++++++++++++++++++++++---- hack/make.ps1 | 43 ++++++++++++++++++++++++++------------ 4 files changed, 120 insertions(+), 24 deletions(-) diff --git a/Dockerfile.windows b/Dockerfile.windows index 3043e1619e453cba83824297c63729cea5efcdea..95d6a89ea4288389716ebd80d5fb5d2b8736d19e 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -166,6 +166,7 @@ FROM microsoft/windowsservercore SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] ARG GO_VERSION=1.13.4 +ARG GOTESTSUM_COMMIT=v0.3.5 # Environment variable notes: # - GO_VERSION must be consistent with 'Dockerfile' used by Linux. @@ -174,7 +175,8 @@ ENV GO_VERSION=${GO_VERSION} ` GIT_VERSION=2.11.1 ` GOPATH=C:\gopath ` GO111MODULE=off ` - FROM_DOCKERFILE=1 + FROM_DOCKERFILE=1 ` + GOTESTSUM_COMMIT=${GOTESTSUM_COMMIT} RUN ` Function Test-Nano() { ` @@ -249,13 +251,46 @@ RUN ` Remove-Item C:\binutils.zip; ` Remove-Item C:\gitsetup.zip; ` ` - Write-Host INFO: Creating source directory...; ` - New-Item -ItemType Directory -Path ${GOPATH}\src\github.com\docker\docker | Out-Null; ` + # Ensure all directories exist that we will require below.... + $srcDir = """$Env:GOPATH`\src\github.com\docker\docker\bundles"""; ` + Write-Host INFO: Ensuring existence of directory $srcDir...; ` + New-Item -Force -ItemType Directory -Path $srcDir | Out-Null; ` ` Write-Host INFO: Configuring git core.autocrlf...; ` - C:\git\cmd\git config --global core.autocrlf true; ` + C:\git\cmd\git config --global core.autocrlf true; + +RUN ` + Function Build-GoTestSum() { ` + Write-Host "INFO: Building gotestsum version $Env:GOTESTSUM_COMMIT in $Env:GOPATH"; ` + $optsForGet = @('"get"', '"-d"', '"gotest.tools/gotestsum"'); ` + &go $optsForGet; ` + $savedExitCode = $LASTEXITCODE; ` + if ($savedExitCode -ne 0) { ` + Throw '"Failed getting gotestsum sources..."' ` + }; ` + Write-Host "INFO: Sources obtained for gotestsum..."; ` + $GotestsumPath=Join-Path -Path $Env:GOPATH -ChildPath "src\gotest.tools\gotestsum"; ` + Push-Location $GotestsumPath; ` + $optsForCheckout = @('"checkout"', '"-q"', """$Env:GOTESTSUM_COMMIT"""); ` + &git $optsForCheckout; ` + $savedExitCode = $LASTEXITCODE; ` + if ($savedExitCode -eq 0) { ` + Write-Host "INFO: Checkout done for gotestsum..."; ` + $optsForBuild = @('"build"', '"-buildmode=exe"'); ` + &go $optsForBuild; ` + $savedExitCode = $LASTEXITCODE; ` + } else { ` + Throw '"gotestsum checkout failed..."'; ` + } ` + Pop-Location; ` + ` + if ($savedExitCode -ne 0) { ` + Throw '"gotestsum build failed..."'; ` + } ` + Write-Host "INFO: Build done for gotestsum..."; ` + } ` ` - Write-Host INFO: Completed + Build-GoTestSum # Make PowerShell the default entrypoint ENTRYPOINT ["powershell.exe"] diff --git a/Jenkinsfile b/Jenkinsfile index cdb27e3e377a1996c004f810c15116136df4209f..f3b7ab9f1304dbd690283df2f5409309306fac2f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -928,13 +928,14 @@ pipeline { } post { always { + junit testResults: 'bundles/junit-report-*.xml', allowEmptyResults: true catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') { powershell ''' $bundleName="windowsRS1-integration" Write-Host -ForegroundColor Green "Creating ${bundleName}-bundles.zip" # archiveArtifacts does not support env-vars to , so save the artifacts in a fixed location - Compress-Archive -Path "${env:TEMP}/CIDUT.out", "${env:TEMP}/CIDUT.err" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip" + Compress-Archive -Path "${env:TEMP}/CIDUT.out", "${env:TEMP}/CIDUT.err", "${env:TEMP}/testresults/unittests/junit-report-unit-tests.xml" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip" ''' archiveArtifacts artifacts: '*-bundles.zip', allowEmptyArchive: true @@ -988,13 +989,14 @@ pipeline { } post { always { + junit testResults: 'bundles/junit-report-*.xml', allowEmptyResults: true catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') { powershell ''' $bundleName="windowsRS5-integration" Write-Host -ForegroundColor Green "Creating ${bundleName}-bundles.zip" # archiveArtifacts does not support env-vars to , so save the artifacts in a fixed location - Compress-Archive -Path "${env:TEMP}/CIDUT.out", "${env:TEMP}/CIDUT.err" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip" + Compress-Archive -Path "${env:TEMP}/CIDUT.out", "${env:TEMP}/CIDUT.err", "${env:TEMP}/junit-report-*.xml" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip" ''' archiveArtifacts artifacts: '*-bundles.zip', allowEmptyArchive: true diff --git a/hack/ci/windows.ps1 b/hack/ci/windows.ps1 index 3bf951a4cb327cee2a108b0f536ce06aefe6aa20..554940d1df50d52addbe8cc2bdbe4e25dc24bffe 100644 --- a/hack/ci/windows.ps1 +++ b/hack/ci/windows.ps1 @@ -447,6 +447,8 @@ Try { $errorActionPreference='Stop' New-Item -ItemType Directory "$env:TEMP" -ErrorAction SilentlyContinue | Out-Null New-Item -ItemType Directory "$env:TEMP\userprofile" -ErrorAction SilentlyContinue | Out-Null + New-Item -ItemType Directory "$env:TEMP\testresults" -ErrorAction SilentlyContinue | Out-Null + New-Item -ItemType Directory "$env:TEMP\testresults\unittests" -ErrorAction SilentlyContinue | Out-Null New-Item -ItemType Directory "$env:TEMP\localappdata" -ErrorAction SilentlyContinue | Out-Null New-Item -ItemType Directory "$env:TEMP\binary" -ErrorAction SilentlyContinue | Out-Null New-Item -ItemType Directory "$env:TEMP\installer" -ErrorAction SilentlyContinue | Out-Null @@ -525,6 +527,16 @@ Try { if (-not($LastExitCode -eq 0)) { Throw "ERROR: Failed to docker cp the daemon binary (dockerd.exe) to $env:TEMP\binary" } + + $GotestsumBinRelLocationInContainer="src\gotest.tools\gotestsum\gotestsum.exe" + if (-not($LastExitCode -eq 0)) { + Throw "ERROR: Failed to docker cp the gotestsum binary (gotestsum.exe) to $env:TEMP\binary" + } + docker cp "$COMMITHASH`:c`:\gopath\$GotestsumBinRelLocationInContainer" $env:TEMP\binary\ + if (-not (Test-Path "$env:TEMP\binary\gotestsum.exe")) { + Throw "ERROR: gotestsum.exe not found...." ` + } + $ErrorActionPreference = "Stop" # Copy the built dockerd.exe to dockerd-$COMMITHASH.exe so that easily spotted in task manager. @@ -774,11 +786,36 @@ Try { # Run the unit tests inside a container unless SKIP_UNIT_TESTS is defined if (($null -eq $env:LCOW_MODE) -and ($null -eq $env:LCOW_BASIC_MODE)) { if ($null -eq $env:SKIP_UNIT_TESTS) { + $ContainerNameForUnitTests = $COMMITHASH + "_UnitTests" Write-Host -ForegroundColor Cyan "INFO: Running unit tests at $(Get-Date)..." $ErrorActionPreference = "SilentlyContinue" - $Duration=$(Measure-Command {docker run -e DOCKER_GITCOMMIT=$COMMITHASH$CommitUnsupported docker hack\make.ps1 -TestUnit | Out-Host }) + $Duration=$(Measure-Command {docker run --name $ContainerNameForUnitTests -e DOCKER_GITCOMMIT=$COMMITHASH$CommitUnsupported docker hack\make.ps1 -TestUnit | Out-Host }) + $TestRunExitCode = $LastExitCode $ErrorActionPreference = "Stop" + + # Saving for artifacts...... + $unitTestsContPath="$ContainerNameForUnitTests`:c`:\gopath\src\github.com\docker\docker\bundles" + $JunitExpectedContFilePath = "$unitTestsContPath\junit-report-unit-tests.xml" + docker cp $JunitExpectedContFilePath "$TEMPORIG" + if (-not($LastExitCode -eq 0)) { + Throw "ERROR: Failed to docker cp the unit tests report ($JunitExpectedContFilePath) to $TEMPORIG" + } + + if (Test-Path "$TEMPORIG\junit-report-unit-tests.xml") { + Write-Host -ForegroundColor Magenta "INFO: Unit tests results($TEMPORIG\junit-report-unit-tests.xml) exist. pwd=$pwd" + } else { + Write-Host -ForegroundColor Magenta "ERROR: Unit tests results($TEMPORIG\junit-report-unit-tests.xml) do not exist. pwd=$pwd" + } + + # Saving where jenkins will take a look at..... + $bundlesDir = "bundles" + New-Item -Force -ItemType Directory $bundlesDir | Out-Null + docker cp $JunitExpectedContFilePath "$bundlesDir" if (-not($LastExitCode -eq 0)) { + Throw "ERROR: Failed to docker cp the unit tests report ($JunitExpectedContFilePath) to $bundlesDir" + } + + if (-not($TestRunExitCode -eq 0)) { Throw "ERROR: Unit tests failed" } Write-Host -ForegroundColor Green "INFO: Unit tests ended at $(Get-Date). Duration`:$Duration" @@ -830,7 +867,9 @@ Try { $env:OrigDOCKER_HOST="$env:DOCKER_HOST" #https://blogs.technet.microsoft.com/heyscriptingguy/2011/09/20/solve-problems-with-external-command-lines-in-powershell/ is useful to see tokenising - $c = "go test " + $jsonFilePath = "..\\bundles\\go-test-report-intcli-tests.json" + $xmlFilePath = "..\\bundles\\junit-report-intcli-tests.xml" + $c = "gotestsum --format=standard-quiet --jsonfile=$jsonFilePath --junitfile=$xmlFilePath -- " $c += "`"-test.v`" " if ($null -ne $env:INTEGRATION_TEST_NAME) { # Makes is quicker for debugging to be able to run only a subset of the integration tests $c += "`"-test.run`" " @@ -864,10 +903,12 @@ Try { if (!($env:INTEGRATION_TESTFLAGS)) { $env:INTEGRATION_TESTFLAGS = "-test.v" } - Set-Location "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker" $start=(Get-Date); Invoke-Expression ".\hack\make.ps1 -TestIntegration"; $Duration=New-Timespan -Start $start -End (Get-Date) + $IntTestsRunResult = $LastExitCode $ErrorActionPreference = "Stop" - if (-not($LastExitCode -eq 0)) { + # Copy all the test results to TEMPORIG for archival + Copy-Item -Path "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker\bundles\junit-report*xml" -Destination $TEMPORIG + if (-not($IntTestsRunResult -eq 0)) { Throw "ERROR: Integration API tests failed at $(Get-Date). Duration`:$Duration" } @@ -877,6 +918,7 @@ Try { Set-Location "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker\integration-cli" # Explicit to not use measure-command otherwise don't get output as it goes $start=(Get-Date); Invoke-Expression $c; $Duration=New-Timespan -Start $start -End (Get-Date) + Copy-Item -Path $xmlFilePath -Destination $TEMPORIG } $ErrorActionPreference = "Stop" if (-not($LastExitCode -eq 0)) { diff --git a/hack/make.ps1 b/hack/make.ps1 index e8ea35d7984b7e0b64a1a28293380ebcd365b6e2..e613e2360e4e0d8854ebbf5cd882cafcb92dc23b 100644 --- a/hack/make.ps1 +++ b/hack/make.ps1 @@ -94,6 +94,7 @@ param( $ErrorActionPreference = "Stop" $ProgressPreference = "SilentlyContinue" $pushed=$False # To restore the directory if we have temporarily pushed to one. +Set-Variable GOTESTSUM_LOCATION -option Constant -value "$env:GOPATH/src/gotest.tools/gotestsum" # Utility function to get the commit ID of the repository Function Get-GitCommit() { @@ -319,31 +320,42 @@ Function Run-UnitTests() { $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/man" $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/integration" $pkgList = $pkgList -replace "`r`n", " " - $goTestCommand = "go test" + $raceParm + " -cover -ldflags -w -tags """ + "autogen daemon" + """ -a """ + "-test.timeout=10m" + """ $pkgList" + $goTestCommand = "$GOTESTSUM_LOCATION\gotestsum.exe --format=standard-quiet --jsonfile=bundles\go-test-report-unit-tests.json --junitfile=bundles\junit-report-unit-tests.xml -- " + $raceParm + " -cover -ldflags -w -a """ + "-test.timeout=10m" + """ $pkgList" + Write-Host "INFO: Invoking unit tests run with $goTestCommand" Invoke-Expression $goTestCommand if ($LASTEXITCODE -ne 0) { Throw "Unit tests failed" } } # Run the integration tests Function Run-IntegrationTests() { - $env:DOCKER_INTEGRATION_DAEMON_DEST = $root + "\bundles\tmp" + $escRoot = [Regex]::Escape($root) + $env:DOCKER_INTEGRATION_DAEMON_DEST = $bundlesDir + "\tmp" $dirs = go list -test -f '{{- if ne .ForTest `"`" -}}{{- .Dir -}}{{- end -}}' .\integration\... - $integration_api_dirs = @() ForEach($dir in $dirs) { - $integration_api_dirs += $dir - Write-Host "Building test suite binary $dir" - go test -c -o "$dir\test.exe" $dir - } - - ForEach($dir in $integration_api_dirs) { + # Normalize directory name for using in the test results files. + $normDir = $dir.Trim() + $normDir = $normDir -replace $escRoot, "" + $normDir = $normDir -replace "\\", "-" + $normDir = $normDir -replace "\/", "-" + $normDir = $normDir -replace "\.", "-" + if ($normDir.StartsWith("-")) + { + $normDir = $normDir.TrimStart("-") + } + if ($normDir.EndsWith("-")) + { + $normDir = $normDir.TrimEnd("-") + } + $jsonFilePath = $bundlesDir + "\go-test-report-int-tests-$normDir" + ".json" + $xmlFilePath = $bundlesDir + "\junit-report-int-tests-$normDir" + ".xml" Set-Location $dir Write-Host "Running $($PWD.Path)" $pinfo = New-Object System.Diagnostics.ProcessStartInfo - $pinfo.FileName = "$($PWD.Path)\test.exe" + $pinfo.FileName = "gotestsum.exe" $pinfo.WorkingDirectory = "$($PWD.Path)" $pinfo.RedirectStandardError = $true $pinfo.UseShellExecute = $false - $pinfo.Arguments = $env:INTEGRATION_TESTFLAGS + $pinfo.Arguments = "--format=standard-quiet --jsonfile=$jsonFilePath --junitfile=$xmlFilePath -- $env:INTEGRATION_TESTFLAGS" $p = New-Object System.Diagnostics.Process $p.StartInfo = $pinfo $p.Start() | Out-Null @@ -351,6 +363,8 @@ Function Run-IntegrationTests() { $err = $p.StandardError.ReadToEnd() if (($LASTEXITCODE -ne 0) -and ($err -notlike "*warning: no tests to run*")) { Throw "Integration tests failed: $err" + } else { + Write-Host "$err" } } } @@ -363,6 +377,11 @@ Try { $root = $(Split-Path $MyInvocation.MyCommand.Definition -Parent | Split-Path -Parent) Push-Location $root + # Ensure the bundles directory exists + $bundlesDir = $root + "\bundles" + Set-Variable bundlesDir -option ReadOnly + New-Item -Force $bundlesDir -ItemType Directory | Out-Null + # Handle the "-All" shortcut to turn on all things we can handle. # Note we expressly only include the items which can run in a container - the validations tests cannot # as they require the .git directory which is excluded from the image by .dockerignore @@ -424,8 +443,6 @@ Try { # Build the binaries if ($Client -or $Daemon) { - # Create the bundles directory if it doesn't exist - if (-not (Test-Path ".\bundles")) { New-Item ".\bundles" -ItemType Directory | Out-Null } # Perform the actual build if ($Daemon) { Execute-Build "daemon" "daemon" "dockerd" }