Build and use gotestsum for running all tests
1. Dockerfile.Windows modified to build gotestsum.exe 2. Use gotestsum.exe in invoking the execution of: (a) Unit tests (run in containers), (b) Integration tests (run outside containers) (c) Integration-cli (run outside containers) No changes made to other categories of tests (e.g. LCOW). 3. Copy .xml files produced by gotestsum in appropriate paths where Jenkins can ingest them 4. Modify Jenkinsfile to mark results output as being jUnit "type" as well as to archive the .xml test result files as artifacts. Signed-off-by: Vikram bir Singh <vikrambir.singh@docker.com> Signed-off-by: Sebastiaan van Stijn <github@gone.nl>
This commit is contained in:
Vikram bir Singh
Sebastiaan van Stijn
parent
5f83f15ecb
commit
8d2e1ee546
4 changed files with 120 additions and 24 deletions
|
|
@ -166,6 +166,7 @@ FROM microsoft/windowsservercore
|
|||
SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
|
||||
|
||||
ARG GO_VERSION=1.13.4
|
||||
ARG GOTESTSUM_COMMIT=v0.3.5
|
||||
|
||||
# Environment variable notes:
|
||||
# - GO_VERSION must be consistent with 'Dockerfile' used by Linux.
|
||||
|
|
@ -174,7 +175,8 @@ ENV GO_VERSION=${GO_VERSION} `
|
|||
GIT_VERSION=2.11.1 `
|
||||
GOPATH=C:\gopath `
|
||||
GO111MODULE=off `
|
||||
FROM_DOCKERFILE=1
|
||||
FROM_DOCKERFILE=1 `
|
||||
GOTESTSUM_COMMIT=${GOTESTSUM_COMMIT}
|
||||
|
||||
RUN `
|
||||
Function Test-Nano() { `
|
||||
|
|
@ -249,13 +251,46 @@ RUN `
|
|||
Remove-Item C:\binutils.zip; `
|
||||
Remove-Item C:\gitsetup.zip; `
|
||||
`
|
||||
Write-Host INFO: Creating source directory...; `
|
||||
New-Item -ItemType Directory -Path ${GOPATH}\src\github.com\docker\docker | Out-Null; `
|
||||
# Ensure all directories exist that we will require below....
|
||||
$srcDir = """$Env:GOPATH`\src\github.com\docker\docker\bundles"""; `
|
||||
Write-Host INFO: Ensuring existence of directory $srcDir...; `
|
||||
New-Item -Force -ItemType Directory -Path $srcDir | Out-Null; `
|
||||
`
|
||||
Write-Host INFO: Configuring git core.autocrlf...; `
|
||||
C:\git\cmd\git config --global core.autocrlf true; `
|
||||
C:\git\cmd\git config --global core.autocrlf true;
|
||||
|
||||
RUN `
|
||||
Function Build-GoTestSum() { `
|
||||
Write-Host "INFO: Building gotestsum version $Env:GOTESTSUM_COMMIT in $Env:GOPATH"; `
|
||||
$optsForGet = @('"get"', '"-d"', '"gotest.tools/gotestsum"'); `
|
||||
&go $optsForGet; `
|
||||
$savedExitCode = $LASTEXITCODE; `
|
||||
if ($savedExitCode -ne 0) { `
|
||||
Throw '"Failed getting gotestsum sources..."' `
|
||||
}; `
|
||||
Write-Host "INFO: Sources obtained for gotestsum..."; `
|
||||
$GotestsumPath=Join-Path -Path $Env:GOPATH -ChildPath "src\gotest.tools\gotestsum"; `
|
||||
Push-Location $GotestsumPath; `
|
||||
$optsForCheckout = @('"checkout"', '"-q"', """$Env:GOTESTSUM_COMMIT"""); `
|
||||
&git $optsForCheckout; `
|
||||
$savedExitCode = $LASTEXITCODE; `
|
||||
if ($savedExitCode -eq 0) { `
|
||||
Write-Host "INFO: Checkout done for gotestsum..."; `
|
||||
$optsForBuild = @('"build"', '"-buildmode=exe"'); `
|
||||
&go $optsForBuild; `
|
||||
$savedExitCode = $LASTEXITCODE; `
|
||||
} else { `
|
||||
Throw '"gotestsum checkout failed..."'; `
|
||||
} `
|
||||
Pop-Location; `
|
||||
`
|
||||
if ($savedExitCode -ne 0) { `
|
||||
Throw '"gotestsum build failed..."'; `
|
||||
} `
|
||||
Write-Host "INFO: Build done for gotestsum..."; `
|
||||
} `
|
||||
`
|
||||
Write-Host INFO: Completed
|
||||
Build-GoTestSum
|
||||
|
||||
# Make PowerShell the default entrypoint
|
||||
ENTRYPOINT ["powershell.exe"]
|
||||
|
|
|
|||
6
Jenkinsfile
vendored
6
Jenkinsfile
vendored
|
|
@ -928,13 +928,14 @@ pipeline {
|
|||
}
|
||||
post {
|
||||
always {
|
||||
junit testResults: 'bundles/junit-report-*.xml', allowEmptyResults: true
|
||||
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') {
|
||||
powershell '''
|
||||
$bundleName="windowsRS1-integration"
|
||||
Write-Host -ForegroundColor Green "Creating ${bundleName}-bundles.zip"
|
||||
|
||||
# archiveArtifacts does not support env-vars to , so save the artifacts in a fixed location
|
||||
Compress-Archive -Path "${env:TEMP}/CIDUT.out", "${env:TEMP}/CIDUT.err" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip"
|
||||
Compress-Archive -Path "${env:TEMP}/CIDUT.out", "${env:TEMP}/CIDUT.err", "${env:TEMP}/testresults/unittests/junit-report-unit-tests.xml" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip"
|
||||
'''
|
||||
|
||||
archiveArtifacts artifacts: '*-bundles.zip', allowEmptyArchive: true
|
||||
|
|
@ -988,13 +989,14 @@ pipeline {
|
|||
}
|
||||
post {
|
||||
always {
|
||||
junit testResults: 'bundles/junit-report-*.xml', allowEmptyResults: true
|
||||
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') {
|
||||
powershell '''
|
||||
$bundleName="windowsRS5-integration"
|
||||
Write-Host -ForegroundColor Green "Creating ${bundleName}-bundles.zip"
|
||||
|
||||
# archiveArtifacts does not support env-vars to , so save the artifacts in a fixed location
|
||||
Compress-Archive -Path "${env:TEMP}/CIDUT.out", "${env:TEMP}/CIDUT.err" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip"
|
||||
Compress-Archive -Path "${env:TEMP}/CIDUT.out", "${env:TEMP}/CIDUT.err", "${env:TEMP}/junit-report-*.xml" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip"
|
||||
'''
|
||||
|
||||
archiveArtifacts artifacts: '*-bundles.zip', allowEmptyArchive: true
|
||||
|
|
|
|||
|
|
@ -447,6 +447,8 @@ Try {
|
|||
$errorActionPreference='Stop'
|
||||
New-Item -ItemType Directory "$env:TEMP" -ErrorAction SilentlyContinue | Out-Null
|
||||
New-Item -ItemType Directory "$env:TEMP\userprofile" -ErrorAction SilentlyContinue | Out-Null
|
||||
New-Item -ItemType Directory "$env:TEMP\testresults" -ErrorAction SilentlyContinue | Out-Null
|
||||
New-Item -ItemType Directory "$env:TEMP\testresults\unittests" -ErrorAction SilentlyContinue | Out-Null
|
||||
New-Item -ItemType Directory "$env:TEMP\localappdata" -ErrorAction SilentlyContinue | Out-Null
|
||||
New-Item -ItemType Directory "$env:TEMP\binary" -ErrorAction SilentlyContinue | Out-Null
|
||||
New-Item -ItemType Directory "$env:TEMP\installer" -ErrorAction SilentlyContinue | Out-Null
|
||||
|
|
@ -525,6 +527,16 @@ Try {
|
|||
if (-not($LastExitCode -eq 0)) {
|
||||
Throw "ERROR: Failed to docker cp the daemon binary (dockerd.exe) to $env:TEMP\binary"
|
||||
}
|
||||
|
||||
$GotestsumBinRelLocationInContainer="src\gotest.tools\gotestsum\gotestsum.exe"
|
||||
if (-not($LastExitCode -eq 0)) {
|
||||
Throw "ERROR: Failed to docker cp the gotestsum binary (gotestsum.exe) to $env:TEMP\binary"
|
||||
}
|
||||
docker cp "$COMMITHASH`:c`:\gopath\$GotestsumBinRelLocationInContainer" $env:TEMP\binary\
|
||||
if (-not (Test-Path "$env:TEMP\binary\gotestsum.exe")) {
|
||||
Throw "ERROR: gotestsum.exe not found...." `
|
||||
}
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
# Copy the built dockerd.exe to dockerd-$COMMITHASH.exe so that easily spotted in task manager.
|
||||
|
|
@ -774,11 +786,36 @@ Try {
|
|||
# Run the unit tests inside a container unless SKIP_UNIT_TESTS is defined
|
||||
if (($null -eq $env:LCOW_MODE) -and ($null -eq $env:LCOW_BASIC_MODE)) {
|
||||
if ($null -eq $env:SKIP_UNIT_TESTS) {
|
||||
$ContainerNameForUnitTests = $COMMITHASH + "_UnitTests"
|
||||
Write-Host -ForegroundColor Cyan "INFO: Running unit tests at $(Get-Date)..."
|
||||
$ErrorActionPreference = "SilentlyContinue"
|
||||
$Duration=$(Measure-Command {docker run -e DOCKER_GITCOMMIT=$COMMITHASH$CommitUnsupported docker hack\make.ps1 -TestUnit | Out-Host })
|
||||
$Duration=$(Measure-Command {docker run --name $ContainerNameForUnitTests -e DOCKER_GITCOMMIT=$COMMITHASH$CommitUnsupported docker hack\make.ps1 -TestUnit | Out-Host })
|
||||
$TestRunExitCode = $LastExitCode
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
# Saving for artifacts......
|
||||
$unitTestsContPath="$ContainerNameForUnitTests`:c`:\gopath\src\github.com\docker\docker\bundles"
|
||||
$JunitExpectedContFilePath = "$unitTestsContPath\junit-report-unit-tests.xml"
|
||||
docker cp $JunitExpectedContFilePath "$TEMPORIG"
|
||||
if (-not($LastExitCode -eq 0)) {
|
||||
Throw "ERROR: Failed to docker cp the unit tests report ($JunitExpectedContFilePath) to $TEMPORIG"
|
||||
}
|
||||
|
||||
if (Test-Path "$TEMPORIG\junit-report-unit-tests.xml") {
|
||||
Write-Host -ForegroundColor Magenta "INFO: Unit tests results($TEMPORIG\junit-report-unit-tests.xml) exist. pwd=$pwd"
|
||||
} else {
|
||||
Write-Host -ForegroundColor Magenta "ERROR: Unit tests results($TEMPORIG\junit-report-unit-tests.xml) do not exist. pwd=$pwd"
|
||||
}
|
||||
|
||||
# Saving where jenkins will take a look at.....
|
||||
$bundlesDir = "bundles"
|
||||
New-Item -Force -ItemType Directory $bundlesDir | Out-Null
|
||||
docker cp $JunitExpectedContFilePath "$bundlesDir"
|
||||
if (-not($LastExitCode -eq 0)) {
|
||||
Throw "ERROR: Failed to docker cp the unit tests report ($JunitExpectedContFilePath) to $bundlesDir"
|
||||
}
|
||||
|
||||
if (-not($TestRunExitCode -eq 0)) {
|
||||
Throw "ERROR: Unit tests failed"
|
||||
}
|
||||
Write-Host -ForegroundColor Green "INFO: Unit tests ended at $(Get-Date). Duration`:$Duration"
|
||||
|
|
@ -830,7 +867,9 @@ Try {
|
|||
$env:OrigDOCKER_HOST="$env:DOCKER_HOST"
|
||||
|
||||
#https://blogs.technet.microsoft.com/heyscriptingguy/2011/09/20/solve-problems-with-external-command-lines-in-powershell/ is useful to see tokenising
|
||||
$c = "go test "
|
||||
$jsonFilePath = "..\\bundles\\go-test-report-intcli-tests.json"
|
||||
$xmlFilePath = "..\\bundles\\junit-report-intcli-tests.xml"
|
||||
$c = "gotestsum --format=standard-quiet --jsonfile=$jsonFilePath --junitfile=$xmlFilePath -- "
|
||||
$c += "`"-test.v`" "
|
||||
if ($null -ne $env:INTEGRATION_TEST_NAME) { # Makes is quicker for debugging to be able to run only a subset of the integration tests
|
||||
$c += "`"-test.run`" "
|
||||
|
|
@ -864,10 +903,12 @@ Try {
|
|||
if (!($env:INTEGRATION_TESTFLAGS)) {
|
||||
$env:INTEGRATION_TESTFLAGS = "-test.v"
|
||||
}
|
||||
Set-Location "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker"
|
||||
$start=(Get-Date); Invoke-Expression ".\hack\make.ps1 -TestIntegration"; $Duration=New-Timespan -Start $start -End (Get-Date)
|
||||
$IntTestsRunResult = $LastExitCode
|
||||
$ErrorActionPreference = "Stop"
|
||||
if (-not($LastExitCode -eq 0)) {
|
||||
# Copy all the test results to TEMPORIG for archival
|
||||
Copy-Item -Path "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker\bundles\junit-report*xml" -Destination $TEMPORIG
|
||||
if (-not($IntTestsRunResult -eq 0)) {
|
||||
Throw "ERROR: Integration API tests failed at $(Get-Date). Duration`:$Duration"
|
||||
}
|
||||
|
||||
|
|
@ -877,6 +918,7 @@ Try {
|
|||
Set-Location "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker\integration-cli"
|
||||
# Explicit to not use measure-command otherwise don't get output as it goes
|
||||
$start=(Get-Date); Invoke-Expression $c; $Duration=New-Timespan -Start $start -End (Get-Date)
|
||||
Copy-Item -Path $xmlFilePath -Destination $TEMPORIG
|
||||
}
|
||||
$ErrorActionPreference = "Stop"
|
||||
if (-not($LastExitCode -eq 0)) {
|
||||
|
|
|
|||
|
|
@ -94,6 +94,7 @@ param(
|
|||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
$pushed=$False # To restore the directory if we have temporarily pushed to one.
|
||||
Set-Variable GOTESTSUM_LOCATION -option Constant -value "$env:GOPATH/src/gotest.tools/gotestsum"
|
||||
|
||||
# Utility function to get the commit ID of the repository
|
||||
Function Get-GitCommit() {
|
||||
|
|
@ -319,31 +320,42 @@ Function Run-UnitTests() {
|
|||
$pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/man"
|
||||
$pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/integration"
|
||||
$pkgList = $pkgList -replace "`r`n", " "
|
||||
$goTestCommand = "go test" + $raceParm + " -cover -ldflags -w -tags """ + "autogen daemon" + """ -a """ + "-test.timeout=10m" + """ $pkgList"
|
||||
$goTestCommand = "$GOTESTSUM_LOCATION\gotestsum.exe --format=standard-quiet --jsonfile=bundles\go-test-report-unit-tests.json --junitfile=bundles\junit-report-unit-tests.xml -- " + $raceParm + " -cover -ldflags -w -a """ + "-test.timeout=10m" + """ $pkgList"
|
||||
Write-Host "INFO: Invoking unit tests run with $goTestCommand"
|
||||
Invoke-Expression $goTestCommand
|
||||
if ($LASTEXITCODE -ne 0) { Throw "Unit tests failed" }
|
||||
}
|
||||
|
||||
# Run the integration tests
|
||||
Function Run-IntegrationTests() {
|
||||
$env:DOCKER_INTEGRATION_DAEMON_DEST = $root + "\bundles\tmp"
|
||||
$escRoot = [Regex]::Escape($root)
|
||||
$env:DOCKER_INTEGRATION_DAEMON_DEST = $bundlesDir + "\tmp"
|
||||
$dirs = go list -test -f '{{- if ne .ForTest `"`" -}}{{- .Dir -}}{{- end -}}' .\integration\...
|
||||
$integration_api_dirs = @()
|
||||
ForEach($dir in $dirs) {
|
||||
$integration_api_dirs += $dir
|
||||
Write-Host "Building test suite binary $dir"
|
||||
go test -c -o "$dir\test.exe" $dir
|
||||
}
|
||||
|
||||
ForEach($dir in $integration_api_dirs) {
|
||||
# Normalize directory name for using in the test results files.
|
||||
$normDir = $dir.Trim()
|
||||
$normDir = $normDir -replace $escRoot, ""
|
||||
$normDir = $normDir -replace "\\", "-"
|
||||
$normDir = $normDir -replace "\/", "-"
|
||||
$normDir = $normDir -replace "\.", "-"
|
||||
if ($normDir.StartsWith("-"))
|
||||
{
|
||||
$normDir = $normDir.TrimStart("-")
|
||||
}
|
||||
if ($normDir.EndsWith("-"))
|
||||
{
|
||||
$normDir = $normDir.TrimEnd("-")
|
||||
}
|
||||
$jsonFilePath = $bundlesDir + "\go-test-report-int-tests-$normDir" + ".json"
|
||||
$xmlFilePath = $bundlesDir + "\junit-report-int-tests-$normDir" + ".xml"
|
||||
Set-Location $dir
|
||||
Write-Host "Running $($PWD.Path)"
|
||||
$pinfo = New-Object System.Diagnostics.ProcessStartInfo
|
||||
$pinfo.FileName = "$($PWD.Path)\test.exe"
|
||||
$pinfo.FileName = "gotestsum.exe"
|
||||
$pinfo.WorkingDirectory = "$($PWD.Path)"
|
||||
$pinfo.RedirectStandardError = $true
|
||||
$pinfo.UseShellExecute = $false
|
||||
$pinfo.Arguments = $env:INTEGRATION_TESTFLAGS
|
||||
$pinfo.Arguments = "--format=standard-quiet --jsonfile=$jsonFilePath --junitfile=$xmlFilePath -- $env:INTEGRATION_TESTFLAGS"
|
||||
$p = New-Object System.Diagnostics.Process
|
||||
$p.StartInfo = $pinfo
|
||||
$p.Start() | Out-Null
|
||||
|
|
@ -351,6 +363,8 @@ Function Run-IntegrationTests() {
|
|||
$err = $p.StandardError.ReadToEnd()
|
||||
if (($LASTEXITCODE -ne 0) -and ($err -notlike "*warning: no tests to run*")) {
|
||||
Throw "Integration tests failed: $err"
|
||||
} else {
|
||||
Write-Host "$err"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -363,6 +377,11 @@ Try {
|
|||
$root = $(Split-Path $MyInvocation.MyCommand.Definition -Parent | Split-Path -Parent)
|
||||
Push-Location $root
|
||||
|
||||
# Ensure the bundles directory exists
|
||||
$bundlesDir = $root + "\bundles"
|
||||
Set-Variable bundlesDir -option ReadOnly
|
||||
New-Item -Force $bundlesDir -ItemType Directory | Out-Null
|
||||
|
||||
# Handle the "-All" shortcut to turn on all things we can handle.
|
||||
# Note we expressly only include the items which can run in a container - the validations tests cannot
|
||||
# as they require the .git directory which is excluded from the image by .dockerignore
|
||||
|
|
@ -424,8 +443,6 @@ Try {
|
|||
|
||||
# Build the binaries
|
||||
if ($Client -or $Daemon) {
|
||||
# Create the bundles directory if it doesn't exist
|
||||
if (-not (Test-Path ".\bundles")) { New-Item ".\bundles" -ItemType Directory | Out-Null }
|
||||
|
||||
# Perform the actual build
|
||||
if ($Daemon) { Execute-Build "daemon" "daemon" "dockerd" }
|
||||
|
|
|
|||
Loading…
Reference in a new issue