diff --git a/Jenkinsfile b/Jenkinsfile index fd3408472e..e012fb7b16 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,11 +8,8 @@ pipeline { timestamps() } parameters { - booleanParam(name: 'unit_validate', defaultValue: true, description: 'amd64 (x86_64) unit tests and vendor check') + booleanParam(name: 'validate', defaultValue: true, description: 'amd64 (x86_64) validate and vendor check') booleanParam(name: 'validate_force', defaultValue: false, description: 'force validation steps to be run, even if no changes were detected') - booleanParam(name: 'amd64', defaultValue: true, description: 'amd64 (x86_64) Build/Test') - booleanParam(name: 'rootless', defaultValue: true, description: 'amd64 (x86_64) Build/Test (Rootless mode)') - booleanParam(name: 'cgroup2', defaultValue: true, description: 'amd64 (x86_64) Build/Test (cgroup v2)') booleanParam(name: 'arm64', defaultValue: true, description: 'ARM (arm64) Build/Test') booleanParam(name: 's390x', defaultValue: false, description: 'IBM Z (s390x) Build/Test') booleanParam(name: 'ppc64le', defaultValue: false, description: 'PowerPC (ppc64le) Build/Test') @@ -60,7 +57,7 @@ pipeline { stage('unit-validate') { when { beforeAgent true - expression { params.unit_validate } + expression { params.validate } } agent { label 'amd64 && ubuntu-1804 && overlay2' } environment { @@ -105,62 +102,6 @@ pipeline { ''' } } - stage("Docker-py") { - steps { - sh ''' - docker run --rm -t --privileged \ - -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \ - --name docker-pr$BUILD_NUMBER \ - -e DOCKER_EXPERIMENTAL \ - -e DOCKER_GITCOMMIT=${GIT_COMMIT} \ - -e DOCKER_GRAPHDRIVER \ - -e VALIDATE_REPO=${GIT_URL} \ - -e VALIDATE_BRANCH=${CHANGE_TARGET} \ - docker:${GIT_COMMIT} \ - hack/make.sh \ - dynbinary \ - test-docker-py - ''' - } - post { - always { - junit testResults: 'bundles/test-docker-py/junit-report.xml', allowEmptyResults: true - - sh ''' - echo "Ensuring container killed." - docker rm -vf docker-pr$BUILD_NUMBER || true - ''' - - sh ''' - echo 'Chowning /workspace to jenkins user' - docker run --rm -v "$WORKSPACE:/workspace" busybox chown -R "$(id -u):$(id -g)" /workspace - ''' - - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') { - sh ''' - bundleName=docker-py - echo "Creating ${bundleName}-bundles.tar.gz" - tar -czf ${bundleName}-bundles.tar.gz bundles/test-docker-py/*.xml bundles/test-docker-py/*.log - ''' - - archiveArtifacts artifacts: '*-bundles.tar.gz', allowEmptyArchive: true - } - } - } - } - stage("Static") { - steps { - sh ''' - docker run --rm -t --privileged \ - -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \ - --name docker-pr$BUILD_NUMBER \ - -e DOCKER_GITCOMMIT=${GIT_COMMIT} \ - -e DOCKER_GRAPHDRIVER \ - docker:${GIT_COMMIT} \ - hack/make.sh binary - ''' - } - } stage("Cross") { steps { sh ''' @@ -174,31 +115,6 @@ pipeline { ''' } } - // needs to be last stage that calls make.sh for the junit report to work - stage("Unit tests") { - steps { - sh ''' - sudo modprobe ip6table_filter - ''' - sh ''' - docker run --rm -t --privileged \ - -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \ - --name docker-pr$BUILD_NUMBER \ - -e DOCKER_EXPERIMENTAL \ - -e DOCKER_GITCOMMIT=${GIT_COMMIT} \ - -e DOCKER_GRAPHDRIVER \ - -e VALIDATE_REPO=${GIT_URL} \ - -e VALIDATE_BRANCH=${CHANGE_TARGET} \ - docker:${GIT_COMMIT} \ - hack/test/unit - ''' - } - post { - always { - junit testResults: 'bundles/junit-report*.xml', allowEmptyResults: true - } - } - } stage("Validate vendor") { steps { sh ''' @@ -246,305 +162,6 @@ pipeline { } } } - stage('amd64') { - when { - beforeAgent true - expression { params.amd64 } - } - agent { label 'amd64 && ubuntu-1804 && overlay2' } - - stages { - stage("Print info") { - steps { - sh 'docker version' - sh 'docker info' - sh ''' - echo "check-config.sh version: ${CHECK_CONFIG_COMMIT}" - curl -fsSL -o ${WORKSPACE}/check-config.sh "https://raw.githubusercontent.com/moby/moby/${CHECK_CONFIG_COMMIT}/contrib/check-config.sh" \ - && bash ${WORKSPACE}/check-config.sh || true - ''' - } - } - stage("Build dev image") { - steps { - sh ''' - # todo: include ip_vs in base image - sudo modprobe ip_vs - - docker build --force-rm --build-arg APT_MIRROR -t docker:${GIT_COMMIT} . - ''' - } - } - stage("Run tests") { - steps { - sh '''#!/bin/bash - # bash is needed so 'jobs -p' works properly - # it also accepts setting inline envvars for functions without explicitly exporting - set -x - - run_tests() { - [ -n "$TESTDEBUG" ] && rm= || rm=--rm; - docker run $rm -t --privileged \ - -v "$WORKSPACE/bundles/${TEST_INTEGRATION_DEST}:/go/src/github.com/docker/docker/bundles" \ - -v "$WORKSPACE/bundles/dynbinary-daemon:/go/src/github.com/docker/docker/bundles/dynbinary-daemon" \ - -v "$WORKSPACE/.git:/go/src/github.com/docker/docker/.git" \ - --name "$CONTAINER_NAME" \ - -e KEEPBUNDLE=1 \ - -e TESTDEBUG \ - -e TESTFLAGS \ - -e TEST_SKIP_INTEGRATION \ - -e TEST_SKIP_INTEGRATION_CLI \ - -e DOCKER_GITCOMMIT=${GIT_COMMIT} \ - -e DOCKER_GRAPHDRIVER \ - -e TIMEOUT \ - -e VALIDATE_REPO=${GIT_URL} \ - -e VALIDATE_BRANCH=${CHANGE_TARGET} \ - docker:${GIT_COMMIT} \ - hack/make.sh \ - "$1" \ - test-integration - } - - trap "exit" INT TERM - trap 'pids=$(jobs -p); echo "Remaining pids to kill: [$pids]"; [ -z "$pids" ] || kill $pids' EXIT - - CONTAINER_NAME=docker-pr$BUILD_NUMBER - - docker run --rm -t --privileged \ - -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \ - -v "$WORKSPACE/.git:/go/src/github.com/docker/docker/.git" \ - --name ${CONTAINER_NAME}-build \ - -e DOCKER_EXPERIMENTAL \ - -e DOCKER_GITCOMMIT=${GIT_COMMIT} \ - -e DOCKER_GRAPHDRIVER \ - docker:${GIT_COMMIT} \ - hack/make.sh \ - dynbinary - - # flaky + integration - TEST_INTEGRATION_DEST=1 CONTAINER_NAME=${CONTAINER_NAME}-1 TEST_SKIP_INTEGRATION_CLI=1 run_tests test-integration-flaky & - - # integration-cli first set - TEST_INTEGRATION_DEST=2 CONTAINER_NAME=${CONTAINER_NAME}-2 TEST_SKIP_INTEGRATION=1 TESTFLAGS="-test.run Test(DockerSuite|DockerNetworkSuite|DockerHubPullSuite|DockerRegistrySuite|DockerSchema1RegistrySuite|DockerRegistryAuthTokenSuite|DockerRegistryAuthHtpasswdSuite)/" run_tests & - - # integration-cli second set - TEST_INTEGRATION_DEST=3 CONTAINER_NAME=${CONTAINER_NAME}-3 TEST_SKIP_INTEGRATION=1 TESTFLAGS="-test.run Test(DockerSwarmSuite|DockerDaemonSuite|DockerExternalVolumeSuite)/" run_tests & - - c=0 - for job in $(jobs -p); do - wait ${job} || c=$? - done - exit $c - ''' - } - post { - always { - junit testResults: 'bundles/**/*-report.xml', allowEmptyResults: true - } - } - } - } - - post { - always { - sh ''' - echo "Ensuring container killed." - cids=$(docker ps -aq -f name=docker-pr${BUILD_NUMBER}-*) - [ -n "$cids" ] && docker rm -vf $cids || true - ''' - - sh ''' - echo "Chowning /workspace to jenkins user" - docker run --rm -v "$WORKSPACE:/workspace" busybox chown -R "$(id -u):$(id -g)" /workspace - ''' - - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') { - sh ''' - bundleName=amd64 - echo "Creating ${bundleName}-bundles.tar.gz" - # exclude overlay2 directories - find bundles -path '*/root/*overlay2' -prune -o -type f \\( -name '*-report.json' -o -name '*.log' -o -name '*.prof' -o -name '*-report.xml' \\) -print | xargs tar -czf ${bundleName}-bundles.tar.gz - ''' - - archiveArtifacts artifacts: '*-bundles.tar.gz', allowEmptyArchive: true - } - } - cleanup { - sh 'make clean' - deleteDir() - } - } - } - stage('rootless') { - when { - beforeAgent true - expression { params.rootless } - } - agent { label 'amd64 && ubuntu-1804 && overlay2' } - stages { - stage("Print info") { - steps { - sh 'docker version' - sh 'docker info' - sh ''' - echo "check-config.sh version: ${CHECK_CONFIG_COMMIT}" - curl -fsSL -o ${WORKSPACE}/check-config.sh "https://raw.githubusercontent.com/moby/moby/${CHECK_CONFIG_COMMIT}/contrib/check-config.sh" \ - && bash ${WORKSPACE}/check-config.sh || true - ''' - } - } - stage("Build dev image") { - steps { - sh ''' - docker build --force-rm --build-arg APT_MIRROR -t docker:${GIT_COMMIT} . - ''' - } - } - stage("Integration tests") { - environment { - DOCKER_ROOTLESS = '1' - TEST_SKIP_INTEGRATION_CLI = '1' - } - steps { - sh ''' - docker run --rm -t --privileged \ - -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \ - --name docker-pr$BUILD_NUMBER \ - -e DOCKER_GITCOMMIT=${GIT_COMMIT} \ - -e DOCKER_GRAPHDRIVER \ - -e DOCKER_EXPERIMENTAL \ - -e DOCKER_ROOTLESS \ - -e TEST_SKIP_INTEGRATION_CLI \ - -e TIMEOUT \ - -e VALIDATE_REPO=${GIT_URL} \ - -e VALIDATE_BRANCH=${CHANGE_TARGET} \ - docker:${GIT_COMMIT} \ - hack/make.sh \ - dynbinary \ - test-integration - ''' - } - post { - always { - junit testResults: 'bundles/**/*-report.xml', allowEmptyResults: true - } - } - } - } - - post { - always { - sh ''' - echo "Ensuring container killed." - docker rm -vf docker-pr$BUILD_NUMBER || true - ''' - - sh ''' - echo "Chowning /workspace to jenkins user" - docker run --rm -v "$WORKSPACE:/workspace" busybox chown -R "$(id -u):$(id -g)" /workspace - ''' - - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') { - sh ''' - bundleName=amd64-rootless - echo "Creating ${bundleName}-bundles.tar.gz" - # exclude overlay2 directories - find bundles -path '*/root/*overlay2' -prune -o -type f \\( -name '*-report.json' -o -name '*.log' -o -name '*.prof' -o -name '*-report.xml' \\) -print | xargs tar -czf ${bundleName}-bundles.tar.gz - ''' - - archiveArtifacts artifacts: '*-bundles.tar.gz', allowEmptyArchive: true - } - } - cleanup { - sh 'make clean' - deleteDir() - } - } - } - - stage('cgroup2') { - when { - beforeAgent true - expression { params.cgroup2 } - } - agent { label 'amd64 && ubuntu-2004 && cgroup2' } - stages { - stage("Print info") { - steps { - sh 'docker version' - sh 'docker info' - } - } - stage("Build dev image") { - steps { - sh ''' - docker build --force-rm --build-arg APT_MIRROR --build-arg SYSTEMD=true -t docker:${GIT_COMMIT} . - ''' - } - } - stage("Integration tests") { - environment { - DOCKER_SYSTEMD = '1' // recommended cgroup driver for v2 - TEST_SKIP_INTEGRATION_CLI = '1' // CLI tests do not support v2 - } - steps { - sh ''' - docker run --rm -t --privileged \ - -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \ - --name docker-pr$BUILD_NUMBER \ - -e DOCKER_GITCOMMIT=${GIT_COMMIT} \ - -e DOCKER_GRAPHDRIVER \ - -e DOCKER_EXPERIMENTAL \ - -e DOCKER_SYSTEMD \ - -e TEST_SKIP_INTEGRATION_CLI \ - -e TIMEOUT \ - -e VALIDATE_REPO=${GIT_URL} \ - -e VALIDATE_BRANCH=${CHANGE_TARGET} \ - docker:${GIT_COMMIT} \ - hack/make.sh \ - dynbinary \ - test-integration - ''' - } - post { - always { - junit testResults: 'bundles/**/*-report.xml', allowEmptyResults: true - } - } - } - } - - post { - always { - sh ''' - echo "Ensuring container killed." - docker rm -vf docker-pr$BUILD_NUMBER || true - ''' - - sh ''' - echo "Chowning /workspace to jenkins user" - docker run --rm -v "$WORKSPACE:/workspace" busybox chown -R "$(id -u):$(id -g)" /workspace - ''' - - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') { - sh ''' - bundleName=amd64-cgroup2 - echo "Creating ${bundleName}-bundles.tar.gz" - # exclude overlay2 directories - find bundles -path '*/root/*overlay2' -prune -o -type f \\( -name '*-report.json' -o -name '*.log' -o -name '*.prof' -o -name '*-report.xml' \\) -print | xargs tar -czf ${bundleName}-bundles.tar.gz - ''' - - archiveArtifacts artifacts: '*-bundles.tar.gz', allowEmptyArchive: true - } - } - cleanup { - sh 'make clean' - deleteDir() - } - } - } - - stage('s390x') { when { beforeAgent true