make.sh 8.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296
  1. #!/usr/bin/env bash
  2. set -e
  3. # This script builds various binary artifacts from a checkout of the docker
  4. # source code.
  5. #
  6. # Requirements:
  7. # - The current directory should be a checkout of the docker source code
  8. # (https://github.com/docker/docker). Whatever version is checked out
  9. # will be built.
  10. # - The VERSION file, at the root of the repository, should exist, and
  11. # will be used as Docker binary version and package version.
  12. # - The hash of the git commit will also be included in the Docker binary,
  13. # with the suffix -dirty if the repository isn't clean.
  14. # - The script is intented to be run inside the docker container specified
  15. # in the Dockerfile at the root of the source. In other words:
  16. # DO NOT CALL THIS SCRIPT DIRECTLY.
  17. # - The right way to call this script is to invoke "make" from
  18. # your checkout of the Docker repository.
  19. # the Makefile will do a "docker build -t docker ." and then
  20. # "docker run hack/make.sh" in the resulting image.
  21. #
  22. set -o pipefail
  23. export DOCKER_PKG='github.com/docker/docker'
  24. export SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
  25. export MAKEDIR="$SCRIPTDIR/make"
  26. # We're a nice, sexy, little shell script, and people might try to run us;
  27. # but really, they shouldn't. We want to be in a container!
  28. if [ "$PWD" != "/go/src/$DOCKER_PKG" ] || [ -z "$DOCKER_CROSSPLATFORMS" ]; then
  29. {
  30. echo "# WARNING! I don't seem to be running in the Docker container."
  31. echo "# The result of this command might be an incorrect build, and will not be"
  32. echo "# officially supported."
  33. echo "#"
  34. echo "# Try this instead: make all"
  35. echo "#"
  36. } >&2
  37. fi
  38. echo
  39. # List of bundles to create when no argument is passed
  40. DEFAULT_BUNDLES=(
  41. validate-dco
  42. validate-gofmt
  43. validate-lint
  44. validate-pkg
  45. validate-test
  46. validate-toml
  47. validate-vet
  48. binary
  49. test-unit
  50. test-integration-cli
  51. test-docker-py
  52. dynbinary
  53. cover
  54. cross
  55. tgz
  56. )
  57. VERSION=$(< ./VERSION)
  58. if command -v git &> /dev/null && git rev-parse &> /dev/null; then
  59. GITCOMMIT=$(git rev-parse --short HEAD)
  60. if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
  61. GITCOMMIT="$GITCOMMIT-dirty"
  62. fi
  63. BUILDTIME=$(date -u)
  64. elif [ "$DOCKER_GITCOMMIT" ]; then
  65. GITCOMMIT="$DOCKER_GITCOMMIT"
  66. else
  67. echo >&2 'error: .git directory missing and DOCKER_GITCOMMIT not specified'
  68. echo >&2 ' Please either build with the .git directory accessible, or specify the'
  69. echo >&2 ' exact (--short) commit hash you are building using DOCKER_GITCOMMIT for'
  70. echo >&2 ' future accountability in diagnosing build issues. Thanks!'
  71. exit 1
  72. fi
  73. if [ "$AUTO_GOPATH" ]; then
  74. rm -rf .gopath
  75. mkdir -p .gopath/src/"$(dirname "${DOCKER_PKG}")"
  76. ln -sf ../../../.. .gopath/src/"${DOCKER_PKG}"
  77. export GOPATH="${PWD}/.gopath:${PWD}/vendor"
  78. fi
  79. if [ ! "$GOPATH" ]; then
  80. echo >&2 'error: missing GOPATH; please see https://golang.org/doc/code.html#GOPATH'
  81. echo >&2 ' alternatively, set AUTO_GOPATH=1'
  82. exit 1
  83. fi
  84. if [ "$DOCKER_EXPERIMENTAL" ] || [ "$DOCKER_REMAP_ROOT" ]; then
  85. echo >&2 '# WARNING! DOCKER_EXPERIMENTAL is set: building experimental features'
  86. echo >&2
  87. DOCKER_BUILDTAGS+=" experimental"
  88. fi
  89. if [ -z "$DOCKER_CLIENTONLY" ]; then
  90. DOCKER_BUILDTAGS+=" daemon"
  91. if pkg-config libsystemd-journal 2> /dev/null ; then
  92. DOCKER_BUILDTAGS+=" journald"
  93. fi
  94. fi
  95. # test whether "btrfs/version.h" exists and apply btrfs_noversion appropriately
  96. if \
  97. command -v gcc &> /dev/null \
  98. && ! gcc -E - &> /dev/null <<<'#include <btrfs/version.h>' \
  99. ; then
  100. DOCKER_BUILDTAGS+=' btrfs_noversion'
  101. fi
  102. # test whether "libdevmapper.h" is new enough to support deferred remove
  103. # functionality.
  104. if \
  105. command -v gcc &> /dev/null \
  106. && ! ( echo -e '#include <libdevmapper.h>\nint main() { dm_task_deferred_remove(NULL); }'| gcc -ldevmapper -xc - &> /dev/null ) \
  107. ; then
  108. DOCKER_BUILDTAGS+=' libdm_no_deferred_remove'
  109. fi
  110. # Use these flags when compiling the tests and final binary
  111. IAMSTATIC='true'
  112. source "$SCRIPTDIR/make/.go-autogen"
  113. if [ -z "$DOCKER_DEBUG" ]; then
  114. LDFLAGS='-w'
  115. fi
  116. LDFLAGS_STATIC='-linkmode external'
  117. # Cgo -H windows is incompatible with -linkmode external.
  118. if [ "$(go env GOOS)" == 'windows' ]; then
  119. LDFLAGS_STATIC=''
  120. fi
  121. EXTLDFLAGS_STATIC='-static'
  122. # ORIG_BUILDFLAGS is necessary for the cross target which cannot always build
  123. # with options like -race.
  124. ORIG_BUILDFLAGS=( -a -tags "autogen netgo static_build sqlite_omit_load_extension $DOCKER_BUILDTAGS" -installsuffix netgo )
  125. # see https://github.com/golang/go/issues/9369#issuecomment-69864440 for why -installsuffix is necessary here
  126. BUILDFLAGS=( $BUILDFLAGS "${ORIG_BUILDFLAGS[@]}" )
  127. # Test timeout.
  128. : ${TIMEOUT:=120m}
  129. TESTFLAGS+=" -test.timeout=${TIMEOUT}"
  130. LDFLAGS_STATIC_DOCKER="
  131. $LDFLAGS_STATIC
  132. -extldflags \"$EXTLDFLAGS_STATIC\"
  133. "
  134. if [ "$(uname -s)" = 'FreeBSD' ]; then
  135. # Tell cgo the compiler is Clang, not GCC
  136. # https://code.google.com/p/go/source/browse/src/cmd/cgo/gcc.go?spec=svne77e74371f2340ee08622ce602e9f7b15f29d8d3&r=e6794866ebeba2bf8818b9261b54e2eef1c9e588#752
  137. export CC=clang
  138. # "-extld clang" is a workaround for
  139. # https://code.google.com/p/go/issues/detail?id=6845
  140. LDFLAGS="$LDFLAGS -extld clang"
  141. fi
  142. # If sqlite3.h doesn't exist under /usr/include,
  143. # check /usr/local/include also just in case
  144. # (e.g. FreeBSD Ports installs it under the directory)
  145. if [ ! -e /usr/include/sqlite3.h ] && [ -e /usr/local/include/sqlite3.h ]; then
  146. export CGO_CFLAGS='-I/usr/local/include'
  147. export CGO_LDFLAGS='-L/usr/local/lib'
  148. fi
  149. HAVE_GO_TEST_COVER=
  150. if \
  151. go help testflag | grep -- -cover > /dev/null \
  152. && go tool -n cover > /dev/null 2>&1 \
  153. ; then
  154. HAVE_GO_TEST_COVER=1
  155. fi
  156. # If $TESTFLAGS is set in the environment, it is passed as extra arguments to 'go test'.
  157. # You can use this to select certain tests to run, eg.
  158. #
  159. # TESTFLAGS='-test.run ^TestBuild$' ./hack/make.sh test-unit
  160. #
  161. # For integration-cli test, we use [gocheck](https://labix.org/gocheck), if you want
  162. # to run certain tests on your local host, you should run with command:
  163. #
  164. # TESTFLAGS='-check.f DockerSuite.TestBuild*' ./hack/make.sh binary test-integration-cli
  165. #
  166. go_test_dir() {
  167. dir=$1
  168. coverpkg=$2
  169. testcover=()
  170. if [ "$HAVE_GO_TEST_COVER" ]; then
  171. # if our current go install has -cover, we want to use it :)
  172. mkdir -p "$DEST/coverprofiles"
  173. coverprofile="docker${dir#.}"
  174. coverprofile="$ABS_DEST/coverprofiles/${coverprofile//\//-}"
  175. testcover=( -cover -coverprofile "$coverprofile" $coverpkg )
  176. fi
  177. (
  178. echo '+ go test' $TESTFLAGS "${DOCKER_PKG}${dir#.}"
  179. cd "$dir"
  180. export DEST="$ABS_DEST" # we're in a subshell, so this is safe -- our integration-cli tests need DEST, and "cd" screws it up
  181. test_env go test ${testcover[@]} -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}" $TESTFLAGS
  182. )
  183. }
  184. test_env() {
  185. # use "env -i" to tightly control the environment variables that bleed into the tests
  186. env -i \
  187. DEST="$DEST" \
  188. DOCKER_GRAPHDRIVER="$DOCKER_GRAPHDRIVER" \
  189. DOCKER_USERLANDPROXY="$DOCKER_USERLANDPROXY" \
  190. DOCKER_HOST="$DOCKER_HOST" \
  191. DOCKER_REMAP_ROOT="$DOCKER_REMAP_ROOT" \
  192. DOCKER_REMOTE_DAEMON="$DOCKER_REMOTE_DAEMON" \
  193. GOPATH="$GOPATH" \
  194. HOME="$ABS_DEST/fake-HOME" \
  195. PATH="$PATH" \
  196. TEMP="$TEMP" \
  197. TEST_DOCKERINIT_PATH="$TEST_DOCKERINIT_PATH" \
  198. "$@"
  199. }
  200. # a helper to provide ".exe" when it's appropriate
  201. binary_extension() {
  202. if [ "$(go env GOOS)" = 'windows' ]; then
  203. echo -n '.exe'
  204. fi
  205. }
  206. hash_files() {
  207. while [ $# -gt 0 ]; do
  208. f="$1"
  209. shift
  210. dir="$(dirname "$f")"
  211. base="$(basename "$f")"
  212. for hashAlgo in md5 sha256; do
  213. if command -v "${hashAlgo}sum" &> /dev/null; then
  214. (
  215. # subshell and cd so that we get output files like:
  216. # $HASH docker-$VERSION
  217. # instead of:
  218. # $HASH /go/src/github.com/.../$VERSION/binary/docker-$VERSION
  219. cd "$dir"
  220. "${hashAlgo}sum" "$base" > "$base.$hashAlgo"
  221. )
  222. fi
  223. done
  224. done
  225. }
  226. bundle() {
  227. local bundle="$1"; shift
  228. echo "---> Making bundle: $(basename "$bundle") (in $DEST)"
  229. source "$SCRIPTDIR/make/$bundle" "$@"
  230. }
  231. main() {
  232. # We want this to fail if the bundles already exist and cannot be removed.
  233. # This is to avoid mixing bundles from different versions of the code.
  234. mkdir -p bundles
  235. if [ -e "bundles/$VERSION" ]; then
  236. echo "bundles/$VERSION already exists. Removing."
  237. rm -fr "bundles/$VERSION" && mkdir "bundles/$VERSION" || exit 1
  238. echo
  239. fi
  240. if [ "$(go env GOHOSTOS)" != 'windows' ]; then
  241. # Windows and symlinks don't get along well
  242. rm -f bundles/latest
  243. ln -s "$VERSION" bundles/latest
  244. fi
  245. if [ $# -lt 1 ]; then
  246. bundles=(${DEFAULT_BUNDLES[@]})
  247. else
  248. bundles=($@)
  249. fi
  250. for bundle in ${bundles[@]}; do
  251. export DEST="bundles/$VERSION/$(basename "$bundle")"
  252. # Cygdrive paths don't play well with go build -o.
  253. if [[ "$(uname -s)" == CYGWIN* ]]; then
  254. export DEST="$(cygpath -mw "$DEST")"
  255. fi
  256. mkdir -p "$DEST"
  257. ABS_DEST="$(cd "$DEST" && pwd -P)"
  258. bundle "$bundle"
  259. echo
  260. done
  261. }
  262. main "$@"