make.ps1 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472
  1. <#
  2. .NOTES
  3. Author: @jhowardmsft
  4. Summary: Windows native build script. This is similar to functionality provided
  5. by hack\make.sh, but uses native Windows PowerShell semantics. It does
  6. not support the full set of options provided by the Linux counterpart.
  7. For example:
  8. - You can't cross-build Linux docker binaries on Windows
  9. - Hashes aren't generated on binaries
  10. - 'Releasing' isn't supported.
  11. - Integration tests. This is because they currently cannot run inside a container,
  12. and require significant external setup.
  13. It does however provided the minimum necessary to support parts of local Windows
  14. development and Windows to Windows CI.
  15. Usage Examples (run from repo root):
  16. "hack\make.ps1 -Client" to build docker.exe client 64-bit binary (remote repo)
  17. "hack\make.ps1 -TestUnit" to run unit tests
  18. "hack\make.ps1 -Daemon -TestUnit" to build the daemon and run unit tests
  19. "hack\make.ps1 -All" to run everything this script knows about that can run in a container
  20. "hack\make.ps1" to build the daemon binary (same as -Daemon)
  21. "hack\make.ps1 -Binary" shortcut to -Client and -Daemon
  22. .PARAMETER Client
  23. Builds the client binaries.
  24. .PARAMETER Daemon
  25. Builds the daemon binary.
  26. .PARAMETER Binary
  27. Builds the client and daemon binaries. A convenient shortcut to `make.ps1 -Client -Daemon`.
  28. .PARAMETER Race
  29. Use -race in go build and go test.
  30. .PARAMETER Noisy
  31. Use -v in go build.
  32. .PARAMETER ForceBuildAll
  33. Use -a in go build.
  34. .PARAMETER NoOpt
  35. Use -gcflags -N -l in go build to disable optimisation (can aide debugging).
  36. .PARAMETER CommitSuffix
  37. Adds a custom string to be appended to the commit ID (spaces are stripped).
  38. .PARAMETER DCO
  39. Runs the DCO (Developer Certificate Of Origin) test (must be run outside a container).
  40. .PARAMETER PkgImports
  41. Runs the pkg\ directory imports test (must be run outside a container).
  42. .PARAMETER GoFormat
  43. Runs the Go formatting test (must be run outside a container).
  44. .PARAMETER TestUnit
  45. Runs unit tests.
  46. .PARAMETER All
  47. Runs everything this script knows about that can run in a container.
  48. TODO
  49. - Unify the head commit
  50. - Add golint and other checks (swagger maybe?)
  51. #>
  52. param(
  53. [Parameter(Mandatory=$False)][switch]$Client,
  54. [Parameter(Mandatory=$False)][switch]$Daemon,
  55. [Parameter(Mandatory=$False)][switch]$Binary,
  56. [Parameter(Mandatory=$False)][switch]$Race,
  57. [Parameter(Mandatory=$False)][switch]$Noisy,
  58. [Parameter(Mandatory=$False)][switch]$ForceBuildAll,
  59. [Parameter(Mandatory=$False)][switch]$NoOpt,
  60. [Parameter(Mandatory=$False)][string]$CommitSuffix="",
  61. [Parameter(Mandatory=$False)][switch]$DCO,
  62. [Parameter(Mandatory=$False)][switch]$PkgImports,
  63. [Parameter(Mandatory=$False)][switch]$GoFormat,
  64. [Parameter(Mandatory=$False)][switch]$TestUnit,
  65. [Parameter(Mandatory=$False)][switch]$All
  66. )
  67. $ErrorActionPreference = "Stop"
  68. $pushed=$False # To restore the directory if we have temporarily pushed to one.
  69. # Utility function to get the commit ID of the repository
  70. Function Get-GitCommit() {
  71. if (-not (Test-Path ".\.git")) {
  72. # If we don't have a .git directory, but we do have the environment
  73. # variable DOCKER_GITCOMMIT set, that can override it.
  74. if ($env:DOCKER_GITCOMMIT.Length -eq 0) {
  75. Throw ".git directory missing and DOCKER_GITCOMMIT environment variable not specified."
  76. }
  77. Write-Host "INFO: Git commit ($env:DOCKER_GITCOMMIT) assumed from DOCKER_GITCOMMIT environment variable"
  78. return $env:DOCKER_GITCOMMIT
  79. }
  80. $gitCommit=$(git rev-parse --short HEAD)
  81. if ($(git status --porcelain --untracked-files=no).Length -ne 0) {
  82. $gitCommit="$gitCommit-unsupported"
  83. Write-Host ""
  84. Write-Warning "This version is unsupported because there are uncommitted file(s)."
  85. Write-Warning "Either commit these changes, or add them to .gitignore."
  86. git status --porcelain --untracked-files=no | Write-Warning
  87. Write-Host ""
  88. }
  89. return $gitCommit
  90. }
  91. # Utility function to get get the current build version of docker
  92. Function Get-DockerVersion() {
  93. if (-not (Test-Path ".\VERSION")) { Throw "VERSION file not found. Is this running from the root of a docker repository?" }
  94. return $(Get-Content ".\VERSION" -raw).ToString().Replace("`n","").Trim()
  95. }
  96. # Utility function to determine if we are running in a container or not.
  97. # In Windows, we get this through an environment variable set in `Dockerfile.Windows`
  98. Function Check-InContainer() {
  99. if ($env:FROM_DOCKERFILE.Length -eq 0) {
  100. Write-Host ""
  101. Write-Warning "Not running in a container. The result might be an incorrect build."
  102. Write-Host ""
  103. return $False
  104. }
  105. return $True
  106. }
  107. # Utility function to warn if the version of go is correct. Used for local builds
  108. # outside of a container where it may be out of date with master.
  109. Function Verify-GoVersion() {
  110. Try {
  111. $goVersionDockerfile=(Get-Content ".\Dockerfile" | Select-String "ENV GO_VERSION").ToString().Split(" ")[2]
  112. $goVersionInstalled=(go version).ToString().Split(" ")[2].SubString(2)
  113. }
  114. Catch [Exception] {
  115. Throw "Failed to validate go version correctness: $_"
  116. }
  117. if (-not($goVersionInstalled -eq $goVersionDockerfile)) {
  118. Write-Host ""
  119. Write-Warning "Building with golang version $goVersionInstalled. You should update to $goVersionDockerfile"
  120. Write-Host ""
  121. }
  122. }
  123. # Utility function to get the commit for HEAD
  124. Function Get-HeadCommit() {
  125. $head = Invoke-Expression "git rev-parse --verify HEAD"
  126. if ($LASTEXITCODE -ne 0) { Throw "Failed getting HEAD commit" }
  127. return $head
  128. }
  129. # Utility function to get the commit for upstream
  130. Function Get-UpstreamCommit() {
  131. Invoke-Expression "git fetch -q https://github.com/docker/docker.git refs/heads/master"
  132. if ($LASTEXITCODE -ne 0) { Throw "Failed fetching" }
  133. $upstream = Invoke-Expression "git rev-parse --verify FETCH_HEAD"
  134. if ($LASTEXITCODE -ne 0) { Throw "Failed getting upstream commit" }
  135. return $upstream
  136. }
  137. # Build a binary (client or daemon)
  138. Function Execute-Build($type, $additionalBuildTags, $directory) {
  139. # Generate the build flags
  140. $buildTags = "autogen"
  141. if ($Noisy) { $verboseParm=" -v" }
  142. if ($Race) { Write-Warning "Using race detector"; $raceParm=" -race"}
  143. if ($ForceBuildAll) { $allParm=" -a" }
  144. if ($NoOpt) { $optParm=" -gcflags "+""""+"-N -l"+"""" }
  145. if ($addtionalBuildTags -ne "") { $buildTags += $(" " + $additionalBuildTags) }
  146. # Do the go build in the appropriate directory
  147. # Note -linkmode=internal is required to be able to debug on Windows.
  148. # https://github.com/golang/go/issues/14319#issuecomment-189576638
  149. Write-Host "INFO: Building $type..."
  150. Push-Location $root\cmd\$directory; $global:pushed=$True
  151. $buildCommand = "go build" + `
  152. $raceParm + `
  153. $verboseParm + `
  154. $allParm + `
  155. $optParm + `
  156. " -tags """ + $buildTags + """" + `
  157. " -ldflags """ + "-linkmode=internal" + """" + `
  158. " -o $root\bundles\"+$directory+".exe"
  159. Invoke-Expression $buildCommand
  160. if ($LASTEXITCODE -ne 0) { Throw "Failed to compile $type" }
  161. Pop-Location; $global:pushed=$False
  162. }
  163. # Validates the DCO marker is present on each commit
  164. Function Validate-DCO($headCommit, $upstreamCommit) {
  165. Write-Host "INFO: Validating Developer Certificate of Origin..."
  166. # Username may only contain alphanumeric characters or dashes and cannot begin with a dash
  167. $usernameRegex='[a-zA-Z0-9][a-zA-Z0-9-]+'
  168. $dcoPrefix="Signed-off-by:"
  169. $dcoRegex="^(Docker-DCO-1.1-)?$dcoPrefix ([^<]+) <([^<>@]+@[^<>]+)>( \(github: ($usernameRegex)\))?$"
  170. $counts = Invoke-Expression "git diff --numstat $upstreamCommit...$headCommit"
  171. if ($LASTEXITCODE -ne 0) { Throw "Failed git diff --numstat" }
  172. # Counts of adds and deletes after removing multiple white spaces. AWK anyone? :(
  173. $adds=0; $dels=0; $($counts -replace '\s+', ' ') | %{
  174. $a=$_.Split(" ");
  175. if ($a[0] -ne "-") { $adds+=[int]$a[0] }
  176. if ($a[1] -ne "-") { $dels+=[int]$a[1] }
  177. }
  178. if (($adds -eq 0) -and ($dels -eq 0)) {
  179. Write-Warning "DCO validation - nothing to validate!"
  180. return
  181. }
  182. $commits = Invoke-Expression "git log $upstreamCommit..$headCommit --format=format:%H%n"
  183. if ($LASTEXITCODE -ne 0) { Throw "Failed git log --format" }
  184. $commits = $($commits -split '\s+' -match '\S')
  185. $badCommits=@()
  186. $commits | %{
  187. # Skip commits with no content such as merge commits etc
  188. if ($(git log -1 --format=format: --name-status $_).Length -gt 0) {
  189. # Ignore exit code on next call - always process regardless
  190. $commitMessage = Invoke-Expression "git log -1 --format=format:%B --name-status $_"
  191. if (($commitMessage -match $dcoRegex).Length -eq 0) { $badCommits+=$_ }
  192. }
  193. }
  194. if ($badCommits.Length -eq 0) {
  195. Write-Host "Congratulations! All commits are properly signed with the DCO!"
  196. } else {
  197. $e = "`nThese commits do not have a proper '$dcoPrefix' marker:`n"
  198. $badCommits | %{ $e+=" - $_`n"}
  199. $e += "`nPlease amend each commit to include a properly formatted DCO marker.`n`n"
  200. $e += "Visit the following URL for information about the Docker DCO:`n"
  201. $e += "https://github.com/docker/docker/blob/master/CONTRIBUTING.md#sign-your-work`n"
  202. Throw $e
  203. }
  204. }
  205. # Validates that .\pkg\... is safely isolated from internal code
  206. Function Validate-PkgImports($headCommit, $upstreamCommit) {
  207. Write-Host "INFO: Validating pkg import isolation..."
  208. # Get a list of go source-code files which have changed under pkg\. Ignore exit code on next call - always process regardless
  209. $files=@(); $files = Invoke-Expression "git diff $upstreamCommit...$headCommit --diff-filter=ACMR --name-only -- `'pkg\*.go`'"
  210. $badFiles=@(); $files | %{
  211. $file=$_
  212. # For the current changed file, get its list of dependencies, sorted and uniqued.
  213. $imports = Invoke-Expression "go list -e -f `'{{ .Deps }}`' $file"
  214. if ($LASTEXITCODE -ne 0) { Throw "Failed go list for dependencies on $file" }
  215. $imports = $imports -Replace "\[" -Replace "\]", "" -Split(" ") | Sort-Object | Get-Unique
  216. # Filter out what we are looking for
  217. $imports = $imports -NotMatch "^github.com/docker/docker/pkg/" `
  218. -NotMatch "^github.com/docker/docker/vendor" `
  219. -Match "^github.com/docker/docker" `
  220. -Replace "`n", ""
  221. $imports | % { $badFiles+="$file imports $_`n" }
  222. }
  223. if ($badFiles.Length -eq 0) {
  224. Write-Host 'Congratulations! ".\pkg\*.go" is safely isolated from internal code.'
  225. } else {
  226. $e = "`nThese files import internal code: (either directly or indirectly)`n"
  227. $badFiles | %{ $e+=" - $_"}
  228. Throw $e
  229. }
  230. }
  231. # Validates that changed files are correctly go-formatted
  232. Function Validate-GoFormat($headCommit, $upstreamCommit) {
  233. Write-Host "INFO: Validating go formatting on changed files..."
  234. # Verify gofmt is installed
  235. if ($(Get-Command gofmt -ErrorAction SilentlyContinue) -eq $nil) { Throw "gofmt does not appear to be installed" }
  236. # Get a list of all go source-code files which have changed. Ignore exit code on next call - always process regardless
  237. $files=@(); $files = Invoke-Expression "git diff $upstreamCommit...$headCommit --diff-filter=ACMR --name-only -- `'*.go`'"
  238. $files = $files | Select-String -NotMatch "^vendor/" | Select-String -NotMatch "^cli/compose/schema/bindata.go"
  239. $badFiles=@(); $files | %{
  240. # Deliberately ignore error on next line - treat as failed
  241. $content=Invoke-Expression "git show $headCommit`:$_"
  242. # Next set of hoops are to ensure we have LF not CRLF semantics as otherwise gofmt on Windows will not succeed.
  243. # Also note that gofmt on Windows does not appear to support stdin piping correctly. Hence go through a temporary file.
  244. $content=$content -join "`n"
  245. $content+="`n"
  246. $outputFile=[System.IO.Path]::GetTempFileName()
  247. if (Test-Path $outputFile) { Remove-Item $outputFile }
  248. [System.IO.File]::WriteAllText($outputFile, $content, (New-Object System.Text.UTF8Encoding($False)))
  249. $currentFile = $_ -Replace("/","\")
  250. Write-Host Checking $currentFile
  251. Invoke-Expression "gofmt -s -l $outputFile"
  252. if ($LASTEXITCODE -ne 0) { $badFiles+=$currentFile }
  253. if (Test-Path $outputFile) { Remove-Item $outputFile }
  254. }
  255. if ($badFiles.Length -eq 0) {
  256. Write-Host 'Congratulations! All Go source files are properly formatted.'
  257. } else {
  258. $e = "`nThese files are not properly gofmt`'d:`n"
  259. $badFiles | %{ $e+=" - $_`n"}
  260. $e+= "`nPlease reformat the above files using `"gofmt -s -w`" and commit the result."
  261. Throw $e
  262. }
  263. }
  264. # Run the unit tests
  265. Function Run-UnitTests() {
  266. Write-Host "INFO: Running unit tests..."
  267. $testPath="./..."
  268. $goListCommand = "go list -e -f '{{if ne .Name """ + '\"github.com/docker/docker\"' + """}}{{.ImportPath}}{{end}}' $testPath"
  269. $pkgList = $(Invoke-Expression $goListCommand)
  270. if ($LASTEXITCODE -ne 0) { Throw "go list for unit tests failed" }
  271. $pkgList = $pkgList | Select-String -Pattern "github.com/docker/docker"
  272. $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/vendor"
  273. $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/man"
  274. $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/integration-cli"
  275. $pkgList = $pkgList -replace "`r`n", " "
  276. $goTestCommand = "go test" + $raceParm + " -cover -ldflags -w -tags """ + "autogen daemon" + """ -a """ + "-test.timeout=10m" + """ $pkgList"
  277. Invoke-Expression $goTestCommand
  278. if ($LASTEXITCODE -ne 0) { Throw "Unit tests failed" }
  279. }
  280. # Start of main code.
  281. Try {
  282. Write-Host -ForegroundColor Cyan "INFO: make.ps1 starting at $(Get-Date)"
  283. # Get to the root of the repo
  284. $root = $(Split-Path $MyInvocation.MyCommand.Definition -Parent | Split-Path -Parent)
  285. Push-Location $root
  286. # Handle the "-All" shortcut to turn on all things we can handle.
  287. # Note we expressly only include the items which can run in a container - the validations tests cannot
  288. # as they require the .git directory which is excluded from the image by .dockerignore
  289. if ($All) { $Client=$True; $Daemon=$True; $TestUnit=$True }
  290. # Handle the "-Binary" shortcut to build both client and daemon.
  291. if ($Binary) { $Client = $True; $Daemon = $True }
  292. # Default to building the daemon if not asked for anything explicitly.
  293. if (-not($Client) -and -not($Daemon) -and -not($DCO) -and -not($PkgImports) -and -not($GoFormat) -and -not($TestUnit)) { $Daemon=$True }
  294. # Verify git is installed
  295. if ($(Get-Command git -ErrorAction SilentlyContinue) -eq $nil) { Throw "Git does not appear to be installed" }
  296. # Verify go is installed
  297. if ($(Get-Command go -ErrorAction SilentlyContinue) -eq $nil) { Throw "GoLang does not appear to be installed" }
  298. # Get the git commit. This will also verify if we are in a repo or not. Then add a custom string if supplied.
  299. $gitCommit=Get-GitCommit
  300. if ($CommitSuffix -ne "") { $gitCommit += "-"+$CommitSuffix -Replace ' ', '' }
  301. # Get the version of docker (eg 17.04.0-dev)
  302. $dockerVersion=Get-DockerVersion
  303. # Give a warning if we are not running in a container and are building binaries or running unit tests.
  304. # Not relevant for validation tests as these are fine to run outside of a container.
  305. if ($Client -or $Daemon -or $TestUnit) { $inContainer=Check-InContainer }
  306. # If we are not in a container, validate the version of GO that is installed.
  307. if (-not $inContainer) { Verify-GoVersion }
  308. # Verify GOPATH is set
  309. if ($env:GOPATH.Length -eq 0) { Throw "Missing GOPATH environment variable. See https://golang.org/doc/code.html#GOPATH" }
  310. # Run autogen if building binaries or running unit tests.
  311. if ($Client -or $Daemon -or $TestUnit) {
  312. Write-Host "INFO: Invoking autogen..."
  313. Try { .\hack\make\.go-autogen.ps1 -CommitString $gitCommit -DockerVersion $dockerVersion }
  314. Catch [Exception] { Throw $_ }
  315. }
  316. # DCO, Package import and Go formatting tests.
  317. if ($DCO -or $PkgImports -or $GoFormat) {
  318. # We need the head and upstream commits for these
  319. $headCommit=Get-HeadCommit
  320. $upstreamCommit=Get-UpstreamCommit
  321. # Run DCO validation
  322. if ($DCO) { Validate-DCO $headCommit $upstreamCommit }
  323. # Run `gofmt` validation
  324. if ($GoFormat) { Validate-GoFormat $headCommit $upstreamCommit }
  325. # Run pkg isolation validation
  326. if ($PkgImports) { Validate-PkgImports $headCommit $upstreamCommit }
  327. }
  328. # Build the binaries
  329. if ($Client -or $Daemon) {
  330. # Create the bundles directory if it doesn't exist
  331. if (-not (Test-Path ".\bundles")) { New-Item ".\bundles" -ItemType Directory | Out-Null }
  332. # Perform the actual build
  333. if ($Daemon) { Execute-Build "daemon" "daemon" "dockerd" }
  334. if ($Client) {
  335. # Get the repo and commit of the client to build.
  336. "hack\dockerfile\binaries-commits" | ForEach-Object {
  337. $dockerCliRepo = ((Get-Content $_ | Select-String "DOCKERCLI_REPO") -split "=")[1]
  338. $dockerCliCommit = ((Get-Content $_ | Select-String "DOCKERCLI_COMMIT") -split "=")[1]
  339. }
  340. # Build from a temporary directory.
  341. $tempLocation = "$env:TEMP\$(New-Guid)"
  342. New-Item -ItemType Directory $tempLocation | Out-Null
  343. # Temporarily override GOPATH, then clone, checkout, and build.
  344. $saveGOPATH = $env:GOPATH
  345. Try {
  346. $env:GOPATH = $tempLocation
  347. $dockerCliRoot = "$env:GOPATH\src\github.com\docker\cli"
  348. Write-Host "INFO: Cloning client repository..."
  349. Invoke-Expression "git clone -q $dockerCliRepo $dockerCliRoot"
  350. if ($LASTEXITCODE -ne 0) { Throw "Failed to clone client repository $dockerCliRepo" }
  351. Invoke-Expression "git -C $dockerCliRoot checkout -q $dockerCliCommit"
  352. if ($LASTEXITCODE -ne 0) { Throw "Failed to checkout client commit $dockerCliCommit" }
  353. Write-Host "INFO: Building client..."
  354. Push-Location "$dockerCliRoot\cmd\docker"; $global:pushed=$True
  355. Invoke-Expression "go build -o $root\bundles\docker.exe"
  356. if ($LASTEXITCODE -ne 0) { Throw "Failed to compile client" }
  357. Pop-Location; $global:pushed=$False
  358. }
  359. Catch [Exception] {
  360. Throw $_
  361. }
  362. Finally {
  363. # Always restore GOPATH and remove the temporary directory.
  364. $env:GOPATH = $saveGOPATH
  365. Remove-Item -Force -Recurse $tempLocation
  366. }
  367. }
  368. }
  369. # Run unit tests
  370. if ($TestUnit) { Run-UnitTests }
  371. # Gratuitous ASCII art.
  372. if ($Daemon -or $Client) {
  373. Write-Host
  374. Write-Host -ForegroundColor Green " ________ ____ __."
  375. Write-Host -ForegroundColor Green " \_____ \ `| `|/ _`|"
  376. Write-Host -ForegroundColor Green " / `| \`| `<"
  377. Write-Host -ForegroundColor Green " / `| \ `| \"
  378. Write-Host -ForegroundColor Green " \_______ /____`|__ \"
  379. Write-Host -ForegroundColor Green " \/ \/"
  380. Write-Host
  381. }
  382. }
  383. Catch [Exception] {
  384. Write-Host -ForegroundColor Red ("`nERROR: make.ps1 failed:`n$_")
  385. # More gratuitous ASCII art.
  386. Write-Host
  387. Write-Host -ForegroundColor Red "___________ .__.__ .___"
  388. Write-Host -ForegroundColor Red "\_ _____/____ `|__`| `| ____ __`| _/"
  389. Write-Host -ForegroundColor Red " `| __) \__ \ `| `| `| _/ __ \ / __ `| "
  390. Write-Host -ForegroundColor Red " `| \ / __ \`| `| `|_\ ___// /_/ `| "
  391. Write-Host -ForegroundColor Red " \___ / (____ /__`|____/\___ `>____ `| "
  392. Write-Host -ForegroundColor Red " \/ \/ \/ \/ "
  393. Write-Host
  394. Throw $_
  395. }
  396. Finally {
  397. Pop-Location # As we pushed to the root of the repo as the very first thing
  398. if ($global:pushed) { Pop-Location }
  399. Write-Host -ForegroundColor Cyan "INFO: make.ps1 ended at $(Get-Date)"
  400. }