docker_cli_daemon_test.go 96 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932
  1. //go:build linux
  2. // +build linux
  3. package main
  4. import (
  5. "bufio"
  6. "bytes"
  7. "context"
  8. "crypto/tls"
  9. "crypto/x509"
  10. "encoding/json"
  11. "fmt"
  12. "io"
  13. "net"
  14. "os"
  15. "os/exec"
  16. "path"
  17. "path/filepath"
  18. "regexp"
  19. "strconv"
  20. "strings"
  21. "sync"
  22. "testing"
  23. "time"
  24. "github.com/cloudflare/cfssl/helpers"
  25. "github.com/creack/pty"
  26. "github.com/docker/docker/api/types"
  27. "github.com/docker/docker/integration-cli/checker"
  28. "github.com/docker/docker/integration-cli/cli"
  29. "github.com/docker/docker/integration-cli/cli/build"
  30. "github.com/docker/docker/integration-cli/daemon"
  31. "github.com/docker/docker/libnetwork/iptables"
  32. "github.com/docker/docker/opts"
  33. testdaemon "github.com/docker/docker/testutil/daemon"
  34. units "github.com/docker/go-units"
  35. "github.com/docker/libtrust"
  36. "github.com/moby/sys/mount"
  37. "golang.org/x/sys/unix"
  38. "gotest.tools/v3/assert"
  39. is "gotest.tools/v3/assert/cmp"
  40. "gotest.tools/v3/icmd"
  41. "gotest.tools/v3/poll"
  42. )
  43. const containerdSocket = "/var/run/docker/containerd/containerd.sock"
  44. // TestLegacyDaemonCommand test starting docker daemon using "deprecated" docker daemon
  45. // command. Remove this test when we remove this.
  46. func (s *DockerDaemonSuite) TestLegacyDaemonCommand(c *testing.T) {
  47. cmd := exec.Command(dockerBinary, "daemon", "--storage-driver=vfs", "--debug")
  48. err := cmd.Start()
  49. go cmd.Wait()
  50. assert.NilError(c, err, "could not start daemon using 'docker daemon'")
  51. assert.NilError(c, cmd.Process.Kill())
  52. }
  53. func (s *DockerDaemonSuite) TestDaemonRestartWithRunningContainersPorts(c *testing.T) {
  54. s.d.StartWithBusybox(c)
  55. cli.Docker(
  56. cli.Args("run", "-d", "--name", "top1", "-p", "1234:80", "--restart", "always", "busybox:latest", "top"),
  57. cli.Daemon(s.d),
  58. ).Assert(c, icmd.Success)
  59. cli.Docker(
  60. cli.Args("run", "-d", "--name", "top2", "-p", "80", "busybox:latest", "top"),
  61. cli.Daemon(s.d),
  62. ).Assert(c, icmd.Success)
  63. testRun := func(m map[string]bool, prefix string) {
  64. var format string
  65. for cont, shouldRun := range m {
  66. out := cli.Docker(cli.Args("ps"), cli.Daemon(s.d)).Assert(c, icmd.Success).Combined()
  67. if shouldRun {
  68. format = "%scontainer %q is not running"
  69. } else {
  70. format = "%scontainer %q is running"
  71. }
  72. if shouldRun != strings.Contains(out, cont) {
  73. c.Fatalf(format, prefix, cont)
  74. }
  75. }
  76. }
  77. testRun(map[string]bool{"top1": true, "top2": true}, "")
  78. s.d.Restart(c)
  79. testRun(map[string]bool{"top1": true, "top2": false}, "After daemon restart: ")
  80. }
  81. func (s *DockerDaemonSuite) TestDaemonRestartWithVolumesRefs(c *testing.T) {
  82. s.d.StartWithBusybox(c)
  83. if out, err := s.d.Cmd("run", "--name", "volrestarttest1", "-v", "/foo", "busybox"); err != nil {
  84. c.Fatal(err, out)
  85. }
  86. s.d.Restart(c)
  87. if out, err := s.d.Cmd("run", "-d", "--volumes-from", "volrestarttest1", "--name", "volrestarttest2", "busybox", "top"); err != nil {
  88. c.Fatal(err, out)
  89. }
  90. if out, err := s.d.Cmd("rm", "-fv", "volrestarttest2"); err != nil {
  91. c.Fatal(err, out)
  92. }
  93. out, err := s.d.Cmd("inspect", "-f", "{{json .Mounts}}", "volrestarttest1")
  94. assert.NilError(c, err, out)
  95. if _, err := inspectMountPointJSON(out, "/foo"); err != nil {
  96. c.Fatalf("Expected volume to exist: /foo, error: %v\n", err)
  97. }
  98. }
  99. // #11008
  100. func (s *DockerDaemonSuite) TestDaemonRestartUnlessStopped(c *testing.T) {
  101. s.d.StartWithBusybox(c)
  102. out, err := s.d.Cmd("run", "-d", "--name", "top1", "--restart", "always", "busybox:latest", "top")
  103. assert.NilError(c, err, "run top1: %v", out)
  104. out, err = s.d.Cmd("run", "-d", "--name", "top2", "--restart", "unless-stopped", "busybox:latest", "top")
  105. assert.NilError(c, err, "run top2: %v", out)
  106. out, err = s.d.Cmd("run", "-d", "--name", "exit", "--restart", "unless-stopped", "busybox:latest", "false")
  107. assert.NilError(c, err, "run exit: %v", out)
  108. testRun := func(m map[string]bool, prefix string) {
  109. var format string
  110. for name, shouldRun := range m {
  111. out, err := s.d.Cmd("ps")
  112. assert.Assert(c, err == nil, "run ps: %v", out)
  113. if shouldRun {
  114. format = "%scontainer %q is not running"
  115. } else {
  116. format = "%scontainer %q is running"
  117. }
  118. assert.Equal(c, strings.Contains(out, name), shouldRun, fmt.Sprintf(format, prefix, name))
  119. }
  120. }
  121. // both running
  122. testRun(map[string]bool{"top1": true, "top2": true, "exit": true}, "")
  123. out, err = s.d.Cmd("stop", "exit")
  124. assert.NilError(c, err, out)
  125. out, err = s.d.Cmd("stop", "top1")
  126. assert.NilError(c, err, out)
  127. out, err = s.d.Cmd("stop", "top2")
  128. assert.NilError(c, err, out)
  129. // both stopped
  130. testRun(map[string]bool{"top1": false, "top2": false, "exit": false}, "")
  131. s.d.Restart(c)
  132. // restart=always running
  133. testRun(map[string]bool{"top1": true, "top2": false, "exit": false}, "After daemon restart: ")
  134. out, err = s.d.Cmd("start", "top2")
  135. assert.NilError(c, err, "start top2: %v", out)
  136. out, err = s.d.Cmd("start", "exit")
  137. assert.NilError(c, err, "start exit: %v", out)
  138. s.d.Restart(c)
  139. // both running
  140. testRun(map[string]bool{"top1": true, "top2": true, "exit": true}, "After second daemon restart: ")
  141. }
  142. func (s *DockerDaemonSuite) TestDaemonRestartOnFailure(c *testing.T) {
  143. s.d.StartWithBusybox(c)
  144. out, err := s.d.Cmd("run", "-d", "--name", "test1", "--restart", "on-failure:3", "busybox:latest", "false")
  145. assert.NilError(c, err, "run top1: %v", out)
  146. // wait test1 to stop
  147. hostArgs := []string{"--host", s.d.Sock()}
  148. err = waitInspectWithArgs("test1", "{{.State.Running}} {{.State.Restarting}}", "false false", 10*time.Second, hostArgs...)
  149. assert.NilError(c, err, "test1 should exit but not")
  150. // record last start time
  151. out, err = s.d.Cmd("inspect", "-f={{.State.StartedAt}}", "test1")
  152. assert.NilError(c, err, "out: %v", out)
  153. lastStartTime := out
  154. s.d.Restart(c)
  155. // test1 shouldn't restart at all
  156. err = waitInspectWithArgs("test1", "{{.State.Running}} {{.State.Restarting}}", "false false", 0, hostArgs...)
  157. assert.NilError(c, err, "test1 should exit but not")
  158. // make sure test1 isn't restarted when daemon restart
  159. // if "StartAt" time updates, means test1 was once restarted.
  160. out, err = s.d.Cmd("inspect", "-f={{.State.StartedAt}}", "test1")
  161. assert.NilError(c, err, "out: %v", out)
  162. assert.Equal(c, out, lastStartTime, "test1 shouldn't start after daemon restarts")
  163. }
  164. func (s *DockerDaemonSuite) TestDaemonStartIptablesFalse(c *testing.T) {
  165. s.d.Start(c, "--iptables=false")
  166. }
  167. // Make sure we cannot shrink base device at daemon restart.
  168. func (s *DockerDaemonSuite) TestDaemonRestartWithInvalidBasesize(c *testing.T) {
  169. testRequires(c, Devicemapper)
  170. s.d.Start(c)
  171. oldBasesizeBytes := getBaseDeviceSize(c, s.d)
  172. var newBasesizeBytes int64 = 1073741824 // 1GB in bytes
  173. if newBasesizeBytes < oldBasesizeBytes {
  174. err := s.d.RestartWithError("--storage-opt", fmt.Sprintf("dm.basesize=%d", newBasesizeBytes))
  175. assert.Assert(c, err != nil, "daemon should not have started as new base device size is less than existing base device size: %v", err)
  176. // 'err != nil' is expected behaviour, no new daemon started,
  177. // so no need to stop daemon.
  178. if err != nil {
  179. return
  180. }
  181. }
  182. s.d.Stop(c)
  183. }
  184. // Make sure we can grow base device at daemon restart.
  185. func (s *DockerDaemonSuite) TestDaemonRestartWithIncreasedBasesize(c *testing.T) {
  186. testRequires(c, Devicemapper)
  187. s.d.Start(c)
  188. oldBasesizeBytes := getBaseDeviceSize(c, s.d)
  189. var newBasesizeBytes int64 = 53687091200 // 50GB in bytes
  190. if newBasesizeBytes < oldBasesizeBytes {
  191. c.Skipf("New base device size (%v) must be greater than (%s)", units.HumanSize(float64(newBasesizeBytes)), units.HumanSize(float64(oldBasesizeBytes)))
  192. }
  193. err := s.d.RestartWithError("--storage-opt", fmt.Sprintf("dm.basesize=%d", newBasesizeBytes))
  194. assert.Assert(c, err == nil, "we should have been able to start the daemon with increased base device size: %v", err)
  195. basesizeAfterRestart := getBaseDeviceSize(c, s.d)
  196. newBasesize, err := convertBasesize(newBasesizeBytes)
  197. assert.Assert(c, err == nil, "Error in converting base device size: %v", err)
  198. assert.Equal(c, newBasesize, basesizeAfterRestart, "Basesize passed is not equal to Basesize set")
  199. s.d.Stop(c)
  200. }
  201. func getBaseDeviceSize(c *testing.T, d *daemon.Daemon) int64 {
  202. info := d.Info(c)
  203. for _, statusLine := range info.DriverStatus {
  204. key, value := statusLine[0], statusLine[1]
  205. if key == "Base Device Size" {
  206. return parseDeviceSize(c, value)
  207. }
  208. }
  209. c.Fatal("failed to parse Base Device Size from info")
  210. return int64(0)
  211. }
  212. func parseDeviceSize(c *testing.T, raw string) int64 {
  213. size, err := units.RAMInBytes(strings.TrimSpace(raw))
  214. assert.NilError(c, err)
  215. return size
  216. }
  217. func convertBasesize(basesizeBytes int64) (int64, error) {
  218. basesize := units.HumanSize(float64(basesizeBytes))
  219. basesize = strings.Trim(basesize, " ")[:len(basesize)-3]
  220. basesizeFloat, err := strconv.ParseFloat(strings.Trim(basesize, " "), 64)
  221. if err != nil {
  222. return 0, err
  223. }
  224. return int64(basesizeFloat) * 1024 * 1024 * 1024, nil
  225. }
  226. // Issue #8444: If docker0 bridge is modified (intentionally or unintentionally) and
  227. // no longer has an IP associated, we should gracefully handle that case and associate
  228. // an IP with it rather than fail daemon start
  229. func (s *DockerDaemonSuite) TestDaemonStartBridgeWithoutIPAssociation(c *testing.T) {
  230. // rather than depending on brctl commands to verify docker0 is created and up
  231. // let's start the daemon and stop it, and then make a modification to run the
  232. // actual test
  233. s.d.Start(c)
  234. s.d.Stop(c)
  235. // now we will remove the ip from docker0 and then try starting the daemon
  236. icmd.RunCommand("ip", "addr", "flush", "dev", "docker0").Assert(c, icmd.Success)
  237. if err := s.d.StartWithError(); err != nil {
  238. warning := "**WARNING: Docker bridge network in bad state--delete docker0 bridge interface to fix"
  239. c.Fatalf("Could not start daemon when docker0 has no IP address: %v\n%s", err, warning)
  240. }
  241. }
  242. func (s *DockerDaemonSuite) TestDaemonIptablesClean(c *testing.T) {
  243. s.d.StartWithBusybox(c)
  244. if out, err := s.d.Cmd("run", "-d", "--name", "top", "-p", "80", "busybox:latest", "top"); err != nil {
  245. c.Fatalf("Could not run top: %s, %v", out, err)
  246. }
  247. ipTablesSearchString := "tcp dpt:80"
  248. // get output from iptables with container running
  249. verifyIPTablesContains(c, ipTablesSearchString)
  250. s.d.Stop(c)
  251. // get output from iptables after restart
  252. verifyIPTablesDoesNotContains(c, ipTablesSearchString)
  253. }
  254. func (s *DockerDaemonSuite) TestDaemonIptablesCreate(c *testing.T) {
  255. s.d.StartWithBusybox(c)
  256. if out, err := s.d.Cmd("run", "-d", "--name", "top", "--restart=always", "-p", "80", "busybox:latest", "top"); err != nil {
  257. c.Fatalf("Could not run top: %s, %v", out, err)
  258. }
  259. // get output from iptables with container running
  260. ipTablesSearchString := "tcp dpt:80"
  261. verifyIPTablesContains(c, ipTablesSearchString)
  262. s.d.Restart(c)
  263. // make sure the container is not running
  264. runningOut, err := s.d.Cmd("inspect", "--format={{.State.Running}}", "top")
  265. if err != nil {
  266. c.Fatalf("Could not inspect on container: %s, %v", runningOut, err)
  267. }
  268. if strings.TrimSpace(runningOut) != "true" {
  269. c.Fatalf("Container should have been restarted after daemon restart. Status running should have been true but was: %q", strings.TrimSpace(runningOut))
  270. }
  271. // get output from iptables after restart
  272. verifyIPTablesContains(c, ipTablesSearchString)
  273. }
  274. func verifyIPTablesContains(c *testing.T, ipTablesSearchString string) {
  275. result := icmd.RunCommand("iptables", "-nvL")
  276. result.Assert(c, icmd.Success)
  277. if !strings.Contains(result.Combined(), ipTablesSearchString) {
  278. c.Fatalf("iptables output should have contained %q, but was %q", ipTablesSearchString, result.Combined())
  279. }
  280. }
  281. func verifyIPTablesDoesNotContains(c *testing.T, ipTablesSearchString string) {
  282. result := icmd.RunCommand("iptables", "-nvL")
  283. result.Assert(c, icmd.Success)
  284. if strings.Contains(result.Combined(), ipTablesSearchString) {
  285. c.Fatalf("iptables output should not have contained %q, but was %q", ipTablesSearchString, result.Combined())
  286. }
  287. }
  288. // TestDaemonIPv6Enabled checks that when the daemon is started with --ipv6=true that the docker0 bridge
  289. // has the fe80::1 address and that a container is assigned a link-local address
  290. func (s *DockerDaemonSuite) TestDaemonIPv6Enabled(c *testing.T) {
  291. testRequires(c, IPv6)
  292. setupV6(c)
  293. defer teardownV6(c)
  294. s.d.StartWithBusybox(c, "--ipv6")
  295. iface, err := net.InterfaceByName("docker0")
  296. if err != nil {
  297. c.Fatalf("Error getting docker0 interface: %v", err)
  298. }
  299. addrs, err := iface.Addrs()
  300. if err != nil {
  301. c.Fatalf("Error getting addresses for docker0 interface: %v", err)
  302. }
  303. var found bool
  304. expected := "fe80::1/64"
  305. for i := range addrs {
  306. if addrs[i].String() == expected {
  307. found = true
  308. break
  309. }
  310. }
  311. if !found {
  312. c.Fatalf("Bridge does not have an IPv6 Address")
  313. }
  314. if out, err := s.d.Cmd("run", "-itd", "--name=ipv6test", "busybox:latest"); err != nil {
  315. c.Fatalf("Could not run container: %s, %v", out, err)
  316. }
  317. out, err := s.d.Cmd("inspect", "--format", "'{{.NetworkSettings.Networks.bridge.LinkLocalIPv6Address}}'", "ipv6test")
  318. if err != nil {
  319. c.Fatalf("Error inspecting container: %s, %v", out, err)
  320. }
  321. out = strings.Trim(out, " \r\n'")
  322. if ip := net.ParseIP(out); ip == nil {
  323. c.Fatalf("Container should have a link-local IPv6 address")
  324. }
  325. out, err = s.d.Cmd("inspect", "--format", "'{{.NetworkSettings.Networks.bridge.GlobalIPv6Address}}'", "ipv6test")
  326. if err != nil {
  327. c.Fatalf("Error inspecting container: %s, %v", out, err)
  328. }
  329. out = strings.Trim(out, " \r\n'")
  330. if ip := net.ParseIP(out); ip != nil {
  331. c.Fatalf("Container should not have a global IPv6 address: %v", out)
  332. }
  333. }
  334. // TestDaemonIPv6FixedCIDR checks that when the daemon is started with --ipv6=true and a fixed CIDR
  335. // that running containers are given a link-local and global IPv6 address
  336. func (s *DockerDaemonSuite) TestDaemonIPv6FixedCIDR(c *testing.T) {
  337. // IPv6 setup is messing with local bridge address.
  338. testRequires(c, testEnv.IsLocalDaemon)
  339. // Delete the docker0 bridge if its left around from previous daemon. It has to be recreated with
  340. // ipv6 enabled
  341. deleteInterface(c, "docker0")
  342. s.d.StartWithBusybox(c, "--ipv6", "--fixed-cidr-v6=2001:db8:2::/64", "--default-gateway-v6=2001:db8:2::100")
  343. out, err := s.d.Cmd("run", "-d", "--name=ipv6test", "busybox:latest", "top")
  344. assert.NilError(c, err, "Could not run container: %s, %v", out, err)
  345. out, err = s.d.Cmd("inspect", "--format", "{{.NetworkSettings.Networks.bridge.GlobalIPv6Address}}", "ipv6test")
  346. assert.NilError(c, err, out)
  347. out = strings.Trim(out, " \r\n'")
  348. ip := net.ParseIP(out)
  349. assert.Assert(c, ip != nil, "Container should have a global IPv6 address")
  350. out, err = s.d.Cmd("inspect", "--format", "{{.NetworkSettings.Networks.bridge.IPv6Gateway}}", "ipv6test")
  351. assert.NilError(c, err, out)
  352. assert.Equal(c, strings.Trim(out, " \r\n'"), "2001:db8:2::100", "Container should have a global IPv6 gateway")
  353. }
  354. // TestDaemonIPv6FixedCIDRAndMac checks that when the daemon is started with ipv6 fixed CIDR
  355. // the running containers are given an IPv6 address derived from the MAC address and the ipv6 fixed CIDR
  356. func (s *DockerDaemonSuite) TestDaemonIPv6FixedCIDRAndMac(c *testing.T) {
  357. // IPv6 setup is messing with local bridge address.
  358. testRequires(c, testEnv.IsLocalDaemon)
  359. // Delete the docker0 bridge if its left around from previous daemon. It has to be recreated with
  360. // ipv6 enabled
  361. deleteInterface(c, "docker0")
  362. s.d.StartWithBusybox(c, "--ipv6", "--fixed-cidr-v6=2001:db8:1::/64")
  363. out, err := s.d.Cmd("run", "-d", "--name=ipv6test", "--mac-address", "AA:BB:CC:DD:EE:FF", "busybox", "top")
  364. assert.NilError(c, err, out)
  365. out, err = s.d.Cmd("inspect", "--format", "{{.NetworkSettings.Networks.bridge.GlobalIPv6Address}}", "ipv6test")
  366. assert.NilError(c, err, out)
  367. assert.Equal(c, strings.Trim(out, " \r\n'"), "2001:db8:1::aabb:ccdd:eeff")
  368. }
  369. // TestDaemonIPv6HostMode checks that when the running a container with
  370. // network=host the host ipv6 addresses are not removed
  371. func (s *DockerDaemonSuite) TestDaemonIPv6HostMode(c *testing.T) {
  372. testRequires(c, testEnv.IsLocalDaemon)
  373. deleteInterface(c, "docker0")
  374. s.d.StartWithBusybox(c, "--ipv6", "--fixed-cidr-v6=2001:db8:2::/64")
  375. out, err := s.d.Cmd("run", "-d", "--name=hostcnt", "--network=host", "busybox:latest", "top")
  376. assert.NilError(c, err, "Could not run container: %s, %v", out, err)
  377. out, err = s.d.Cmd("exec", "hostcnt", "ip", "-6", "addr", "show", "docker0")
  378. assert.NilError(c, err, out)
  379. assert.Assert(c, strings.Contains(strings.Trim(out, " \r\n'"), "2001:db8:2::1"))
  380. }
  381. func (s *DockerDaemonSuite) TestDaemonLogLevelWrong(c *testing.T) {
  382. assert.Assert(c, s.d.StartWithError("--log-level=bogus") != nil, "Daemon shouldn't start with wrong log level")
  383. }
  384. func (s *DockerDaemonSuite) TestDaemonLogLevelDebug(c *testing.T) {
  385. s.d.Start(c, "--log-level=debug")
  386. content, err := s.d.ReadLogFile()
  387. assert.NilError(c, err)
  388. if !strings.Contains(string(content), `level=debug`) {
  389. c.Fatalf(`Missing level="debug" in log file:\n%s`, string(content))
  390. }
  391. }
  392. func (s *DockerDaemonSuite) TestDaemonLogLevelFatal(c *testing.T) {
  393. // we creating new daemons to create new logFile
  394. s.d.Start(c, "--log-level=fatal")
  395. content, err := s.d.ReadLogFile()
  396. assert.NilError(c, err)
  397. if strings.Contains(string(content), `level=debug`) {
  398. c.Fatalf(`Should not have level="debug" in log file:\n%s`, string(content))
  399. }
  400. }
  401. func (s *DockerDaemonSuite) TestDaemonFlagD(c *testing.T) {
  402. s.d.Start(c, "-D")
  403. content, err := s.d.ReadLogFile()
  404. assert.NilError(c, err)
  405. if !strings.Contains(string(content), `level=debug`) {
  406. c.Fatalf(`Should have level="debug" in log file using -D:\n%s`, string(content))
  407. }
  408. }
  409. func (s *DockerDaemonSuite) TestDaemonFlagDebug(c *testing.T) {
  410. s.d.Start(c, "--debug")
  411. content, err := s.d.ReadLogFile()
  412. assert.NilError(c, err)
  413. if !strings.Contains(string(content), `level=debug`) {
  414. c.Fatalf(`Should have level="debug" in log file using --debug:\n%s`, string(content))
  415. }
  416. }
  417. func (s *DockerDaemonSuite) TestDaemonFlagDebugLogLevelFatal(c *testing.T) {
  418. s.d.Start(c, "--debug", "--log-level=fatal")
  419. content, err := s.d.ReadLogFile()
  420. assert.NilError(c, err)
  421. if !strings.Contains(string(content), `level=debug`) {
  422. c.Fatalf(`Should have level="debug" in log file when using both --debug and --log-level=fatal:\n%s`, string(content))
  423. }
  424. }
  425. func (s *DockerDaemonSuite) TestDaemonAllocatesListeningPort(c *testing.T) {
  426. type listener struct {
  427. daemon string
  428. client string
  429. port string
  430. }
  431. listeningPorts := []listener{
  432. {"0.0.0.0", "0.0.0.0", "5678"},
  433. {"127.0.0.1", "127.0.0.1", "1234"},
  434. {"localhost", "127.0.0.1", "1235"},
  435. }
  436. cmdArgs := make([]string, 0, len(listeningPorts)*2)
  437. for _, l := range listeningPorts {
  438. cmdArgs = append(cmdArgs, "--tls=false", "--host", "tcp://"+net.JoinHostPort(l.daemon, l.port))
  439. }
  440. s.d.StartWithBusybox(c, cmdArgs...)
  441. for _, l := range listeningPorts {
  442. output, err := s.d.Cmd("run", "-p", fmt.Sprintf("%s:%s:80", l.client, l.port), "busybox", "true")
  443. if err == nil {
  444. c.Fatalf("Container should not start, expected port already allocated error: %q", output)
  445. } else if !strings.Contains(output, "port is already allocated") {
  446. c.Fatalf("Expected port is already allocated error: %q", output)
  447. }
  448. }
  449. }
  450. func (s *DockerDaemonSuite) TestDaemonKeyGeneration(c *testing.T) {
  451. // TODO: skip or update for Windows daemon
  452. os.Remove("/etc/docker/key.json")
  453. c.Setenv("DOCKER_ALLOW_SCHEMA1_PUSH_DONOTUSE", "1")
  454. s.d.Start(c)
  455. s.d.Stop(c)
  456. k, err := libtrust.LoadKeyFile("/etc/docker/key.json")
  457. if err != nil {
  458. c.Fatalf("Error opening key file")
  459. }
  460. kid := k.KeyID()
  461. // Test Key ID is a valid fingerprint (e.g. QQXN:JY5W:TBXI:MK3X:GX6P:PD5D:F56N:NHCS:LVRZ:JA46:R24J:XEFF)
  462. if len(kid) != 59 {
  463. c.Fatalf("Bad key ID: %s", kid)
  464. }
  465. }
  466. // GH#11320 - verify that the daemon exits on failure properly
  467. // Note that this explicitly tests the conflict of {-b,--bridge} and {--bip} options as the means
  468. // to get a daemon init failure; no other tests for -b/--bip conflict are therefore required
  469. func (s *DockerDaemonSuite) TestDaemonExitOnFailure(c *testing.T) {
  470. // attempt to start daemon with incorrect flags (we know -b and --bip conflict)
  471. if err := s.d.StartWithError("--bridge", "nosuchbridge", "--bip", "1.1.1.1"); err != nil {
  472. // verify we got the right error
  473. if !strings.Contains(err.Error(), "daemon exited") {
  474. c.Fatalf("Expected daemon not to start, got %v", err)
  475. }
  476. // look in the log and make sure we got the message that daemon is shutting down
  477. icmd.RunCommand("grep", "failed to start daemon", s.d.LogFileName()).Assert(c, icmd.Success)
  478. } else {
  479. // if we didn't get an error and the daemon is running, this is a failure
  480. c.Fatal("Conflicting options should cause the daemon to error out with a failure")
  481. }
  482. }
  483. func (s *DockerDaemonSuite) TestDaemonBridgeExternal(c *testing.T) {
  484. d := s.d
  485. err := d.StartWithError("--bridge", "nosuchbridge")
  486. assert.ErrorContains(c, err, "", `--bridge option with an invalid bridge should cause the daemon to fail`)
  487. defer d.Restart(c)
  488. // make sure the default docker0 bridge doesn't interfere with the test,
  489. // which may happen if it was created with the same IP range.
  490. deleteInterface(c, "docker0")
  491. bridgeName := "ext-bridge1"
  492. bridgeIP := "192.169.1.1/24"
  493. _, bridgeIPNet, _ := net.ParseCIDR(bridgeIP)
  494. createInterface(c, "bridge", bridgeName, bridgeIP)
  495. defer deleteInterface(c, bridgeName)
  496. d.StartWithBusybox(c, "--bridge", bridgeName)
  497. ipTablesSearchString := bridgeIPNet.String()
  498. icmd.RunCommand("iptables", "-t", "nat", "-nvL").Assert(c, icmd.Expected{
  499. Out: ipTablesSearchString,
  500. })
  501. out, err := d.Cmd("run", "-d", "--name", "ExtContainer", "busybox", "top")
  502. assert.NilError(c, err, out)
  503. containerIP := d.FindContainerIP(c, "ExtContainer")
  504. ip := net.ParseIP(containerIP)
  505. assert.Assert(c, bridgeIPNet.Contains(ip), "Container IP-Address must be in the same subnet range : %s", containerIP)
  506. }
  507. func (s *DockerDaemonSuite) TestDaemonBridgeNone(c *testing.T) {
  508. // start with bridge none
  509. d := s.d
  510. d.StartWithBusybox(c, "--bridge", "none")
  511. defer d.Restart(c)
  512. // verify docker0 iface is not there
  513. icmd.RunCommand("ifconfig", "docker0").Assert(c, icmd.Expected{
  514. ExitCode: 1,
  515. Error: "exit status 1",
  516. Err: "Device not found",
  517. })
  518. // verify default "bridge" network is not there
  519. out, err := d.Cmd("network", "inspect", "bridge")
  520. assert.ErrorContains(c, err, "", `"bridge" network should not be present if daemon started with --bridge=none`)
  521. assert.Assert(c, strings.Contains(out, "No such network"))
  522. }
  523. func createInterface(c *testing.T, ifType string, ifName string, ipNet string) {
  524. icmd.RunCommand("ip", "link", "add", "name", ifName, "type", ifType).Assert(c, icmd.Success)
  525. icmd.RunCommand("ifconfig", ifName, ipNet, "up").Assert(c, icmd.Success)
  526. }
  527. func deleteInterface(c *testing.T, ifName string) {
  528. icmd.RunCommand("ip", "link", "delete", ifName).Assert(c, icmd.Success)
  529. icmd.RunCommand("iptables", "-t", "nat", "--flush").Assert(c, icmd.Success)
  530. icmd.RunCommand("iptables", "--flush").Assert(c, icmd.Success)
  531. }
  532. func (s *DockerDaemonSuite) TestDaemonBridgeIP(c *testing.T) {
  533. // TestDaemonBridgeIP Steps
  534. // 1. Delete the existing docker0 Bridge
  535. // 2. Set --bip daemon configuration and start the new Docker Daemon
  536. // 3. Check if the bip config has taken effect using ifconfig and iptables commands
  537. // 4. Launch a Container and make sure the IP-Address is in the expected subnet
  538. // 5. Delete the docker0 Bridge
  539. // 6. Restart the Docker Daemon (via deferred action)
  540. // This Restart takes care of bringing docker0 interface back to auto-assigned IP
  541. defaultNetworkBridge := "docker0"
  542. deleteInterface(c, defaultNetworkBridge)
  543. d := s.d
  544. bridgeIP := "192.169.1.1/24"
  545. ip, bridgeIPNet, _ := net.ParseCIDR(bridgeIP)
  546. d.StartWithBusybox(c, "--bip", bridgeIP)
  547. defer d.Restart(c)
  548. ifconfigSearchString := ip.String()
  549. icmd.RunCommand("ifconfig", defaultNetworkBridge).Assert(c, icmd.Expected{
  550. Out: ifconfigSearchString,
  551. })
  552. ipTablesSearchString := bridgeIPNet.String()
  553. icmd.RunCommand("iptables", "-t", "nat", "-nvL").Assert(c, icmd.Expected{
  554. Out: ipTablesSearchString,
  555. })
  556. out, err := d.Cmd("run", "-d", "--name", "test", "busybox", "top")
  557. assert.NilError(c, err, out)
  558. containerIP := d.FindContainerIP(c, "test")
  559. ip = net.ParseIP(containerIP)
  560. assert.Equal(c, bridgeIPNet.Contains(ip), true, fmt.Sprintf("Container IP-Address must be in the same subnet range : %s", containerIP))
  561. deleteInterface(c, defaultNetworkBridge)
  562. }
  563. func (s *DockerDaemonSuite) TestDaemonRestartWithBridgeIPChange(c *testing.T) {
  564. s.d.Start(c)
  565. defer s.d.Restart(c)
  566. s.d.Stop(c)
  567. // now we will change the docker0's IP and then try starting the daemon
  568. bridgeIP := "192.169.100.1/24"
  569. _, bridgeIPNet, _ := net.ParseCIDR(bridgeIP)
  570. icmd.RunCommand("ifconfig", "docker0", bridgeIP).Assert(c, icmd.Success)
  571. s.d.Start(c, "--bip", bridgeIP)
  572. // check if the iptables contains new bridgeIP MASQUERADE rule
  573. ipTablesSearchString := bridgeIPNet.String()
  574. icmd.RunCommand("iptables", "-t", "nat", "-nvL").Assert(c, icmd.Expected{
  575. Out: ipTablesSearchString,
  576. })
  577. }
  578. func (s *DockerDaemonSuite) TestDaemonBridgeFixedCidr(c *testing.T) {
  579. d := s.d
  580. // make sure the default docker0 bridge doesn't interfere with the test,
  581. // which may happen if it was created with the same IP range.
  582. deleteInterface(c, "docker0")
  583. bridgeName := "ext-bridge2"
  584. bridgeIP := "192.169.1.1/24"
  585. createInterface(c, "bridge", bridgeName, bridgeIP)
  586. defer deleteInterface(c, bridgeName)
  587. args := []string{"--bridge", bridgeName, "--fixed-cidr", "192.169.1.0/30"}
  588. d.StartWithBusybox(c, args...)
  589. defer d.Restart(c)
  590. for i := 0; i < 4; i++ {
  591. cName := "Container" + strconv.Itoa(i)
  592. out, err := d.Cmd("run", "-d", "--name", cName, "busybox", "top")
  593. if err != nil {
  594. assert.Assert(c, strings.Contains(out, "no available IPv4 addresses"), "Could not run a Container : %s %s", err.Error(), out)
  595. }
  596. }
  597. }
  598. func (s *DockerDaemonSuite) TestDaemonBridgeFixedCidr2(c *testing.T) {
  599. d := s.d
  600. // make sure the default docker0 bridge doesn't interfere with the test,
  601. // which may happen if it was created with the same IP range.
  602. deleteInterface(c, "docker0")
  603. bridgeName := "ext-bridge3"
  604. bridgeIP := "10.2.2.1/16"
  605. createInterface(c, "bridge", bridgeName, bridgeIP)
  606. defer deleteInterface(c, bridgeName)
  607. d.StartWithBusybox(c, "--bip", bridgeIP, "--fixed-cidr", "10.2.2.0/24")
  608. defer s.d.Restart(c)
  609. out, err := d.Cmd("run", "-d", "--name", "bb", "busybox", "top")
  610. assert.NilError(c, err, out)
  611. defer d.Cmd("stop", "bb")
  612. out, err = d.Cmd("exec", "bb", "/bin/sh", "-c", "ifconfig eth0 | awk '/inet addr/{print substr($2,6)}'")
  613. assert.NilError(c, err)
  614. assert.Equal(c, out, "10.2.2.0\n")
  615. out, err = d.Cmd("run", "--rm", "busybox", "/bin/sh", "-c", "ifconfig eth0 | awk '/inet addr/{print substr($2,6)}'")
  616. assert.NilError(c, err, out)
  617. assert.Equal(c, out, "10.2.2.2\n")
  618. }
  619. func (s *DockerDaemonSuite) TestDaemonBridgeFixedCIDREqualBridgeNetwork(c *testing.T) {
  620. d := s.d
  621. // make sure the default docker0 bridge doesn't interfere with the test,
  622. // which may happen if it was created with the same IP range.
  623. deleteInterface(c, "docker0")
  624. bridgeName := "ext-bridge4"
  625. bridgeIP := "172.27.42.1/16"
  626. createInterface(c, "bridge", bridgeName, bridgeIP)
  627. defer deleteInterface(c, bridgeName)
  628. d.StartWithBusybox(c, "--bridge", bridgeName, "--fixed-cidr", bridgeIP)
  629. defer s.d.Restart(c)
  630. out, err := d.Cmd("run", "-d", "busybox", "top")
  631. assert.NilError(c, err, out)
  632. cid1 := strings.TrimSpace(out)
  633. defer d.Cmd("stop", cid1)
  634. }
  635. func (s *DockerDaemonSuite) TestDaemonDefaultGatewayIPv4Implicit(c *testing.T) {
  636. defaultNetworkBridge := "docker0"
  637. deleteInterface(c, defaultNetworkBridge)
  638. d := s.d
  639. bridgeIP := "192.169.1.1"
  640. bridgeIPNet := fmt.Sprintf("%s/24", bridgeIP)
  641. d.StartWithBusybox(c, "--bip", bridgeIPNet)
  642. defer d.Restart(c)
  643. expectedMessage := fmt.Sprintf("default via %s dev", bridgeIP)
  644. out, err := d.Cmd("run", "busybox", "ip", "-4", "route", "list", "0/0")
  645. assert.NilError(c, err, out)
  646. assert.Equal(c, strings.Contains(out, expectedMessage), true, fmt.Sprintf("Implicit default gateway should be bridge IP %s, but default route was '%s'", bridgeIP, strings.TrimSpace(out)))
  647. deleteInterface(c, defaultNetworkBridge)
  648. }
  649. func (s *DockerDaemonSuite) TestDaemonDefaultGatewayIPv4Explicit(c *testing.T) {
  650. defaultNetworkBridge := "docker0"
  651. deleteInterface(c, defaultNetworkBridge)
  652. d := s.d
  653. bridgeIP := "192.169.1.1"
  654. bridgeIPNet := fmt.Sprintf("%s/24", bridgeIP)
  655. gatewayIP := "192.169.1.254"
  656. d.StartWithBusybox(c, "--bip", bridgeIPNet, "--default-gateway", gatewayIP)
  657. defer d.Restart(c)
  658. expectedMessage := fmt.Sprintf("default via %s dev", gatewayIP)
  659. out, err := d.Cmd("run", "busybox", "ip", "-4", "route", "list", "0/0")
  660. assert.NilError(c, err, out)
  661. assert.Equal(c, strings.Contains(out, expectedMessage), true, fmt.Sprintf("Explicit default gateway should be %s, but default route was '%s'", gatewayIP, strings.TrimSpace(out)))
  662. deleteInterface(c, defaultNetworkBridge)
  663. }
  664. func (s *DockerDaemonSuite) TestDaemonDefaultGatewayIPv4ExplicitOutsideContainerSubnet(c *testing.T) {
  665. defaultNetworkBridge := "docker0"
  666. deleteInterface(c, defaultNetworkBridge)
  667. // Program a custom default gateway outside of the container subnet, daemon should accept it and start
  668. s.d.StartWithBusybox(c, "--bip", "172.16.0.10/16", "--fixed-cidr", "172.16.1.0/24", "--default-gateway", "172.16.0.254")
  669. deleteInterface(c, defaultNetworkBridge)
  670. s.d.Restart(c)
  671. }
  672. func (s *DockerDaemonSuite) TestDaemonIP(c *testing.T) {
  673. d := s.d
  674. // make sure the default docker0 bridge doesn't interfere with the test,
  675. // which may happen if it was created with the same IP range.
  676. deleteInterface(c, "docker0")
  677. ipStr := "192.170.1.1/24"
  678. ip, _, _ := net.ParseCIDR(ipStr)
  679. args := []string{"--ip", ip.String()}
  680. d.StartWithBusybox(c, args...)
  681. defer d.Restart(c)
  682. out, err := d.Cmd("run", "-d", "-p", "8000:8000", "busybox", "top")
  683. assert.Assert(c, err != nil, "Running a container must fail with an invalid --ip option")
  684. assert.Equal(c, strings.Contains(out, "Error starting userland proxy"), true)
  685. ifName := "dummy"
  686. createInterface(c, "dummy", ifName, ipStr)
  687. defer deleteInterface(c, ifName)
  688. _, err = d.Cmd("run", "-d", "-p", "8000:8000", "busybox", "top")
  689. assert.NilError(c, err, out)
  690. result := icmd.RunCommand("iptables", "-t", "nat", "-nvL")
  691. result.Assert(c, icmd.Success)
  692. regex := fmt.Sprintf("DNAT.*%s.*dpt:8000", ip.String())
  693. matched, _ := regexp.MatchString(regex, result.Combined())
  694. assert.Equal(c, matched, true, fmt.Sprintf("iptables output should have contained %q, but was %q", regex, result.Combined()))
  695. }
  696. func (s *DockerDaemonSuite) TestDaemonICCPing(c *testing.T) {
  697. testRequires(c, bridgeNfIptables)
  698. d := s.d
  699. // make sure the default docker0 bridge doesn't interfere with the test,
  700. // which may happen if it was created with the same IP range.
  701. deleteInterface(c, "docker0")
  702. bridgeName := "ext-bridge5"
  703. bridgeIP := "192.169.1.1/24"
  704. createInterface(c, "bridge", bridgeName, bridgeIP)
  705. defer deleteInterface(c, bridgeName)
  706. d.StartWithBusybox(c, "--bridge", bridgeName, "--icc=false")
  707. defer d.Restart(c)
  708. result := icmd.RunCommand("iptables", "-nvL", "FORWARD")
  709. result.Assert(c, icmd.Success)
  710. regex := fmt.Sprintf("DROP.*all.*%s.*%s", bridgeName, bridgeName)
  711. matched, _ := regexp.MatchString(regex, result.Combined())
  712. assert.Equal(c, matched, true, fmt.Sprintf("iptables output should have contained %q, but was %q", regex, result.Combined()))
  713. // Pinging another container must fail with --icc=false
  714. pingContainers(c, d, true)
  715. ipStr := "192.171.1.1/24"
  716. ip, _, _ := net.ParseCIDR(ipStr)
  717. ifName := "icc-dummy"
  718. createInterface(c, "dummy", ifName, ipStr)
  719. defer deleteInterface(c, ifName)
  720. // But, Pinging external or a Host interface must succeed
  721. pingCmd := fmt.Sprintf("ping -c 1 %s -W 1", ip.String())
  722. runArgs := []string{"run", "--rm", "busybox", "sh", "-c", pingCmd}
  723. out, err := d.Cmd(runArgs...)
  724. assert.NilError(c, err, out)
  725. }
  726. func (s *DockerDaemonSuite) TestDaemonICCLinkExpose(c *testing.T) {
  727. d := s.d
  728. // make sure the default docker0 bridge doesn't interfere with the test,
  729. // which may happen if it was created with the same IP range.
  730. deleteInterface(c, "docker0")
  731. bridgeName := "ext-bridge6"
  732. bridgeIP := "192.169.1.1/24"
  733. createInterface(c, "bridge", bridgeName, bridgeIP)
  734. defer deleteInterface(c, bridgeName)
  735. d.StartWithBusybox(c, "--bridge", bridgeName, "--icc=false")
  736. defer d.Restart(c)
  737. result := icmd.RunCommand("iptables", "-nvL", "FORWARD")
  738. result.Assert(c, icmd.Success)
  739. regex := fmt.Sprintf("DROP.*all.*%s.*%s", bridgeName, bridgeName)
  740. matched, _ := regexp.MatchString(regex, result.Combined())
  741. assert.Equal(c, matched, true, fmt.Sprintf("iptables output should have contained %q, but was %q", regex, result.Combined()))
  742. out, err := d.Cmd("run", "-d", "--expose", "4567", "--name", "icc1", "busybox", "nc", "-l", "-p", "4567")
  743. assert.NilError(c, err, out)
  744. out, err = d.Cmd("run", "--link", "icc1:icc1", "busybox", "nc", "icc1", "4567")
  745. assert.NilError(c, err, out)
  746. }
  747. func (s *DockerDaemonSuite) TestDaemonLinksIpTablesRulesWhenLinkAndUnlink(c *testing.T) {
  748. // make sure the default docker0 bridge doesn't interfere with the test,
  749. // which may happen if it was created with the same IP range.
  750. deleteInterface(c, "docker0")
  751. bridgeName := "ext-bridge7"
  752. bridgeIP := "192.169.1.1/24"
  753. createInterface(c, "bridge", bridgeName, bridgeIP)
  754. defer deleteInterface(c, bridgeName)
  755. s.d.StartWithBusybox(c, "--bridge", bridgeName, "--icc=false")
  756. defer s.d.Restart(c)
  757. out, err := s.d.Cmd("run", "-d", "--name", "child", "--publish", "8080:80", "busybox", "top")
  758. assert.NilError(c, err, out)
  759. out, err = s.d.Cmd("run", "-d", "--name", "parent", "--link", "child:http", "busybox", "top")
  760. assert.NilError(c, err, out)
  761. childIP := s.d.FindContainerIP(c, "child")
  762. parentIP := s.d.FindContainerIP(c, "parent")
  763. sourceRule := []string{"-i", bridgeName, "-o", bridgeName, "-p", "tcp", "-s", childIP, "--sport", "80", "-d", parentIP, "-j", "ACCEPT"}
  764. destinationRule := []string{"-i", bridgeName, "-o", bridgeName, "-p", "tcp", "-s", parentIP, "--dport", "80", "-d", childIP, "-j", "ACCEPT"}
  765. iptable := iptables.GetIptable(iptables.IPv4)
  766. if !iptable.Exists("filter", "DOCKER", sourceRule...) || !iptable.Exists("filter", "DOCKER", destinationRule...) {
  767. c.Fatal("Iptables rules not found")
  768. }
  769. s.d.Cmd("rm", "--link", "parent/http")
  770. if iptable.Exists("filter", "DOCKER", sourceRule...) || iptable.Exists("filter", "DOCKER", destinationRule...) {
  771. c.Fatal("Iptables rules should be removed when unlink")
  772. }
  773. s.d.Cmd("kill", "child")
  774. s.d.Cmd("kill", "parent")
  775. }
  776. func (s *DockerDaemonSuite) TestDaemonUlimitDefaults(c *testing.T) {
  777. s.d.StartWithBusybox(c, "--default-ulimit", "nofile=42:42", "--default-ulimit", "nproc=1024:1024")
  778. out, err := s.d.Cmd("run", "--ulimit", "nproc=2048", "--name=test", "busybox", "/bin/sh", "-c", "echo $(ulimit -n); echo $(ulimit -u)")
  779. if err != nil {
  780. c.Fatal(err, out)
  781. }
  782. outArr := strings.Split(out, "\n")
  783. if len(outArr) < 2 {
  784. c.Fatalf("got unexpected output: %s", out)
  785. }
  786. nofile := strings.TrimSpace(outArr[0])
  787. nproc := strings.TrimSpace(outArr[1])
  788. if nofile != "42" {
  789. c.Fatalf("expected `ulimit -n` to be `42`, got: %s", nofile)
  790. }
  791. if nproc != "2048" {
  792. c.Fatalf("expected `ulimit -u` to be 2048, got: %s", nproc)
  793. }
  794. // Now restart daemon with a new default
  795. s.d.Restart(c, "--default-ulimit", "nofile=43")
  796. out, err = s.d.Cmd("start", "-a", "test")
  797. if err != nil {
  798. c.Fatal(err, out)
  799. }
  800. outArr = strings.Split(out, "\n")
  801. if len(outArr) < 2 {
  802. c.Fatalf("got unexpected output: %s", out)
  803. }
  804. nofile = strings.TrimSpace(outArr[0])
  805. nproc = strings.TrimSpace(outArr[1])
  806. if nofile != "43" {
  807. c.Fatalf("expected `ulimit -n` to be `43`, got: %s", nofile)
  808. }
  809. if nproc != "2048" {
  810. c.Fatalf("expected `ulimit -u` to be 2048, got: %s", nproc)
  811. }
  812. }
  813. // #11315
  814. func (s *DockerDaemonSuite) TestDaemonRestartRenameContainer(c *testing.T) {
  815. s.d.StartWithBusybox(c)
  816. if out, err := s.d.Cmd("run", "--name=test", "busybox"); err != nil {
  817. c.Fatal(err, out)
  818. }
  819. if out, err := s.d.Cmd("rename", "test", "test2"); err != nil {
  820. c.Fatal(err, out)
  821. }
  822. s.d.Restart(c)
  823. if out, err := s.d.Cmd("start", "test2"); err != nil {
  824. c.Fatal(err, out)
  825. }
  826. }
  827. func (s *DockerDaemonSuite) TestDaemonLoggingDriverDefault(c *testing.T) {
  828. s.d.StartWithBusybox(c)
  829. out, err := s.d.Cmd("run", "--name=test", "busybox", "echo", "testline")
  830. assert.NilError(c, err, out)
  831. id, err := s.d.GetIDByName("test")
  832. assert.NilError(c, err)
  833. logPath := filepath.Join(s.d.Root, "containers", id, id+"-json.log")
  834. if _, err := os.Stat(logPath); err != nil {
  835. c.Fatal(err)
  836. }
  837. f, err := os.Open(logPath)
  838. if err != nil {
  839. c.Fatal(err)
  840. }
  841. defer f.Close()
  842. var res struct {
  843. Log string `json:"log"`
  844. Stream string `json:"stream"`
  845. Time time.Time `json:"time"`
  846. }
  847. if err := json.NewDecoder(f).Decode(&res); err != nil {
  848. c.Fatal(err)
  849. }
  850. if res.Log != "testline\n" {
  851. c.Fatalf("Unexpected log line: %q, expected: %q", res.Log, "testline\n")
  852. }
  853. if res.Stream != "stdout" {
  854. c.Fatalf("Unexpected stream: %q, expected: %q", res.Stream, "stdout")
  855. }
  856. if !time.Now().After(res.Time) {
  857. c.Fatalf("Log time %v in future", res.Time)
  858. }
  859. }
  860. func (s *DockerDaemonSuite) TestDaemonLoggingDriverDefaultOverride(c *testing.T) {
  861. s.d.StartWithBusybox(c)
  862. out, err := s.d.Cmd("run", "--name=test", "--log-driver=none", "busybox", "echo", "testline")
  863. if err != nil {
  864. c.Fatal(out, err)
  865. }
  866. id, err := s.d.GetIDByName("test")
  867. assert.NilError(c, err)
  868. logPath := filepath.Join(s.d.Root, "containers", id, id+"-json.log")
  869. if _, err := os.Stat(logPath); err == nil || !os.IsNotExist(err) {
  870. c.Fatalf("%s shouldn't exits, error on Stat: %s", logPath, err)
  871. }
  872. }
  873. func (s *DockerDaemonSuite) TestDaemonLoggingDriverNone(c *testing.T) {
  874. s.d.StartWithBusybox(c, "--log-driver=none")
  875. out, err := s.d.Cmd("run", "--name=test", "busybox", "echo", "testline")
  876. if err != nil {
  877. c.Fatal(out, err)
  878. }
  879. id, err := s.d.GetIDByName("test")
  880. assert.NilError(c, err)
  881. logPath := filepath.Join(s.d.Root, "containers", id, id+"-json.log")
  882. if _, err := os.Stat(logPath); err == nil || !os.IsNotExist(err) {
  883. c.Fatalf("%s shouldn't exits, error on Stat: %s", logPath, err)
  884. }
  885. }
  886. func (s *DockerDaemonSuite) TestDaemonLoggingDriverNoneOverride(c *testing.T) {
  887. s.d.StartWithBusybox(c, "--log-driver=none")
  888. out, err := s.d.Cmd("run", "--name=test", "--log-driver=json-file", "busybox", "echo", "testline")
  889. if err != nil {
  890. c.Fatal(out, err)
  891. }
  892. id, err := s.d.GetIDByName("test")
  893. assert.NilError(c, err)
  894. logPath := filepath.Join(s.d.Root, "containers", id, id+"-json.log")
  895. if _, err := os.Stat(logPath); err != nil {
  896. c.Fatal(err)
  897. }
  898. f, err := os.Open(logPath)
  899. if err != nil {
  900. c.Fatal(err)
  901. }
  902. defer f.Close()
  903. var res struct {
  904. Log string `json:"log"`
  905. Stream string `json:"stream"`
  906. Time time.Time `json:"time"`
  907. }
  908. if err := json.NewDecoder(f).Decode(&res); err != nil {
  909. c.Fatal(err)
  910. }
  911. if res.Log != "testline\n" {
  912. c.Fatalf("Unexpected log line: %q, expected: %q", res.Log, "testline\n")
  913. }
  914. if res.Stream != "stdout" {
  915. c.Fatalf("Unexpected stream: %q, expected: %q", res.Stream, "stdout")
  916. }
  917. if !time.Now().After(res.Time) {
  918. c.Fatalf("Log time %v in future", res.Time)
  919. }
  920. }
  921. func (s *DockerDaemonSuite) TestDaemonLoggingDriverNoneLogsError(c *testing.T) {
  922. s.d.StartWithBusybox(c, "--log-driver=none")
  923. out, err := s.d.Cmd("run", "--name=test", "busybox", "echo", "testline")
  924. assert.NilError(c, err, out)
  925. out, err = s.d.Cmd("logs", "test")
  926. assert.Assert(c, err != nil, "Logs should fail with 'none' driver")
  927. expected := `configured logging driver does not support reading`
  928. assert.Assert(c, strings.Contains(out, expected))
  929. }
  930. func (s *DockerDaemonSuite) TestDaemonLoggingDriverShouldBeIgnoredForBuild(c *testing.T) {
  931. s.d.StartWithBusybox(c, "--log-driver=splunk")
  932. result := cli.BuildCmd(c, "busyboxs", cli.Daemon(s.d),
  933. build.WithDockerfile(`
  934. FROM busybox
  935. RUN echo foo`),
  936. build.WithoutCache,
  937. )
  938. comment := fmt.Sprintf("Failed to build image. output %s, exitCode %d, err %v", result.Combined(), result.ExitCode, result.Error)
  939. assert.Assert(c, result.Error == nil, comment)
  940. assert.Equal(c, result.ExitCode, 0, comment)
  941. assert.Assert(c, strings.Contains(result.Combined(), "foo"), comment)
  942. }
  943. func (s *DockerDaemonSuite) TestDaemonUnixSockCleanedUp(c *testing.T) {
  944. dir, err := os.MkdirTemp("", "socket-cleanup-test")
  945. if err != nil {
  946. c.Fatal(err)
  947. }
  948. defer os.RemoveAll(dir)
  949. sockPath := filepath.Join(dir, "docker.sock")
  950. s.d.Start(c, "--host", "unix://"+sockPath)
  951. if _, err := os.Stat(sockPath); err != nil {
  952. c.Fatal("socket does not exist")
  953. }
  954. s.d.Stop(c)
  955. if _, err := os.Stat(sockPath); err == nil || !os.IsNotExist(err) {
  956. c.Fatal("unix socket is not cleaned up")
  957. }
  958. }
  959. func (s *DockerDaemonSuite) TestDaemonWithWrongkey(c *testing.T) {
  960. type Config struct {
  961. Crv string `json:"crv"`
  962. D string `json:"d"`
  963. Kid string `json:"kid"`
  964. Kty string `json:"kty"`
  965. X string `json:"x"`
  966. Y string `json:"y"`
  967. }
  968. os.Remove("/etc/docker/key.json")
  969. c.Setenv("DOCKER_ALLOW_SCHEMA1_PUSH_DONOTUSE", "1")
  970. s.d.Start(c)
  971. s.d.Stop(c)
  972. config := &Config{}
  973. bytes, err := os.ReadFile("/etc/docker/key.json")
  974. if err != nil {
  975. c.Fatalf("Error reading key.json file: %s", err)
  976. }
  977. // byte[] to Data-Struct
  978. if err := json.Unmarshal(bytes, &config); err != nil {
  979. c.Fatalf("Error Unmarshal: %s", err)
  980. }
  981. // replace config.Kid with the fake value
  982. config.Kid = "VSAJ:FUYR:X3H2:B2VZ:KZ6U:CJD5:K7BX:ZXHY:UZXT:P4FT:MJWG:HRJ4"
  983. // NEW Data-Struct to byte[]
  984. newBytes, err := json.Marshal(&config)
  985. if err != nil {
  986. c.Fatalf("Error Marshal: %s", err)
  987. }
  988. // write back
  989. if err := os.WriteFile("/etc/docker/key.json", newBytes, 0400); err != nil {
  990. c.Fatalf("Error os.WriteFile: %s", err)
  991. }
  992. defer os.Remove("/etc/docker/key.json")
  993. if err := s.d.StartWithError(); err == nil {
  994. c.Fatalf("It should not be successful to start daemon with wrong key: %v", err)
  995. }
  996. content, err := s.d.ReadLogFile()
  997. assert.Assert(c, err == nil)
  998. if !strings.Contains(string(content), "Public Key ID does not match") {
  999. c.Fatalf("Missing KeyID message from daemon logs: %s", string(content))
  1000. }
  1001. }
  1002. func (s *DockerDaemonSuite) TestDaemonRestartKillWait(c *testing.T) {
  1003. s.d.StartWithBusybox(c)
  1004. out, err := s.d.Cmd("run", "-id", "busybox", "/bin/cat")
  1005. if err != nil {
  1006. c.Fatalf("Could not run /bin/cat: err=%v\n%s", err, out)
  1007. }
  1008. containerID := strings.TrimSpace(out)
  1009. if out, err := s.d.Cmd("kill", containerID); err != nil {
  1010. c.Fatalf("Could not kill %s: err=%v\n%s", containerID, err, out)
  1011. }
  1012. s.d.Restart(c)
  1013. errchan := make(chan error, 1)
  1014. go func() {
  1015. if out, err := s.d.Cmd("wait", containerID); err != nil {
  1016. errchan <- fmt.Errorf("%v:\n%s", err, out)
  1017. }
  1018. close(errchan)
  1019. }()
  1020. select {
  1021. case <-time.After(5 * time.Second):
  1022. c.Fatal("Waiting on a stopped (killed) container timed out")
  1023. case err := <-errchan:
  1024. if err != nil {
  1025. c.Fatal(err)
  1026. }
  1027. }
  1028. }
  1029. // TestHTTPSInfo connects via two-way authenticated HTTPS to the info endpoint
  1030. func (s *DockerDaemonSuite) TestHTTPSInfo(c *testing.T) {
  1031. const (
  1032. testDaemonHTTPSAddr = "tcp://localhost:4271"
  1033. )
  1034. s.d.Start(c,
  1035. "--tlsverify",
  1036. "--tlscacert", "fixtures/https/ca.pem",
  1037. "--tlscert", "fixtures/https/server-cert.pem",
  1038. "--tlskey", "fixtures/https/server-key.pem",
  1039. "-H", testDaemonHTTPSAddr)
  1040. args := []string{
  1041. "--host", testDaemonHTTPSAddr,
  1042. "--tlsverify",
  1043. "--tlscacert", "fixtures/https/ca.pem",
  1044. "--tlscert", "fixtures/https/client-cert.pem",
  1045. "--tlskey", "fixtures/https/client-key.pem",
  1046. "info",
  1047. }
  1048. out, err := s.d.Cmd(args...)
  1049. if err != nil {
  1050. c.Fatalf("Error Occurred: %s and output: %s", err, out)
  1051. }
  1052. }
  1053. // TestHTTPSRun connects via two-way authenticated HTTPS to the create, attach, start, and wait endpoints.
  1054. // https://github.com/docker/docker/issues/19280
  1055. func (s *DockerDaemonSuite) TestHTTPSRun(c *testing.T) {
  1056. const (
  1057. testDaemonHTTPSAddr = "tcp://localhost:4271"
  1058. )
  1059. s.d.StartWithBusybox(c, "--tlsverify", "--tlscacert", "fixtures/https/ca.pem", "--tlscert", "fixtures/https/server-cert.pem",
  1060. "--tlskey", "fixtures/https/server-key.pem", "-H", testDaemonHTTPSAddr)
  1061. args := []string{
  1062. "--host", testDaemonHTTPSAddr,
  1063. "--tlsverify", "--tlscacert", "fixtures/https/ca.pem",
  1064. "--tlscert", "fixtures/https/client-cert.pem",
  1065. "--tlskey", "fixtures/https/client-key.pem",
  1066. "run", "busybox", "echo", "TLS response",
  1067. }
  1068. out, err := s.d.Cmd(args...)
  1069. if err != nil {
  1070. c.Fatalf("Error Occurred: %s and output: %s", err, out)
  1071. }
  1072. if !strings.Contains(out, "TLS response") {
  1073. c.Fatalf("expected output to include `TLS response`, got %v", out)
  1074. }
  1075. }
  1076. // TestTLSVerify verifies that --tlsverify=false turns on tls
  1077. func (s *DockerDaemonSuite) TestTLSVerify(c *testing.T) {
  1078. out, err := exec.Command(dockerdBinary, "--tlsverify=false").CombinedOutput()
  1079. if err == nil || !strings.Contains(string(out), "Could not load X509 key pair") {
  1080. c.Fatalf("Daemon should not have started due to missing certs: %v\n%s", err, string(out))
  1081. }
  1082. }
  1083. // TestHTTPSInfoRogueCert connects via two-way authenticated HTTPS to the info endpoint
  1084. // by using a rogue client certificate and checks that it fails with the expected error.
  1085. func (s *DockerDaemonSuite) TestHTTPSInfoRogueCert(c *testing.T) {
  1086. const (
  1087. errBadCertificate = "bad certificate"
  1088. testDaemonHTTPSAddr = "tcp://localhost:4271"
  1089. )
  1090. s.d.Start(c,
  1091. "--tlsverify",
  1092. "--tlscacert", "fixtures/https/ca.pem",
  1093. "--tlscert", "fixtures/https/server-cert.pem",
  1094. "--tlskey", "fixtures/https/server-key.pem",
  1095. "-H", testDaemonHTTPSAddr)
  1096. args := []string{
  1097. "--host", testDaemonHTTPSAddr,
  1098. "--tlsverify",
  1099. "--tlscacert", "fixtures/https/ca.pem",
  1100. "--tlscert", "fixtures/https/client-rogue-cert.pem",
  1101. "--tlskey", "fixtures/https/client-rogue-key.pem",
  1102. "info",
  1103. }
  1104. out, err := s.d.Cmd(args...)
  1105. if err == nil || !strings.Contains(out, errBadCertificate) {
  1106. c.Fatalf("Expected err: %s, got instead: %s and output: %s", errBadCertificate, err, out)
  1107. }
  1108. }
  1109. // TestHTTPSInfoRogueServerCert connects via two-way authenticated HTTPS to the info endpoint
  1110. // which provides a rogue server certificate and checks that it fails with the expected error
  1111. func (s *DockerDaemonSuite) TestHTTPSInfoRogueServerCert(c *testing.T) {
  1112. const (
  1113. errCaUnknown = "x509: certificate signed by unknown authority"
  1114. testDaemonRogueHTTPSAddr = "tcp://localhost:4272"
  1115. )
  1116. s.d.Start(c,
  1117. "--tlsverify",
  1118. "--tlscacert", "fixtures/https/ca.pem",
  1119. "--tlscert", "fixtures/https/server-rogue-cert.pem",
  1120. "--tlskey", "fixtures/https/server-rogue-key.pem",
  1121. "-H", testDaemonRogueHTTPSAddr)
  1122. args := []string{
  1123. "--host", testDaemonRogueHTTPSAddr,
  1124. "--tlsverify",
  1125. "--tlscacert", "fixtures/https/ca.pem",
  1126. "--tlscert", "fixtures/https/client-rogue-cert.pem",
  1127. "--tlskey", "fixtures/https/client-rogue-key.pem",
  1128. "info",
  1129. }
  1130. out, err := s.d.Cmd(args...)
  1131. if err == nil || !strings.Contains(out, errCaUnknown) {
  1132. c.Fatalf("Expected err: %s, got instead: %s and output: %s", errCaUnknown, err, out)
  1133. }
  1134. }
  1135. func pingContainers(c *testing.T, d *daemon.Daemon, expectFailure bool) {
  1136. var dargs []string
  1137. if d != nil {
  1138. dargs = []string{"--host", d.Sock()}
  1139. }
  1140. args := append(dargs, "run", "-d", "--name", "container1", "busybox", "top")
  1141. dockerCmd(c, args...)
  1142. args = append(dargs, "run", "--rm", "--link", "container1:alias1", "busybox", "sh", "-c")
  1143. pingCmd := "ping -c 1 %s -W 1"
  1144. args = append(args, fmt.Sprintf(pingCmd, "alias1"))
  1145. _, _, err := dockerCmdWithError(args...)
  1146. if expectFailure {
  1147. assert.ErrorContains(c, err, "")
  1148. } else {
  1149. assert.NilError(c, err)
  1150. }
  1151. args = append(dargs, "rm", "-f", "container1")
  1152. dockerCmd(c, args...)
  1153. }
  1154. func (s *DockerDaemonSuite) TestDaemonRestartWithSocketAsVolume(c *testing.T) {
  1155. s.d.StartWithBusybox(c)
  1156. socket := filepath.Join(s.d.Folder, "docker.sock")
  1157. out, err := s.d.Cmd("run", "--restart=always", "-v", socket+":/sock", "busybox")
  1158. assert.NilError(c, err, "Output: %s", out)
  1159. s.d.Restart(c)
  1160. }
  1161. // os.Kill should kill daemon ungracefully, leaving behind container mounts.
  1162. // A subsequent daemon restart should clean up said mounts.
  1163. func (s *DockerDaemonSuite) TestCleanupMountsAfterDaemonAndContainerKill(c *testing.T) {
  1164. d := daemon.New(c, dockerBinary, dockerdBinary, testdaemon.WithEnvironment(testEnv.Execution))
  1165. d.StartWithBusybox(c)
  1166. out, err := d.Cmd("run", "-d", "busybox", "top")
  1167. assert.NilError(c, err, "Output: %s", out)
  1168. id := strings.TrimSpace(out)
  1169. // If there are no mounts with container id visible from the host
  1170. // (as those are in container's own mount ns), there is nothing
  1171. // to check here and the test should be skipped.
  1172. mountOut, err := os.ReadFile("/proc/self/mountinfo")
  1173. assert.NilError(c, err, "Output: %s", mountOut)
  1174. if !strings.Contains(string(mountOut), id) {
  1175. d.Stop(c)
  1176. c.Skip("no container mounts visible in host ns")
  1177. }
  1178. // kill the daemon
  1179. assert.NilError(c, d.Kill())
  1180. // kill the container
  1181. icmd.RunCommand(ctrBinary, "--address", containerdSocket,
  1182. "--namespace", d.ContainersNamespace(), "tasks", "kill", id).Assert(c, icmd.Success)
  1183. // restart daemon.
  1184. d.Restart(c)
  1185. // Now, container mounts should be gone.
  1186. mountOut, err = os.ReadFile("/proc/self/mountinfo")
  1187. assert.NilError(c, err, "Output: %s", mountOut)
  1188. assert.Assert(c, !strings.Contains(string(mountOut), id), "%s is still mounted from older daemon start:\nDaemon root repository %s\n%s", id, d.Root, mountOut)
  1189. d.Stop(c)
  1190. }
  1191. // os.Interrupt should perform a graceful daemon shutdown and hence cleanup mounts.
  1192. func (s *DockerDaemonSuite) TestCleanupMountsAfterGracefulShutdown(c *testing.T) {
  1193. d := daemon.New(c, dockerBinary, dockerdBinary, testdaemon.WithEnvironment(testEnv.Execution))
  1194. d.StartWithBusybox(c)
  1195. out, err := d.Cmd("run", "-d", "busybox", "top")
  1196. assert.NilError(c, err, "Output: %s", out)
  1197. id := strings.TrimSpace(out)
  1198. // Send SIGINT and daemon should clean up
  1199. assert.NilError(c, d.Signal(os.Interrupt))
  1200. // Wait for the daemon to stop.
  1201. assert.NilError(c, <-d.Wait)
  1202. mountOut, err := os.ReadFile("/proc/self/mountinfo")
  1203. assert.NilError(c, err, "Output: %s", mountOut)
  1204. assert.Assert(c, !strings.Contains(string(mountOut), id), "%s is still mounted from older daemon start:\nDaemon root repository %s\n%s", id, d.Root, mountOut)
  1205. }
  1206. func (s *DockerDaemonSuite) TestDaemonRestartWithContainerRunning(t *testing.T) {
  1207. s.d.StartWithBusybox(t)
  1208. if out, err := s.d.Cmd("run", "-d", "--name", "test", "busybox", "top"); err != nil {
  1209. t.Fatal(out, err)
  1210. }
  1211. s.d.Restart(t)
  1212. // Container 'test' should be removed without error
  1213. if out, err := s.d.Cmd("rm", "test"); err != nil {
  1214. t.Fatal(out, err)
  1215. }
  1216. }
  1217. func (s *DockerDaemonSuite) TestDaemonRestartCleanupNetns(c *testing.T) {
  1218. s.d.StartWithBusybox(c)
  1219. out, err := s.d.Cmd("run", "--name", "netns", "-d", "busybox", "top")
  1220. if err != nil {
  1221. c.Fatal(out, err)
  1222. }
  1223. // Get sandbox key via inspect
  1224. out, err = s.d.Cmd("inspect", "--format", "'{{.NetworkSettings.SandboxKey}}'", "netns")
  1225. if err != nil {
  1226. c.Fatalf("Error inspecting container: %s, %v", out, err)
  1227. }
  1228. fileName := strings.Trim(out, " \r\n'")
  1229. if out, err := s.d.Cmd("stop", "netns"); err != nil {
  1230. c.Fatal(out, err)
  1231. }
  1232. // Test if the file still exists
  1233. icmd.RunCommand("stat", "-c", "%n", fileName).Assert(c, icmd.Expected{
  1234. Out: fileName,
  1235. })
  1236. // Remove the container and restart the daemon
  1237. if out, err := s.d.Cmd("rm", "netns"); err != nil {
  1238. c.Fatal(out, err)
  1239. }
  1240. s.d.Restart(c)
  1241. // Test again and see now the netns file does not exist
  1242. icmd.RunCommand("stat", "-c", "%n", fileName).Assert(c, icmd.Expected{
  1243. Err: "No such file or directory",
  1244. ExitCode: 1,
  1245. })
  1246. }
  1247. // tests regression detailed in #13964 where DOCKER_TLS_VERIFY env is ignored
  1248. func (s *DockerDaemonSuite) TestDaemonTLSVerifyIssue13964(c *testing.T) {
  1249. host := "tcp://localhost:4271"
  1250. s.d.Start(c, "-H", host)
  1251. icmd.RunCmd(icmd.Cmd{
  1252. Command: []string{dockerBinary, "-H", host, "info"},
  1253. Env: []string{"DOCKER_TLS_VERIFY=1", "DOCKER_CERT_PATH=fixtures/https"},
  1254. }).Assert(c, icmd.Expected{
  1255. ExitCode: 1,
  1256. Err: "error during connect",
  1257. })
  1258. }
  1259. func setupV6(c *testing.T) {
  1260. // Hack to get the right IPv6 address on docker0, which has already been created
  1261. result := icmd.RunCommand("ip", "addr", "add", "fe80::1/64", "dev", "docker0")
  1262. result.Assert(c, icmd.Success)
  1263. }
  1264. func teardownV6(c *testing.T) {
  1265. result := icmd.RunCommand("ip", "addr", "del", "fe80::1/64", "dev", "docker0")
  1266. result.Assert(c, icmd.Success)
  1267. }
  1268. func (s *DockerDaemonSuite) TestDaemonRestartWithContainerWithRestartPolicyAlways(c *testing.T) {
  1269. s.d.StartWithBusybox(c)
  1270. out, err := s.d.Cmd("run", "-d", "--restart", "always", "busybox", "top")
  1271. assert.NilError(c, err, out)
  1272. id := strings.TrimSpace(out)
  1273. out, err = s.d.Cmd("stop", id)
  1274. assert.NilError(c, err, out)
  1275. out, err = s.d.Cmd("wait", id)
  1276. assert.NilError(c, err, out)
  1277. out, err = s.d.Cmd("ps", "-q")
  1278. assert.NilError(c, err, out)
  1279. assert.Equal(c, out, "")
  1280. s.d.Restart(c)
  1281. out, err = s.d.Cmd("ps", "-q")
  1282. assert.NilError(c, err, out)
  1283. assert.Equal(c, strings.TrimSpace(out), id[:12])
  1284. }
  1285. func (s *DockerDaemonSuite) TestDaemonWideLogConfig(c *testing.T) {
  1286. s.d.StartWithBusybox(c, "--log-opt=max-size=1k")
  1287. name := "logtest"
  1288. out, err := s.d.Cmd("run", "-d", "--log-opt=max-file=5", "--name", name, "busybox", "top")
  1289. assert.NilError(c, err, "Output: %s, err: %v", out, err)
  1290. out, err = s.d.Cmd("inspect", "-f", "{{ .HostConfig.LogConfig.Config }}", name)
  1291. assert.NilError(c, err, "Output: %s", out)
  1292. assert.Assert(c, strings.Contains(out, "max-size:1k"))
  1293. assert.Assert(c, strings.Contains(out, "max-file:5"))
  1294. out, err = s.d.Cmd("inspect", "-f", "{{ .HostConfig.LogConfig.Type }}", name)
  1295. assert.NilError(c, err, "Output: %s", out)
  1296. assert.Equal(c, strings.TrimSpace(out), "json-file")
  1297. }
  1298. func (s *DockerDaemonSuite) TestDaemonRestartWithPausedContainer(c *testing.T) {
  1299. s.d.StartWithBusybox(c)
  1300. if out, err := s.d.Cmd("run", "-i", "-d", "--name", "test", "busybox", "top"); err != nil {
  1301. c.Fatal(err, out)
  1302. }
  1303. if out, err := s.d.Cmd("pause", "test"); err != nil {
  1304. c.Fatal(err, out)
  1305. }
  1306. s.d.Restart(c)
  1307. errchan := make(chan error, 1)
  1308. go func() {
  1309. out, err := s.d.Cmd("start", "test")
  1310. if err != nil {
  1311. errchan <- fmt.Errorf("%v:\n%s", err, out)
  1312. return
  1313. }
  1314. name := strings.TrimSpace(out)
  1315. if name != "test" {
  1316. errchan <- fmt.Errorf("Paused container start error on docker daemon restart, expected 'test' but got '%s'", name)
  1317. return
  1318. }
  1319. close(errchan)
  1320. }()
  1321. select {
  1322. case <-time.After(5 * time.Second):
  1323. c.Fatal("Waiting on start a container timed out")
  1324. case err := <-errchan:
  1325. if err != nil {
  1326. c.Fatal(err)
  1327. }
  1328. }
  1329. }
  1330. func (s *DockerDaemonSuite) TestDaemonRestartRmVolumeInUse(c *testing.T) {
  1331. s.d.StartWithBusybox(c)
  1332. out, err := s.d.Cmd("create", "-v", "test:/foo", "busybox")
  1333. assert.NilError(c, err, out)
  1334. s.d.Restart(c)
  1335. out, err = s.d.Cmd("volume", "rm", "test")
  1336. assert.Assert(c, err != nil, "should not be able to remove in use volume after daemon restart")
  1337. assert.Assert(c, strings.Contains(out, "in use"))
  1338. }
  1339. func (s *DockerDaemonSuite) TestDaemonRestartLocalVolumes(c *testing.T) {
  1340. s.d.Start(c)
  1341. out, err := s.d.Cmd("volume", "create", "test")
  1342. assert.NilError(c, err, out)
  1343. s.d.Restart(c)
  1344. out, err = s.d.Cmd("volume", "inspect", "test")
  1345. assert.NilError(c, err, out)
  1346. }
  1347. // FIXME(vdemeester) Use a new daemon instance instead of the Suite one
  1348. func (s *DockerDaemonSuite) TestDaemonStartWithoutHost(c *testing.T) {
  1349. s.d.UseDefaultHost = true
  1350. defer func() {
  1351. s.d.UseDefaultHost = false
  1352. }()
  1353. s.d.Start(c)
  1354. }
  1355. // FIXME(vdemeester) Use a new daemon instance instead of the Suite one
  1356. func (s *DockerDaemonSuite) TestDaemonStartWithDefaultTLSHost(c *testing.T) {
  1357. s.d.UseDefaultTLSHost = true
  1358. defer func() {
  1359. s.d.UseDefaultTLSHost = false
  1360. }()
  1361. s.d.Start(c,
  1362. "--tlsverify",
  1363. "--tlscacert", "fixtures/https/ca.pem",
  1364. "--tlscert", "fixtures/https/server-cert.pem",
  1365. "--tlskey", "fixtures/https/server-key.pem")
  1366. // The client with --tlsverify should also use default host localhost:2376
  1367. c.Setenv("DOCKER_HOST", "")
  1368. out, _ := dockerCmd(
  1369. c,
  1370. "--tlsverify",
  1371. "--tlscacert", "fixtures/https/ca.pem",
  1372. "--tlscert", "fixtures/https/client-cert.pem",
  1373. "--tlskey", "fixtures/https/client-key.pem",
  1374. "version",
  1375. )
  1376. if !strings.Contains(out, "Server") {
  1377. c.Fatalf("docker version should return information of server side")
  1378. }
  1379. // ensure when connecting to the server that only a single acceptable CA is requested
  1380. contents, err := os.ReadFile("fixtures/https/ca.pem")
  1381. assert.NilError(c, err)
  1382. rootCert, err := helpers.ParseCertificatePEM(contents)
  1383. assert.NilError(c, err)
  1384. rootPool := x509.NewCertPool()
  1385. rootPool.AddCert(rootCert)
  1386. var certRequestInfo *tls.CertificateRequestInfo
  1387. conn, err := tls.Dial("tcp", fmt.Sprintf("%s:%d", opts.DefaultHTTPHost, opts.DefaultTLSHTTPPort), &tls.Config{
  1388. RootCAs: rootPool,
  1389. GetClientCertificate: func(cri *tls.CertificateRequestInfo) (*tls.Certificate, error) {
  1390. certRequestInfo = cri
  1391. cert, err := tls.LoadX509KeyPair("fixtures/https/client-cert.pem", "fixtures/https/client-key.pem")
  1392. if err != nil {
  1393. return nil, err
  1394. }
  1395. return &cert, nil
  1396. },
  1397. })
  1398. assert.NilError(c, err)
  1399. conn.Close()
  1400. assert.Assert(c, certRequestInfo != nil)
  1401. assert.Equal(c, len(certRequestInfo.AcceptableCAs), 1)
  1402. assert.DeepEqual(c, certRequestInfo.AcceptableCAs[0], rootCert.RawSubject)
  1403. }
  1404. func (s *DockerDaemonSuite) TestBridgeIPIsExcludedFromAllocatorPool(c *testing.T) {
  1405. defaultNetworkBridge := "docker0"
  1406. deleteInterface(c, defaultNetworkBridge)
  1407. bridgeIP := "192.169.1.1"
  1408. bridgeRange := bridgeIP + "/30"
  1409. s.d.StartWithBusybox(c, "--bip", bridgeRange)
  1410. defer s.d.Restart(c)
  1411. var cont int
  1412. for {
  1413. contName := fmt.Sprintf("container%d", cont)
  1414. _, err := s.d.Cmd("run", "--name", contName, "-d", "busybox", "/bin/sleep", "2")
  1415. if err != nil {
  1416. // pool exhausted
  1417. break
  1418. }
  1419. ip, err := s.d.Cmd("inspect", "--format", "'{{.NetworkSettings.IPAddress}}'", contName)
  1420. assert.Assert(c, err == nil, ip)
  1421. assert.Assert(c, ip != bridgeIP)
  1422. cont++
  1423. }
  1424. }
  1425. // Test daemon for no space left on device error
  1426. func (s *DockerDaemonSuite) TestDaemonNoSpaceLeftOnDeviceError(c *testing.T) {
  1427. testRequires(c, testEnv.IsLocalDaemon, DaemonIsLinux, Network)
  1428. testDir, err := os.MkdirTemp("", "no-space-left-on-device-test")
  1429. assert.NilError(c, err)
  1430. defer os.RemoveAll(testDir)
  1431. assert.Assert(c, mount.MakeRShared(testDir) == nil)
  1432. defer mount.Unmount(testDir)
  1433. // create a 3MiB image (with a 2MiB ext4 fs) and mount it as graph root
  1434. // Why in a container? Because `mount` sometimes behaves weirdly and often fails outright on this test in debian:bullseye (which is what the test suite runs under if run from the Makefile)
  1435. dockerCmd(c, "run", "--rm", "-v", testDir+":/test", "busybox", "sh", "-c", "dd of=/test/testfs.img bs=1M seek=3 count=0")
  1436. icmd.RunCommand("mkfs.ext4", "-F", filepath.Join(testDir, "testfs.img")).Assert(c, icmd.Success)
  1437. dockerCmd(c, "run", "--privileged", "--rm", "-v", testDir+":/test:shared", "busybox", "sh", "-c", "mkdir -p /test/test-mount/vfs && mount -n -t ext4 /test/testfs.img /test/test-mount/vfs")
  1438. defer mount.Unmount(filepath.Join(testDir, "test-mount"))
  1439. s.d.Start(c, "--storage-driver", "vfs", "--data-root", filepath.Join(testDir, "test-mount"))
  1440. defer s.d.Stop(c)
  1441. // pull a repository large enough to overfill the mounted filesystem
  1442. pullOut, err := s.d.Cmd("pull", "debian:bullseye-slim")
  1443. assert.Assert(c, err != nil, pullOut)
  1444. assert.Assert(c, strings.Contains(pullOut, "no space left on device"))
  1445. }
  1446. // Test daemon restart with container links + auto restart
  1447. func (s *DockerDaemonSuite) TestDaemonRestartContainerLinksRestart(c *testing.T) {
  1448. s.d.StartWithBusybox(c)
  1449. var parent1Args []string
  1450. var parent2Args []string
  1451. wg := sync.WaitGroup{}
  1452. maxChildren := 10
  1453. chErr := make(chan error, maxChildren)
  1454. for i := 0; i < maxChildren; i++ {
  1455. wg.Add(1)
  1456. name := fmt.Sprintf("test%d", i)
  1457. if i < maxChildren/2 {
  1458. parent1Args = append(parent1Args, []string{"--link", name}...)
  1459. } else {
  1460. parent2Args = append(parent2Args, []string{"--link", name}...)
  1461. }
  1462. go func() {
  1463. _, err := s.d.Cmd("run", "-d", "--name", name, "--restart=always", "busybox", "top")
  1464. chErr <- err
  1465. wg.Done()
  1466. }()
  1467. }
  1468. wg.Wait()
  1469. close(chErr)
  1470. for err := range chErr {
  1471. assert.NilError(c, err)
  1472. }
  1473. parent1Args = append([]string{"run", "-d"}, parent1Args...)
  1474. parent1Args = append(parent1Args, []string{"--name=parent1", "--restart=always", "busybox", "top"}...)
  1475. parent2Args = append([]string{"run", "-d"}, parent2Args...)
  1476. parent2Args = append(parent2Args, []string{"--name=parent2", "--restart=always", "busybox", "top"}...)
  1477. _, err := s.d.Cmd(parent1Args...)
  1478. assert.NilError(c, err)
  1479. _, err = s.d.Cmd(parent2Args...)
  1480. assert.NilError(c, err)
  1481. s.d.Stop(c)
  1482. // clear the log file -- we don't need any of it but may for the next part
  1483. // can ignore the error here, this is just a cleanup
  1484. os.Truncate(s.d.LogFileName(), 0)
  1485. s.d.Start(c)
  1486. for _, num := range []string{"1", "2"} {
  1487. out, err := s.d.Cmd("inspect", "-f", "{{ .State.Running }}", "parent"+num)
  1488. assert.NilError(c, err)
  1489. if strings.TrimSpace(out) != "true" {
  1490. log, _ := os.ReadFile(s.d.LogFileName())
  1491. c.Fatalf("parent container is not running\n%s", string(log))
  1492. }
  1493. }
  1494. }
  1495. func (s *DockerDaemonSuite) TestDaemonCgroupParent(c *testing.T) {
  1496. testRequires(c, DaemonIsLinux)
  1497. cgroupParent := "test"
  1498. name := "cgroup-test"
  1499. s.d.StartWithBusybox(c, "--cgroup-parent", cgroupParent)
  1500. defer s.d.Restart(c)
  1501. out, err := s.d.Cmd("run", "--name", name, "busybox", "cat", "/proc/self/cgroup")
  1502. assert.NilError(c, err)
  1503. cgroupPaths := ParseCgroupPaths(out)
  1504. assert.Assert(c, len(cgroupPaths) != 0, "unexpected output - %q", out)
  1505. out, err = s.d.Cmd("inspect", "-f", "{{.Id}}", name)
  1506. assert.NilError(c, err)
  1507. id := strings.TrimSpace(out)
  1508. expectedCgroup := path.Join(cgroupParent, id)
  1509. found := false
  1510. for _, path := range cgroupPaths {
  1511. if strings.HasSuffix(path, expectedCgroup) {
  1512. found = true
  1513. break
  1514. }
  1515. }
  1516. assert.Assert(c, found, "Cgroup path for container (%s) doesn't found in cgroups file: %s", expectedCgroup, cgroupPaths)
  1517. }
  1518. func (s *DockerDaemonSuite) TestDaemonRestartWithLinks(c *testing.T) {
  1519. testRequires(c, DaemonIsLinux) // Windows does not support links
  1520. s.d.StartWithBusybox(c)
  1521. out, err := s.d.Cmd("run", "-d", "--name=test", "busybox", "top")
  1522. assert.NilError(c, err, out)
  1523. out, err = s.d.Cmd("run", "--name=test2", "--link", "test:abc", "busybox", "sh", "-c", "ping -c 1 -w 1 abc")
  1524. assert.NilError(c, err, out)
  1525. s.d.Restart(c)
  1526. // should fail since test is not running yet
  1527. out, err = s.d.Cmd("start", "test2")
  1528. assert.ErrorContains(c, err, "", out)
  1529. out, err = s.d.Cmd("start", "test")
  1530. assert.NilError(c, err, out)
  1531. out, err = s.d.Cmd("start", "-a", "test2")
  1532. assert.NilError(c, err, out)
  1533. assert.Equal(c, strings.Contains(out, "1 packets transmitted, 1 packets received"), true, out)
  1534. }
  1535. func (s *DockerDaemonSuite) TestDaemonRestartWithNames(c *testing.T) {
  1536. testRequires(c, DaemonIsLinux) // Windows does not support links
  1537. s.d.StartWithBusybox(c)
  1538. out, err := s.d.Cmd("create", "--name=test", "busybox")
  1539. assert.NilError(c, err, out)
  1540. out, err = s.d.Cmd("run", "-d", "--name=test2", "busybox", "top")
  1541. assert.NilError(c, err, out)
  1542. test2ID := strings.TrimSpace(out)
  1543. out, err = s.d.Cmd("run", "-d", "--name=test3", "--link", "test2:abc", "busybox", "top")
  1544. assert.NilError(c, err)
  1545. test3ID := strings.TrimSpace(out)
  1546. s.d.Restart(c)
  1547. _, err = s.d.Cmd("create", "--name=test", "busybox")
  1548. assert.ErrorContains(c, err, "", "expected error trying to create container with duplicate name")
  1549. // this one is no longer needed, removing simplifies the remainder of the test
  1550. out, err = s.d.Cmd("rm", "-f", "test")
  1551. assert.NilError(c, err, out)
  1552. out, err = s.d.Cmd("ps", "-a", "--no-trunc")
  1553. assert.NilError(c, err, out)
  1554. lines := strings.Split(strings.TrimSpace(out), "\n")[1:]
  1555. test2validated := false
  1556. test3validated := false
  1557. for _, line := range lines {
  1558. fields := strings.Fields(line)
  1559. names := fields[len(fields)-1]
  1560. switch fields[0] {
  1561. case test2ID:
  1562. assert.Equal(c, names, "test2,test3/abc")
  1563. test2validated = true
  1564. case test3ID:
  1565. assert.Equal(c, names, "test3")
  1566. test3validated = true
  1567. }
  1568. }
  1569. assert.Assert(c, test2validated)
  1570. assert.Assert(c, test3validated)
  1571. }
  1572. // TestDaemonRestartWithKilledRunningContainer requires live restore of running containers
  1573. func (s *DockerDaemonSuite) TestDaemonRestartWithKilledRunningContainer(t *testing.T) {
  1574. testRequires(t, DaemonIsLinux)
  1575. s.d.StartWithBusybox(t)
  1576. cid, err := s.d.Cmd("run", "-d", "--name", "test", "busybox", "top")
  1577. defer s.d.Stop(t)
  1578. if err != nil {
  1579. t.Fatal(cid, err)
  1580. }
  1581. cid = strings.TrimSpace(cid)
  1582. pid, err := s.d.Cmd("inspect", "-f", "{{.State.Pid}}", cid)
  1583. assert.NilError(t, err)
  1584. pid = strings.TrimSpace(pid)
  1585. // Kill the daemon
  1586. if err := s.d.Kill(); err != nil {
  1587. t.Fatal(err)
  1588. }
  1589. // kill the container
  1590. icmd.RunCommand(ctrBinary, "--address", containerdSocket,
  1591. "--namespace", s.d.ContainersNamespace(), "tasks", "kill", cid).Assert(t, icmd.Success)
  1592. // Give time to containerd to process the command if we don't
  1593. // the exit event might be received after we do the inspect
  1594. result := icmd.RunCommand("kill", "-0", pid)
  1595. for result.ExitCode == 0 {
  1596. time.Sleep(1 * time.Second)
  1597. // FIXME(vdemeester) should we check it doesn't error out ?
  1598. result = icmd.RunCommand("kill", "-0", pid)
  1599. }
  1600. // restart the daemon
  1601. s.d.Start(t)
  1602. // Check that we've got the correct exit code
  1603. out, err := s.d.Cmd("inspect", "-f", "{{.State.ExitCode}}", cid)
  1604. assert.NilError(t, err)
  1605. out = strings.TrimSpace(out)
  1606. if out != "143" {
  1607. t.Fatalf("Expected exit code '%s' got '%s' for container '%s'\n", "143", out, cid)
  1608. }
  1609. }
  1610. // os.Kill should kill daemon ungracefully, leaving behind live containers.
  1611. // The live containers should be known to the restarted daemon. Stopping
  1612. // them now, should remove the mounts.
  1613. func (s *DockerDaemonSuite) TestCleanupMountsAfterDaemonCrash(c *testing.T) {
  1614. testRequires(c, DaemonIsLinux)
  1615. s.d.StartWithBusybox(c, "--live-restore")
  1616. out, err := s.d.Cmd("run", "-d", "busybox", "top")
  1617. assert.NilError(c, err, "Output: %s", out)
  1618. id := strings.TrimSpace(out)
  1619. // kill the daemon
  1620. assert.Assert(c, s.d.Kill() == nil)
  1621. // Check if there are mounts with container id visible from the host.
  1622. // If not, those mounts exist in container's own mount ns, and so
  1623. // the following check for mounts being cleared is pointless.
  1624. skipMountCheck := false
  1625. mountOut, err := os.ReadFile("/proc/self/mountinfo")
  1626. assert.Assert(c, err == nil, "Output: %s", mountOut)
  1627. if !strings.Contains(string(mountOut), id) {
  1628. skipMountCheck = true
  1629. }
  1630. // restart daemon.
  1631. s.d.Start(c, "--live-restore")
  1632. // container should be running.
  1633. out, err = s.d.Cmd("inspect", "--format={{.State.Running}}", id)
  1634. assert.NilError(c, err, "Output: %s", out)
  1635. out = strings.TrimSpace(out)
  1636. if out != "true" {
  1637. c.Fatalf("Container %s expected to stay alive after daemon restart", id)
  1638. }
  1639. // 'docker stop' should work.
  1640. out, err = s.d.Cmd("stop", id)
  1641. assert.NilError(c, err, "Output: %s", out)
  1642. if skipMountCheck {
  1643. return
  1644. }
  1645. // Now, container mounts should be gone.
  1646. mountOut, err = os.ReadFile("/proc/self/mountinfo")
  1647. assert.Assert(c, err == nil, "Output: %s", mountOut)
  1648. comment := fmt.Sprintf("%s is still mounted from older daemon start:\nDaemon root repository %s\n%s", id, s.d.Root, mountOut)
  1649. assert.Equal(c, strings.Contains(string(mountOut), id), false, comment)
  1650. }
  1651. // TestDaemonRestartWithUnpausedRunningContainer requires live restore of running containers.
  1652. func (s *DockerDaemonSuite) TestDaemonRestartWithUnpausedRunningContainer(t *testing.T) {
  1653. testRequires(t, DaemonIsLinux)
  1654. s.d.StartWithBusybox(t, "--live-restore")
  1655. cid, err := s.d.Cmd("run", "-d", "--name", "test", "busybox", "top")
  1656. defer s.d.Stop(t)
  1657. if err != nil {
  1658. t.Fatal(cid, err)
  1659. }
  1660. cid = strings.TrimSpace(cid)
  1661. pid, err := s.d.Cmd("inspect", "-f", "{{.State.Pid}}", cid)
  1662. assert.NilError(t, err)
  1663. // pause the container
  1664. if _, err := s.d.Cmd("pause", cid); err != nil {
  1665. t.Fatal(cid, err)
  1666. }
  1667. // Kill the daemon
  1668. if err := s.d.Kill(); err != nil {
  1669. t.Fatal(err)
  1670. }
  1671. // resume the container
  1672. result := icmd.RunCommand(
  1673. ctrBinary,
  1674. "--address", containerdSocket,
  1675. "--namespace", s.d.ContainersNamespace(),
  1676. "tasks", "resume", cid)
  1677. result.Assert(t, icmd.Success)
  1678. // Give time to containerd to process the command if we don't
  1679. // the resume event might be received after we do the inspect
  1680. poll.WaitOn(t, pollCheck(t, func(*testing.T) (interface{}, string) {
  1681. result := icmd.RunCommand("kill", "-0", strings.TrimSpace(pid))
  1682. return result.ExitCode, ""
  1683. }, checker.Equals(0)), poll.WithTimeout(defaultReconciliationTimeout))
  1684. // restart the daemon
  1685. s.d.Start(t, "--live-restore")
  1686. // Check that we've got the correct status
  1687. out, err := s.d.Cmd("inspect", "-f", "{{.State.Status}}", cid)
  1688. assert.NilError(t, err)
  1689. out = strings.TrimSpace(out)
  1690. if out != "running" {
  1691. t.Fatalf("Expected exit code '%s' got '%s' for container '%s'\n", "running", out, cid)
  1692. }
  1693. if _, err := s.d.Cmd("kill", cid); err != nil {
  1694. t.Fatal(err)
  1695. }
  1696. }
  1697. // TestRunLinksChanged checks that creating a new container with the same name does not update links
  1698. // this ensures that the old, pre gh#16032 functionality continues on
  1699. func (s *DockerDaemonSuite) TestRunLinksChanged(c *testing.T) {
  1700. testRequires(c, DaemonIsLinux) // Windows does not support links
  1701. s.d.StartWithBusybox(c)
  1702. out, err := s.d.Cmd("run", "-d", "--name=test", "busybox", "top")
  1703. assert.NilError(c, err, out)
  1704. out, err = s.d.Cmd("run", "--name=test2", "--link=test:abc", "busybox", "sh", "-c", "ping -c 1 abc")
  1705. assert.NilError(c, err, out)
  1706. assert.Assert(c, strings.Contains(out, "1 packets transmitted, 1 packets received"))
  1707. out, err = s.d.Cmd("rm", "-f", "test")
  1708. assert.NilError(c, err, out)
  1709. out, err = s.d.Cmd("run", "-d", "--name=test", "busybox", "top")
  1710. assert.NilError(c, err, out)
  1711. out, err = s.d.Cmd("start", "-a", "test2")
  1712. assert.ErrorContains(c, err, "", out)
  1713. assert.Assert(c, !strings.Contains(out, "1 packets transmitted, 1 packets received"))
  1714. s.d.Restart(c)
  1715. out, err = s.d.Cmd("start", "-a", "test2")
  1716. assert.ErrorContains(c, err, "", out)
  1717. assert.Assert(c, !strings.Contains(out, "1 packets transmitted, 1 packets received"))
  1718. }
  1719. func (s *DockerDaemonSuite) TestDaemonStartWithoutColors(c *testing.T) {
  1720. testRequires(c, DaemonIsLinux)
  1721. infoLog := "\x1b[36mINFO\x1b"
  1722. b := bytes.NewBuffer(nil)
  1723. done := make(chan bool)
  1724. p, tty, err := pty.Open()
  1725. assert.NilError(c, err)
  1726. defer func() {
  1727. tty.Close()
  1728. p.Close()
  1729. }()
  1730. go func() {
  1731. io.Copy(b, p)
  1732. done <- true
  1733. }()
  1734. // Enable coloring explicitly
  1735. s.d.StartWithLogFile(tty, "--raw-logs=false")
  1736. s.d.Stop(c)
  1737. // Wait for io.Copy() before checking output
  1738. <-done
  1739. assert.Assert(c, strings.Contains(b.String(), infoLog))
  1740. b.Reset()
  1741. // "tty" is already closed in prev s.d.Stop(),
  1742. // we have to close the other side "p" and open another pair of
  1743. // pty for the next test.
  1744. p.Close()
  1745. p, tty, err = pty.Open()
  1746. assert.NilError(c, err)
  1747. go func() {
  1748. io.Copy(b, p)
  1749. done <- true
  1750. }()
  1751. // Disable coloring explicitly
  1752. s.d.StartWithLogFile(tty, "--raw-logs=true")
  1753. s.d.Stop(c)
  1754. // Wait for io.Copy() before checking output
  1755. <-done
  1756. assert.Assert(c, b.String() != "")
  1757. assert.Assert(c, !strings.Contains(b.String(), infoLog))
  1758. }
  1759. func (s *DockerDaemonSuite) TestDaemonDebugLog(c *testing.T) {
  1760. testRequires(c, DaemonIsLinux)
  1761. debugLog := "\x1b[37mDEBU\x1b"
  1762. p, tty, err := pty.Open()
  1763. assert.NilError(c, err)
  1764. defer func() {
  1765. tty.Close()
  1766. p.Close()
  1767. }()
  1768. b := bytes.NewBuffer(nil)
  1769. go io.Copy(b, p)
  1770. s.d.StartWithLogFile(tty, "--debug")
  1771. s.d.Stop(c)
  1772. assert.Assert(c, strings.Contains(b.String(), debugLog))
  1773. }
  1774. // Test for #21956
  1775. func (s *DockerDaemonSuite) TestDaemonLogOptions(c *testing.T) {
  1776. s.d.StartWithBusybox(c, "--log-driver=syslog", "--log-opt=syslog-address=udp://127.0.0.1:514")
  1777. out, err := s.d.Cmd("run", "-d", "--log-driver=json-file", "busybox", "top")
  1778. assert.NilError(c, err, out)
  1779. id := strings.TrimSpace(out)
  1780. out, err = s.d.Cmd("inspect", "--format='{{.HostConfig.LogConfig}}'", id)
  1781. assert.NilError(c, err, out)
  1782. assert.Assert(c, strings.Contains(out, "{json-file map[]}"))
  1783. }
  1784. // Test case for #20936, #22443
  1785. func (s *DockerDaemonSuite) TestDaemonMaxConcurrency(c *testing.T) {
  1786. s.d.Start(c, "--max-concurrent-uploads=6", "--max-concurrent-downloads=8")
  1787. expectedMaxConcurrentUploads := `level=debug msg="Max Concurrent Uploads: 6"`
  1788. expectedMaxConcurrentDownloads := `level=debug msg="Max Concurrent Downloads: 8"`
  1789. content, err := s.d.ReadLogFile()
  1790. assert.NilError(c, err)
  1791. assert.Assert(c, strings.Contains(string(content), expectedMaxConcurrentUploads))
  1792. assert.Assert(c, strings.Contains(string(content), expectedMaxConcurrentDownloads))
  1793. }
  1794. // Test case for #20936, #22443
  1795. func (s *DockerDaemonSuite) TestDaemonMaxConcurrencyWithConfigFile(c *testing.T) {
  1796. testRequires(c, testEnv.IsLocalDaemon, DaemonIsLinux)
  1797. // daemon config file
  1798. configFilePath := "test.json"
  1799. configFile, err := os.Create(configFilePath)
  1800. assert.NilError(c, err)
  1801. defer os.Remove(configFilePath)
  1802. daemonConfig := `{ "max-concurrent-downloads" : 8 }`
  1803. fmt.Fprintf(configFile, "%s", daemonConfig)
  1804. configFile.Close()
  1805. s.d.Start(c, fmt.Sprintf("--config-file=%s", configFilePath))
  1806. expectedMaxConcurrentUploads := `level=debug msg="Max Concurrent Uploads: 5"`
  1807. expectedMaxConcurrentDownloads := `level=debug msg="Max Concurrent Downloads: 8"`
  1808. content, err := s.d.ReadLogFile()
  1809. assert.NilError(c, err)
  1810. assert.Assert(c, strings.Contains(string(content), expectedMaxConcurrentUploads))
  1811. assert.Assert(c, strings.Contains(string(content), expectedMaxConcurrentDownloads))
  1812. configFile, err = os.Create(configFilePath)
  1813. assert.NilError(c, err)
  1814. daemonConfig = `{ "max-concurrent-uploads" : 7, "max-concurrent-downloads" : 9 }`
  1815. fmt.Fprintf(configFile, "%s", daemonConfig)
  1816. configFile.Close()
  1817. assert.Assert(c, s.d.Signal(unix.SIGHUP) == nil)
  1818. // unix.Kill(s.d.cmd.Process.Pid, unix.SIGHUP)
  1819. time.Sleep(3 * time.Second)
  1820. expectedMaxConcurrentUploads = `level=debug msg="Reset Max Concurrent Uploads: 7"`
  1821. expectedMaxConcurrentDownloads = `level=debug msg="Reset Max Concurrent Downloads: 9"`
  1822. content, err = s.d.ReadLogFile()
  1823. assert.NilError(c, err)
  1824. assert.Assert(c, strings.Contains(string(content), expectedMaxConcurrentUploads))
  1825. assert.Assert(c, strings.Contains(string(content), expectedMaxConcurrentDownloads))
  1826. }
  1827. // Test case for #20936, #22443
  1828. func (s *DockerDaemonSuite) TestDaemonMaxConcurrencyWithConfigFileReload(c *testing.T) {
  1829. testRequires(c, testEnv.IsLocalDaemon, DaemonIsLinux)
  1830. // daemon config file
  1831. configFilePath := "test.json"
  1832. configFile, err := os.Create(configFilePath)
  1833. assert.NilError(c, err)
  1834. defer os.Remove(configFilePath)
  1835. daemonConfig := `{ "max-concurrent-uploads" : null }`
  1836. fmt.Fprintf(configFile, "%s", daemonConfig)
  1837. configFile.Close()
  1838. s.d.Start(c, fmt.Sprintf("--config-file=%s", configFilePath))
  1839. expectedMaxConcurrentUploads := `level=debug msg="Max Concurrent Uploads: 5"`
  1840. expectedMaxConcurrentDownloads := `level=debug msg="Max Concurrent Downloads: 3"`
  1841. content, err := s.d.ReadLogFile()
  1842. assert.NilError(c, err)
  1843. assert.Assert(c, strings.Contains(string(content), expectedMaxConcurrentUploads))
  1844. assert.Assert(c, strings.Contains(string(content), expectedMaxConcurrentDownloads))
  1845. configFile, err = os.Create(configFilePath)
  1846. assert.NilError(c, err)
  1847. daemonConfig = `{ "max-concurrent-uploads" : 1, "max-concurrent-downloads" : null }`
  1848. fmt.Fprintf(configFile, "%s", daemonConfig)
  1849. configFile.Close()
  1850. assert.Assert(c, s.d.Signal(unix.SIGHUP) == nil)
  1851. // unix.Kill(s.d.cmd.Process.Pid, unix.SIGHUP)
  1852. time.Sleep(3 * time.Second)
  1853. expectedMaxConcurrentUploads = `level=debug msg="Reset Max Concurrent Uploads: 1"`
  1854. expectedMaxConcurrentDownloads = `level=debug msg="Reset Max Concurrent Downloads: 3"`
  1855. content, err = s.d.ReadLogFile()
  1856. assert.NilError(c, err)
  1857. assert.Assert(c, strings.Contains(string(content), expectedMaxConcurrentUploads))
  1858. assert.Assert(c, strings.Contains(string(content), expectedMaxConcurrentDownloads))
  1859. configFile, err = os.Create(configFilePath)
  1860. assert.NilError(c, err)
  1861. daemonConfig = `{ "labels":["foo=bar"] }`
  1862. fmt.Fprintf(configFile, "%s", daemonConfig)
  1863. configFile.Close()
  1864. assert.Assert(c, s.d.Signal(unix.SIGHUP) == nil)
  1865. time.Sleep(3 * time.Second)
  1866. expectedMaxConcurrentUploads = `level=debug msg="Reset Max Concurrent Uploads: 5"`
  1867. expectedMaxConcurrentDownloads = `level=debug msg="Reset Max Concurrent Downloads: 3"`
  1868. content, err = s.d.ReadLogFile()
  1869. assert.NilError(c, err)
  1870. assert.Assert(c, strings.Contains(string(content), expectedMaxConcurrentUploads))
  1871. assert.Assert(c, strings.Contains(string(content), expectedMaxConcurrentDownloads))
  1872. }
  1873. func (s *DockerDaemonSuite) TestBuildOnDisabledBridgeNetworkDaemon(c *testing.T) {
  1874. s.d.StartWithBusybox(c, "-b=none", "--iptables=false")
  1875. result := cli.BuildCmd(c, "busyboxs", cli.Daemon(s.d),
  1876. build.WithDockerfile(`
  1877. FROM busybox
  1878. RUN cat /etc/hosts`),
  1879. build.WithoutCache,
  1880. )
  1881. comment := fmt.Sprintf("Failed to build image. output %s, exitCode %d, err %v", result.Combined(), result.ExitCode, result.Error)
  1882. assert.Assert(c, result.Error == nil, comment)
  1883. assert.Equal(c, result.ExitCode, 0, comment)
  1884. }
  1885. // Test case for #21976
  1886. func (s *DockerDaemonSuite) TestDaemonDNSFlagsInHostMode(c *testing.T) {
  1887. testRequires(c, testEnv.IsLocalDaemon, DaemonIsLinux)
  1888. s.d.StartWithBusybox(c, "--dns", "1.2.3.4", "--dns-search", "example.com", "--dns-opt", "timeout:3")
  1889. expectedOutput := "nameserver 1.2.3.4"
  1890. out, _ := s.d.Cmd("run", "--net=host", "busybox", "cat", "/etc/resolv.conf")
  1891. assert.Assert(c, strings.Contains(out, expectedOutput), "Expected '%s', but got %q", expectedOutput, out)
  1892. expectedOutput = "search example.com"
  1893. assert.Assert(c, strings.Contains(out, expectedOutput), "Expected '%s', but got %q", expectedOutput, out)
  1894. expectedOutput = "options timeout:3"
  1895. assert.Assert(c, strings.Contains(out, expectedOutput), "Expected '%s', but got %q", expectedOutput, out)
  1896. }
  1897. func (s *DockerDaemonSuite) TestRunWithRuntimeFromConfigFile(c *testing.T) {
  1898. conf, err := os.CreateTemp("", "config-file-")
  1899. assert.NilError(c, err)
  1900. configName := conf.Name()
  1901. conf.Close()
  1902. defer os.Remove(configName)
  1903. config := `
  1904. {
  1905. "runtimes": {
  1906. "oci": {
  1907. "path": "runc"
  1908. },
  1909. "vm": {
  1910. "path": "/usr/local/bin/vm-manager",
  1911. "runtimeArgs": [
  1912. "--debug"
  1913. ]
  1914. }
  1915. }
  1916. }
  1917. `
  1918. os.WriteFile(configName, []byte(config), 0644)
  1919. s.d.StartWithBusybox(c, "--config-file", configName)
  1920. // Run with default runtime
  1921. out, err := s.d.Cmd("run", "--rm", "busybox", "ls")
  1922. assert.NilError(c, err, out)
  1923. // Run with default runtime explicitly
  1924. out, err = s.d.Cmd("run", "--rm", "--runtime=runc", "busybox", "ls")
  1925. assert.NilError(c, err, out)
  1926. // Run with oci (same path as default) but keep it around
  1927. out, err = s.d.Cmd("run", "--name", "oci-runtime-ls", "--runtime=oci", "busybox", "ls")
  1928. assert.NilError(c, err, out)
  1929. // Run with "vm"
  1930. out, err = s.d.Cmd("run", "--rm", "--runtime=vm", "busybox", "ls")
  1931. assert.ErrorContains(c, err, "", out)
  1932. assert.Assert(c, is.Contains(out, "/usr/local/bin/vm-manager: no such file or directory"))
  1933. // Reset config to only have the default
  1934. config = `
  1935. {
  1936. "runtimes": {
  1937. }
  1938. }
  1939. `
  1940. os.WriteFile(configName, []byte(config), 0644)
  1941. assert.Assert(c, s.d.Signal(unix.SIGHUP) == nil)
  1942. // Give daemon time to reload config
  1943. <-time.After(1 * time.Second)
  1944. // Run with default runtime
  1945. out, err = s.d.Cmd("run", "--rm", "--runtime=runc", "busybox", "ls")
  1946. assert.NilError(c, err, out)
  1947. // Run with "oci"
  1948. out, err = s.d.Cmd("run", "--rm", "--runtime=oci", "busybox", "ls")
  1949. assert.ErrorContains(c, err, "", out)
  1950. assert.Assert(c, is.Contains(out, "unknown or invalid runtime name: oci"))
  1951. // Start previously created container with oci
  1952. out, err = s.d.Cmd("start", "oci-runtime-ls")
  1953. assert.ErrorContains(c, err, "", out)
  1954. assert.Assert(c, is.Contains(out, "unknown or invalid runtime name: oci"))
  1955. // Check that we can't override the default runtime
  1956. config = `
  1957. {
  1958. "runtimes": {
  1959. "runc": {
  1960. "path": "my-runc"
  1961. }
  1962. }
  1963. }
  1964. `
  1965. os.WriteFile(configName, []byte(config), 0644)
  1966. assert.Assert(c, s.d.Signal(unix.SIGHUP) == nil)
  1967. // Give daemon time to reload config
  1968. <-time.After(1 * time.Second)
  1969. content, err := s.d.ReadLogFile()
  1970. assert.NilError(c, err)
  1971. assert.Assert(c, is.Contains(string(content), `file configuration validation failed: runtime name 'runc' is reserved`))
  1972. // Check that we can select a default runtime
  1973. config = `
  1974. {
  1975. "default-runtime": "vm",
  1976. "runtimes": {
  1977. "oci": {
  1978. "path": "runc"
  1979. },
  1980. "vm": {
  1981. "path": "/usr/local/bin/vm-manager",
  1982. "runtimeArgs": [
  1983. "--debug"
  1984. ]
  1985. }
  1986. }
  1987. }
  1988. `
  1989. os.WriteFile(configName, []byte(config), 0644)
  1990. assert.Assert(c, s.d.Signal(unix.SIGHUP) == nil)
  1991. // Give daemon time to reload config
  1992. <-time.After(1 * time.Second)
  1993. out, err = s.d.Cmd("run", "--rm", "busybox", "ls")
  1994. assert.ErrorContains(c, err, "", out)
  1995. assert.Assert(c, is.Contains(out, "/usr/local/bin/vm-manager: no such file or directory"))
  1996. // Run with default runtime explicitly
  1997. out, err = s.d.Cmd("run", "--rm", "--runtime=runc", "busybox", "ls")
  1998. assert.NilError(c, err, out)
  1999. }
  2000. func (s *DockerDaemonSuite) TestRunWithRuntimeFromCommandLine(c *testing.T) {
  2001. s.d.StartWithBusybox(c, "--add-runtime", "oci=runc", "--add-runtime", "vm=/usr/local/bin/vm-manager")
  2002. // Run with default runtime
  2003. out, err := s.d.Cmd("run", "--rm", "busybox", "ls")
  2004. assert.NilError(c, err, out)
  2005. // Run with default runtime explicitly
  2006. out, err = s.d.Cmd("run", "--rm", "--runtime=runc", "busybox", "ls")
  2007. assert.NilError(c, err, out)
  2008. // Run with oci (same path as default) but keep it around
  2009. out, err = s.d.Cmd("run", "--name", "oci-runtime-ls", "--runtime=oci", "busybox", "ls")
  2010. assert.NilError(c, err, out)
  2011. // Run with "vm"
  2012. out, err = s.d.Cmd("run", "--rm", "--runtime=vm", "busybox", "ls")
  2013. assert.ErrorContains(c, err, "", out)
  2014. assert.Assert(c, is.Contains(out, "/usr/local/bin/vm-manager: no such file or directory"))
  2015. // Start a daemon without any extra runtimes
  2016. s.d.Stop(c)
  2017. s.d.StartWithBusybox(c)
  2018. // Run with default runtime
  2019. out, err = s.d.Cmd("run", "--rm", "--runtime=runc", "busybox", "ls")
  2020. assert.NilError(c, err, out)
  2021. // Run with "oci"
  2022. out, err = s.d.Cmd("run", "--rm", "--runtime=oci", "busybox", "ls")
  2023. assert.ErrorContains(c, err, "", out)
  2024. assert.Assert(c, is.Contains(out, "unknown or invalid runtime name: oci"))
  2025. // Start previously created container with oci
  2026. out, err = s.d.Cmd("start", "oci-runtime-ls")
  2027. assert.ErrorContains(c, err, "", out)
  2028. assert.Assert(c, is.Contains(out, "unknown or invalid runtime name: oci"))
  2029. // Check that we can't override the default runtime
  2030. s.d.Stop(c)
  2031. assert.Assert(c, s.d.StartWithError("--add-runtime", "runc=my-runc") != nil)
  2032. content, err := s.d.ReadLogFile()
  2033. assert.NilError(c, err)
  2034. assert.Assert(c, is.Contains(string(content), `runtime name 'runc' is reserved`))
  2035. // Check that we can select a default runtime
  2036. s.d.Stop(c)
  2037. s.d.StartWithBusybox(c, "--default-runtime=vm", "--add-runtime", "oci=runc", "--add-runtime", "vm=/usr/local/bin/vm-manager")
  2038. out, err = s.d.Cmd("run", "--rm", "busybox", "ls")
  2039. assert.ErrorContains(c, err, "", out)
  2040. assert.Assert(c, is.Contains(out, "/usr/local/bin/vm-manager: no such file or directory"))
  2041. // Run with default runtime explicitly
  2042. out, err = s.d.Cmd("run", "--rm", "--runtime=runc", "busybox", "ls")
  2043. assert.NilError(c, err, out)
  2044. }
  2045. func (s *DockerDaemonSuite) TestDaemonRestartWithAutoRemoveContainer(c *testing.T) {
  2046. s.d.StartWithBusybox(c)
  2047. // top1 will exist after daemon restarts
  2048. out, err := s.d.Cmd("run", "-d", "--name", "top1", "busybox:latest", "top")
  2049. assert.Assert(c, err == nil, "run top1: %v", out)
  2050. // top2 will be removed after daemon restarts
  2051. out, err = s.d.Cmd("run", "-d", "--rm", "--name", "top2", "busybox:latest", "top")
  2052. assert.Assert(c, err == nil, "run top2: %v", out)
  2053. out, err = s.d.Cmd("ps")
  2054. assert.NilError(c, err)
  2055. assert.Assert(c, strings.Contains(out, "top1"), "top1 should be running")
  2056. assert.Assert(c, strings.Contains(out, "top2"), "top2 should be running")
  2057. // now restart daemon gracefully
  2058. s.d.Restart(c)
  2059. out, err = s.d.Cmd("ps", "-a")
  2060. assert.NilError(c, err, "out: %v", out)
  2061. assert.Assert(c, strings.Contains(out, "top1"), "top1 should exist after daemon restarts")
  2062. assert.Assert(c, !strings.Contains(out, "top2"), "top2 should be removed after daemon restarts")
  2063. }
  2064. func (s *DockerDaemonSuite) TestDaemonRestartSaveContainerExitCode(c *testing.T) {
  2065. s.d.StartWithBusybox(c)
  2066. containerName := "error-values"
  2067. // Make a container with both a non 0 exit code and an error message
  2068. // We explicitly disable `--init` for this test, because `--init` is enabled by default
  2069. // on "experimental". Enabling `--init` results in a different behavior; because the "init"
  2070. // process itself is PID1, the container does not fail on _startup_ (i.e., `docker-init` starting),
  2071. // but directly after. The exit code of the container is still 127, but the Error Message is not
  2072. // captured, so `.State.Error` is empty.
  2073. // See the discussion on https://github.com/docker/docker/pull/30227#issuecomment-274161426,
  2074. // and https://github.com/docker/docker/pull/26061#r78054578 for more information.
  2075. _, err := s.d.Cmd("run", "--name", containerName, "--init=false", "busybox", "toto")
  2076. assert.ErrorContains(c, err, "")
  2077. // Check that those values were saved on disk
  2078. out, err := s.d.Cmd("inspect", "-f", "{{.State.ExitCode}}", containerName)
  2079. out = strings.TrimSpace(out)
  2080. assert.NilError(c, err)
  2081. assert.Equal(c, out, "127")
  2082. errMsg1, err := s.d.Cmd("inspect", "-f", "{{.State.Error}}", containerName)
  2083. errMsg1 = strings.TrimSpace(errMsg1)
  2084. assert.NilError(c, err)
  2085. assert.Assert(c, strings.Contains(errMsg1, "executable file not found"))
  2086. // now restart daemon
  2087. s.d.Restart(c)
  2088. // Check that those values are still around
  2089. out, err = s.d.Cmd("inspect", "-f", "{{.State.ExitCode}}", containerName)
  2090. out = strings.TrimSpace(out)
  2091. assert.NilError(c, err)
  2092. assert.Equal(c, out, "127")
  2093. out, err = s.d.Cmd("inspect", "-f", "{{.State.Error}}", containerName)
  2094. out = strings.TrimSpace(out)
  2095. assert.NilError(c, err)
  2096. assert.Equal(c, out, errMsg1)
  2097. }
  2098. func (s *DockerDaemonSuite) TestDaemonWithUserlandProxyPath(c *testing.T) {
  2099. testRequires(c, testEnv.IsLocalDaemon, DaemonIsLinux)
  2100. dockerProxyPath, err := exec.LookPath("docker-proxy")
  2101. assert.NilError(c, err)
  2102. tmpDir, err := os.MkdirTemp("", "test-docker-proxy")
  2103. assert.NilError(c, err)
  2104. newProxyPath := filepath.Join(tmpDir, "docker-proxy")
  2105. cmd := exec.Command("cp", dockerProxyPath, newProxyPath)
  2106. assert.NilError(c, cmd.Run())
  2107. // custom one
  2108. s.d.StartWithBusybox(c, "--userland-proxy-path", newProxyPath)
  2109. out, err := s.d.Cmd("run", "-p", "5000:5000", "busybox:latest", "true")
  2110. assert.NilError(c, err, out)
  2111. // try with the original one
  2112. s.d.Restart(c, "--userland-proxy-path", dockerProxyPath)
  2113. out, err = s.d.Cmd("run", "-p", "5000:5000", "busybox:latest", "true")
  2114. assert.NilError(c, err, out)
  2115. // not exist
  2116. s.d.Restart(c, "--userland-proxy-path", "/does/not/exist")
  2117. out, err = s.d.Cmd("run", "-p", "5000:5000", "busybox:latest", "true")
  2118. assert.ErrorContains(c, err, "", out)
  2119. assert.Assert(c, strings.Contains(out, "driver failed programming external connectivity on endpoint"))
  2120. assert.Assert(c, strings.Contains(out, "/does/not/exist: no such file or directory"))
  2121. }
  2122. // Test case for #22471
  2123. func (s *DockerDaemonSuite) TestDaemonShutdownTimeout(c *testing.T) {
  2124. testRequires(c, testEnv.IsLocalDaemon)
  2125. s.d.StartWithBusybox(c, "--shutdown-timeout=3")
  2126. _, err := s.d.Cmd("run", "-d", "busybox", "top")
  2127. assert.NilError(c, err)
  2128. assert.Assert(c, s.d.Signal(unix.SIGINT) == nil)
  2129. select {
  2130. case <-s.d.Wait:
  2131. case <-time.After(5 * time.Second):
  2132. }
  2133. expectedMessage := `level=debug msg="daemon configured with a 3 seconds minimum shutdown timeout"`
  2134. content, err := s.d.ReadLogFile()
  2135. assert.NilError(c, err)
  2136. assert.Assert(c, strings.Contains(string(content), expectedMessage))
  2137. }
  2138. // Test case for #22471
  2139. func (s *DockerDaemonSuite) TestDaemonShutdownTimeoutWithConfigFile(c *testing.T) {
  2140. testRequires(c, testEnv.IsLocalDaemon)
  2141. // daemon config file
  2142. configFilePath := "test.json"
  2143. configFile, err := os.Create(configFilePath)
  2144. assert.NilError(c, err)
  2145. defer os.Remove(configFilePath)
  2146. daemonConfig := `{ "shutdown-timeout" : 8 }`
  2147. fmt.Fprintf(configFile, "%s", daemonConfig)
  2148. configFile.Close()
  2149. s.d.Start(c, fmt.Sprintf("--config-file=%s", configFilePath))
  2150. configFile, err = os.Create(configFilePath)
  2151. assert.NilError(c, err)
  2152. daemonConfig = `{ "shutdown-timeout" : 5 }`
  2153. fmt.Fprintf(configFile, "%s", daemonConfig)
  2154. configFile.Close()
  2155. assert.Assert(c, s.d.Signal(unix.SIGHUP) == nil)
  2156. select {
  2157. case <-s.d.Wait:
  2158. case <-time.After(3 * time.Second):
  2159. }
  2160. expectedMessage := `level=debug msg="Reset Shutdown Timeout: 5"`
  2161. content, err := s.d.ReadLogFile()
  2162. assert.NilError(c, err)
  2163. assert.Assert(c, strings.Contains(string(content), expectedMessage))
  2164. }
  2165. // Test case for 29342
  2166. func (s *DockerDaemonSuite) TestExecWithUserAfterLiveRestore(c *testing.T) {
  2167. testRequires(c, DaemonIsLinux)
  2168. s.d.StartWithBusybox(c, "--live-restore")
  2169. out, err := s.d.Cmd("run", "--init", "-d", "--name=top", "busybox", "sh", "-c", "addgroup -S test && adduser -S -G test test -D -s /bin/sh && touch /adduser_end && exec top")
  2170. assert.NilError(c, err, "Output: %s", out)
  2171. s.d.WaitRun("top")
  2172. // Wait for shell command to be completed
  2173. _, err = s.d.Cmd("exec", "top", "sh", "-c", `for i in $(seq 1 5); do if [ -e /adduser_end ]; then rm -f /adduser_end && break; else sleep 1 && false; fi; done`)
  2174. assert.Assert(c, err == nil, "Timeout waiting for shell command to be completed")
  2175. out1, err := s.d.Cmd("exec", "-u", "test", "top", "id")
  2176. // uid=100(test) gid=101(test) groups=101(test)
  2177. assert.Assert(c, err == nil, "Output: %s", out1)
  2178. // restart daemon.
  2179. s.d.Restart(c, "--live-restore")
  2180. out2, err := s.d.Cmd("exec", "-u", "test", "top", "id")
  2181. assert.Assert(c, err == nil, "Output: %s", out2)
  2182. assert.Equal(c, out2, out1, fmt.Sprintf("Output: before restart '%s', after restart '%s'", out1, out2))
  2183. out, err = s.d.Cmd("stop", "top")
  2184. assert.NilError(c, err, "Output: %s", out)
  2185. }
  2186. func (s *DockerDaemonSuite) TestRemoveContainerAfterLiveRestore(c *testing.T) {
  2187. testRequires(c, DaemonIsLinux, overlayFSSupported, testEnv.IsLocalDaemon)
  2188. s.d.StartWithBusybox(c, "--live-restore", "--storage-driver", "overlay2")
  2189. out, err := s.d.Cmd("run", "-d", "--name=top", "busybox", "top")
  2190. assert.NilError(c, err, "Output: %s", out)
  2191. s.d.WaitRun("top")
  2192. // restart daemon.
  2193. s.d.Restart(c, "--live-restore", "--storage-driver", "overlay2")
  2194. out, err = s.d.Cmd("stop", "top")
  2195. assert.NilError(c, err, "Output: %s", out)
  2196. // test if the rootfs mountpoint still exist
  2197. mountpoint, err := s.d.InspectField("top", ".GraphDriver.Data.MergedDir")
  2198. assert.NilError(c, err)
  2199. f, err := os.Open("/proc/self/mountinfo")
  2200. assert.NilError(c, err)
  2201. defer f.Close()
  2202. sc := bufio.NewScanner(f)
  2203. for sc.Scan() {
  2204. line := sc.Text()
  2205. if strings.Contains(line, mountpoint) {
  2206. c.Fatalf("mountinfo should not include the mountpoint of stop container")
  2207. }
  2208. }
  2209. out, err = s.d.Cmd("rm", "top")
  2210. assert.NilError(c, err, "Output: %s", out)
  2211. }
  2212. // #29598
  2213. func (s *DockerDaemonSuite) TestRestartPolicyWithLiveRestore(c *testing.T) {
  2214. testRequires(c, DaemonIsLinux, testEnv.IsLocalDaemon)
  2215. s.d.StartWithBusybox(c, "--live-restore")
  2216. out, err := s.d.Cmd("run", "-d", "--restart", "always", "busybox", "top")
  2217. assert.NilError(c, err, "Output: %s", out)
  2218. id := strings.TrimSpace(out)
  2219. type state struct {
  2220. Running bool
  2221. StartedAt time.Time
  2222. }
  2223. out, err = s.d.Cmd("inspect", "-f", "{{json .State}}", id)
  2224. assert.Assert(c, err == nil, "output: %s", out)
  2225. var origState state
  2226. err = json.Unmarshal([]byte(strings.TrimSpace(out)), &origState)
  2227. assert.NilError(c, err)
  2228. s.d.Restart(c, "--live-restore")
  2229. pid, err := s.d.Cmd("inspect", "-f", "{{.State.Pid}}", id)
  2230. assert.NilError(c, err)
  2231. pidint, err := strconv.Atoi(strings.TrimSpace(pid))
  2232. assert.NilError(c, err)
  2233. assert.Assert(c, pidint > 0)
  2234. assert.NilError(c, unix.Kill(pidint, unix.SIGKILL))
  2235. ticker := time.NewTicker(50 * time.Millisecond)
  2236. timeout := time.After(10 * time.Second)
  2237. for range ticker.C {
  2238. select {
  2239. case <-timeout:
  2240. c.Fatal("timeout waiting for container restart")
  2241. default:
  2242. }
  2243. out, err := s.d.Cmd("inspect", "-f", "{{json .State}}", id)
  2244. assert.Assert(c, err == nil, "output: %s", out)
  2245. var newState state
  2246. err = json.Unmarshal([]byte(strings.TrimSpace(out)), &newState)
  2247. assert.NilError(c, err)
  2248. if !newState.Running {
  2249. continue
  2250. }
  2251. if newState.StartedAt.After(origState.StartedAt) {
  2252. break
  2253. }
  2254. }
  2255. out, err = s.d.Cmd("stop", id)
  2256. assert.NilError(c, err, "Output: %s", out)
  2257. }
  2258. func (s *DockerDaemonSuite) TestShmSize(c *testing.T) {
  2259. testRequires(c, DaemonIsLinux)
  2260. size := 67108864 * 2
  2261. pattern := regexp.MustCompile(fmt.Sprintf("shm on /dev/shm type tmpfs(.*)size=%dk", size/1024))
  2262. s.d.StartWithBusybox(c, "--default-shm-size", fmt.Sprintf("%v", size))
  2263. name := "shm1"
  2264. out, err := s.d.Cmd("run", "--name", name, "busybox", "mount")
  2265. assert.NilError(c, err, "Output: %s", out)
  2266. assert.Assert(c, pattern.MatchString(out))
  2267. out, err = s.d.Cmd("inspect", "--format", "{{.HostConfig.ShmSize}}", name)
  2268. assert.NilError(c, err, "Output: %s", out)
  2269. assert.Equal(c, strings.TrimSpace(out), fmt.Sprintf("%v", size))
  2270. }
  2271. func (s *DockerDaemonSuite) TestShmSizeReload(c *testing.T) {
  2272. testRequires(c, DaemonIsLinux)
  2273. configPath, err := os.MkdirTemp("", "test-daemon-shm-size-reload-config")
  2274. assert.Assert(c, err == nil, "could not create temp file for config reload")
  2275. defer os.RemoveAll(configPath) // clean up
  2276. configFile := filepath.Join(configPath, "config.json")
  2277. size := 67108864 * 2
  2278. configData := []byte(fmt.Sprintf(`{"default-shm-size": "%dM"}`, size/1024/1024))
  2279. assert.Assert(c, os.WriteFile(configFile, configData, 0666) == nil, "could not write temp file for config reload")
  2280. pattern := regexp.MustCompile(fmt.Sprintf("shm on /dev/shm type tmpfs(.*)size=%dk", size/1024))
  2281. s.d.StartWithBusybox(c, "--config-file", configFile)
  2282. name := "shm1"
  2283. out, err := s.d.Cmd("run", "--name", name, "busybox", "mount")
  2284. assert.NilError(c, err, "Output: %s", out)
  2285. assert.Assert(c, pattern.MatchString(out))
  2286. out, err = s.d.Cmd("inspect", "--format", "{{.HostConfig.ShmSize}}", name)
  2287. assert.NilError(c, err, "Output: %s", out)
  2288. assert.Equal(c, strings.TrimSpace(out), fmt.Sprintf("%v", size))
  2289. size = 67108864 * 3
  2290. configData = []byte(fmt.Sprintf(`{"default-shm-size": "%dM"}`, size/1024/1024))
  2291. assert.Assert(c, os.WriteFile(configFile, configData, 0666) == nil, "could not write temp file for config reload")
  2292. pattern = regexp.MustCompile(fmt.Sprintf("shm on /dev/shm type tmpfs(.*)size=%dk", size/1024))
  2293. err = s.d.ReloadConfig()
  2294. assert.Assert(c, err == nil, "error reloading daemon config")
  2295. name = "shm2"
  2296. out, err = s.d.Cmd("run", "--name", name, "busybox", "mount")
  2297. assert.NilError(c, err, "Output: %s", out)
  2298. assert.Assert(c, pattern.MatchString(out))
  2299. out, err = s.d.Cmd("inspect", "--format", "{{.HostConfig.ShmSize}}", name)
  2300. assert.NilError(c, err, "Output: %s", out)
  2301. assert.Equal(c, strings.TrimSpace(out), fmt.Sprintf("%v", size))
  2302. }
  2303. func testDaemonStartIpcMode(c *testing.T, from, mode string, valid bool) {
  2304. d := daemon.New(c, dockerBinary, dockerdBinary, testdaemon.WithEnvironment(testEnv.Execution))
  2305. c.Logf("Checking IpcMode %s set from %s\n", mode, from)
  2306. var serr error
  2307. switch from {
  2308. case "config":
  2309. f, err := os.CreateTemp("", "test-daemon-ipc-config")
  2310. assert.NilError(c, err)
  2311. defer os.Remove(f.Name())
  2312. config := `{"default-ipc-mode": "` + mode + `"}`
  2313. _, err = f.WriteString(config)
  2314. assert.NilError(c, f.Close())
  2315. assert.NilError(c, err)
  2316. serr = d.StartWithError("--config-file", f.Name())
  2317. case "cli":
  2318. serr = d.StartWithError("--default-ipc-mode", mode)
  2319. default:
  2320. c.Fatalf("testDaemonStartIpcMode: invalid 'from' argument")
  2321. }
  2322. if serr == nil {
  2323. d.Stop(c)
  2324. }
  2325. if valid {
  2326. assert.NilError(c, serr)
  2327. } else {
  2328. assert.ErrorContains(c, serr, "")
  2329. icmd.RunCommand("grep", "-E", "IPC .* is (invalid|not supported)", d.LogFileName()).Assert(c, icmd.Success)
  2330. }
  2331. }
  2332. // TestDaemonStartWithIpcModes checks that daemon starts fine given correct
  2333. // arguments for default IPC mode, and bails out with incorrect ones.
  2334. // Both CLI option (--default-ipc-mode) and config parameter are tested.
  2335. func (s *DockerDaemonSuite) TestDaemonStartWithIpcModes(c *testing.T) {
  2336. testRequires(c, DaemonIsLinux, testEnv.IsLocalDaemon)
  2337. ipcModes := []struct {
  2338. mode string
  2339. valid bool
  2340. }{
  2341. {"private", true},
  2342. {"shareable", true},
  2343. {"host", false},
  2344. {"container:123", false},
  2345. {"nosuchmode", false},
  2346. }
  2347. for _, from := range []string{"config", "cli"} {
  2348. for _, m := range ipcModes {
  2349. testDaemonStartIpcMode(c, from, m.mode, m.valid)
  2350. }
  2351. }
  2352. }
  2353. // TestFailedPluginRemove makes sure that a failed plugin remove does not block
  2354. // the daemon from starting
  2355. func (s *DockerDaemonSuite) TestFailedPluginRemove(c *testing.T) {
  2356. testRequires(c, DaemonIsLinux, IsAmd64, testEnv.IsLocalDaemon)
  2357. d := daemon.New(c, dockerBinary, dockerdBinary)
  2358. d.Start(c)
  2359. cli := d.NewClientT(c)
  2360. ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second)
  2361. defer cancel()
  2362. name := "test-plugin-rm-fail"
  2363. out, err := cli.PluginInstall(ctx, name, types.PluginInstallOptions{
  2364. Disabled: true,
  2365. AcceptAllPermissions: true,
  2366. RemoteRef: "cpuguy83/docker-logdriver-test",
  2367. })
  2368. assert.NilError(c, err)
  2369. defer out.Close()
  2370. io.Copy(io.Discard, out)
  2371. ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second)
  2372. defer cancel()
  2373. p, _, err := cli.PluginInspectWithRaw(ctx, name)
  2374. assert.NilError(c, err)
  2375. // simulate a bad/partial removal by removing the plugin config.
  2376. configPath := filepath.Join(d.Root, "plugins", p.ID, "config.json")
  2377. assert.NilError(c, os.Remove(configPath))
  2378. d.Restart(c)
  2379. ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second)
  2380. defer cancel()
  2381. _, err = cli.Ping(ctx)
  2382. assert.NilError(c, err)
  2383. _, _, err = cli.PluginInspectWithRaw(ctx, name)
  2384. // plugin should be gone since the config.json is gone
  2385. assert.ErrorContains(c, err, "")
  2386. }