From b682a8ea9e9325278cdb08d5f3da821ee97c5feb Mon Sep 17 00:00:00 2001 From: Ken Cochrane Date: Thu, 2 May 2013 14:32:38 -0400 Subject: [PATCH 01/97] added Docker Index search API doc --- docs/sources/index.rst | 1 + docs/sources/index/index.rst | 15 +++++++++++++++ docs/sources/index/search.rst | 20 ++++++++++++++++++++ 3 files changed, 36 insertions(+) create mode 100644 docs/sources/index/index.rst create mode 100644 docs/sources/index/search.rst diff --git a/docs/sources/index.rst b/docs/sources/index.rst index 4e724a0cd2..e6a1482ccd 100644 --- a/docs/sources/index.rst +++ b/docs/sources/index.rst @@ -16,6 +16,7 @@ This documentation has the following resources: contributing/index commandline/index registry/index + index/index faq diff --git a/docs/sources/index/index.rst b/docs/sources/index/index.rst new file mode 100644 index 0000000000..7637a4e779 --- /dev/null +++ b/docs/sources/index/index.rst @@ -0,0 +1,15 @@ +:title: Docker Index documentation +:description: Documentation for docker Index +:keywords: docker, index, api + + + +Index +===== + +Contents: + +.. toctree:: + :maxdepth: 2 + + search diff --git a/docs/sources/index/search.rst b/docs/sources/index/search.rst new file mode 100644 index 0000000000..0c7624de72 --- /dev/null +++ b/docs/sources/index/search.rst @@ -0,0 +1,20 @@ +======================= +Docker Index Search API +======================= + +Search +------ + +**URL:** /v1/search?q={{search_term}} + +**Results:** + +.. code-block:: json + + {"query":"{{search_term}}", + "num_results": 27, + "results" : [ + {"name": "dotcloud/base", "description": "A base ubuntu64 image..."}, + {"name": "base2", "description": "A base ubuntu64 image..."}, + ] + } \ No newline at end of file From f4de9d919d7ad6cf677c8157eabf82cf65aa4396 Mon Sep 17 00:00:00 2001 From: Nate Jones Date: Tue, 30 Apr 2013 22:39:48 -0700 Subject: [PATCH 02/97] add image graph output (dot/graphviz) --- commands.go | 171 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 106 insertions(+), 65 deletions(-) diff --git a/commands.go b/commands.go index 95a6753b95..3593e2463c 100644 --- a/commands.go +++ b/commands.go @@ -608,85 +608,126 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri //limit := cmd.Int("l", 0, "Only show the N most recent versions of each image") quiet := cmd.Bool("q", false, "only show numeric IDs") flAll := cmd.Bool("a", false, "show all images") + flViz := cmd.Bool("viz", false, "output graph in graphviz format") if err := cmd.Parse(args); err != nil { return nil } - if cmd.NArg() > 1 { - cmd.Usage() - return nil - } - var nameFilter string - if cmd.NArg() == 1 { - nameFilter = cmd.Arg(0) - } - w := tabwriter.NewWriter(stdout, 20, 1, 3, ' ', 0) - if !*quiet { - fmt.Fprintln(w, "REPOSITORY\tTAG\tID\tCREATED") - } - var allImages map[string]*Image - var err error - if *flAll { - allImages, err = srv.runtime.graph.Map() - } else { - allImages, err = srv.runtime.graph.Heads() - } - if err != nil { - return err - } - for name, repository := range srv.runtime.repositories.Repositories { - if nameFilter != "" && name != nameFilter { - continue + + if *flViz { + images, _ := srv.runtime.graph.All() + if images == nil { + return nil } - for tag, id := range repository { - image, err := srv.runtime.graph.Get(id) + + fmt.Fprintf(stdout, "digraph G {\n") + + var parentImage *Image + var err error + for _, image := range images { + parentImage, err = image.GetParent() if err != nil { - log.Printf("Warning: couldn't load %s from %s/%s: %s", id, name, tag, err) + fmt.Errorf("Error while getting parent image: %v", err) + return nil + } + if parentImage != nil { + fmt.Fprintf(stdout, " \"%s\" -> \"%s\"\n", parentImage.ShortId(), image.ShortId()) + } else { + fmt.Fprintf(stdout, " base -> \"%s\" [style=invis]\n", image.ShortId()) + } + } + + reporefs := make(map[string][]string) + + for name, repository := range srv.runtime.repositories.Repositories { + for tag, id := range repository { + reporefs[TruncateId(id)] = append(reporefs[TruncateId(id)], fmt.Sprintf("%s:%s", name, tag)) + } + } + + for id, repos := range reporefs { + fmt.Fprintf(stdout, " \"%s\" [label=\"%s\\n%s\",shape=box,fillcolor=\"paleturquoise\",style=\"filled,rounded\"];\n", id, id, strings.Join(repos, "\\n")) + } + + fmt.Fprintf(stdout, " base [style=invisible]\n") + fmt.Fprintf(stdout, "}\n") + } else { + if cmd.NArg() > 1 { + cmd.Usage() + return nil + } + var nameFilter string + if cmd.NArg() == 1 { + nameFilter = cmd.Arg(0) + } + w := tabwriter.NewWriter(stdout, 20, 1, 3, ' ', 0) + if !*quiet { + fmt.Fprintln(w, "REPOSITORY\tTAG\tID\tCREATED") + } + var allImages map[string]*Image + var err error + if *flAll { + allImages, err = srv.runtime.graph.Map() + } else { + allImages, err = srv.runtime.graph.Heads() + } + if err != nil { + return err + } + for name, repository := range srv.runtime.repositories.Repositories { + if nameFilter != "" && name != nameFilter { continue } - delete(allImages, id) - if !*quiet { - for idx, field := range []string{ - /* REPOSITORY */ name, - /* TAG */ tag, - /* ID */ TruncateId(id), - /* CREATED */ HumanDuration(time.Now().Sub(image.Created)) + " ago", - } { - if idx == 0 { - w.Write([]byte(field)) - } else { - w.Write([]byte("\t" + field)) - } + for tag, id := range repository { + image, err := srv.runtime.graph.Get(id) + if err != nil { + log.Printf("Warning: couldn't load %s from %s/%s: %s", id, name, tag, err) + continue + } + delete(allImages, id) + if !*quiet { + for idx, field := range []string{ + /* REPOSITORY */ name, + /* TAG */ tag, + /* ID */ TruncateId(id), + /* CREATED */ HumanDuration(time.Now().Sub(image.Created)) + " ago", + } { + if idx == 0 { + w.Write([]byte(field)) + } else { + w.Write([]byte("\t" + field)) + } + } + w.Write([]byte{'\n'}) + } else { + stdout.Write([]byte(image.ShortId() + "\n")) } - w.Write([]byte{'\n'}) - } else { - stdout.Write([]byte(image.ShortId() + "\n")) } } - } - // Display images which aren't part of a - if nameFilter == "" { - for id, image := range allImages { - if !*quiet { - for idx, field := range []string{ - /* REPOSITORY */ "", - /* TAG */ "", - /* ID */ TruncateId(id), - /* CREATED */ HumanDuration(time.Now().Sub(image.Created)) + " ago", - } { - if idx == 0 { - w.Write([]byte(field)) - } else { - w.Write([]byte("\t" + field)) + // Display images which aren't part of a + if nameFilter == "" { + for id, image := range allImages { + if !*quiet { + for idx, field := range []string{ + /* REPOSITORY */ "", + /* TAG */ "", + /* ID */ TruncateId(id), + /* CREATED */ HumanDuration(time.Now().Sub(image.Created)) + " ago", + } { + if idx == 0 { + w.Write([]byte(field)) + } else { + w.Write([]byte("\t" + field)) + } } + w.Write([]byte{'\n'}) + } else { + stdout.Write([]byte(image.ShortId() + "\n")) } - w.Write([]byte{'\n'}) - } else { - stdout.Write([]byte(image.ShortId() + "\n")) } } - } - if !*quiet { - w.Flush() + if !*quiet { + w.Flush() + } } return nil } From 3dba4022adbd1f21a8b2ec4a6035cacfe143b380 Mon Sep 17 00:00:00 2001 From: Nate Jones Date: Fri, 3 May 2013 09:47:52 -0700 Subject: [PATCH 03/97] add tests for 'images' subcommand --- commands.go | 2 +- commands_test.go | 71 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/commands.go b/commands.go index 3593e2463c..217732d89a 100644 --- a/commands.go +++ b/commands.go @@ -619,7 +619,7 @@ func (srv *Server) CmdImages(stdin io.ReadCloser, stdout io.Writer, args ...stri return nil } - fmt.Fprintf(stdout, "digraph G {\n") + fmt.Fprintf(stdout, "digraph docker {\n") var parentImage *Image var err error diff --git a/commands_test.go b/commands_test.go index 83b480d52a..3d84dd4b6d 100644 --- a/commands_test.go +++ b/commands_test.go @@ -73,6 +73,77 @@ func cmdWait(srv *Server, container *Container) error { return closeWrap(stdout, stdoutPipe) } +func cmdImages(srv *Server, args ...string) (string, error) { + stdout, stdoutPipe := io.Pipe() + + go func() { + if err := srv.CmdImages(nil, stdoutPipe, args...); err != nil { + return + } + + // force the pipe closed, so that the code below gets an EOF + stdoutPipe.Close() + }() + + output, err := ioutil.ReadAll(stdout) + if err != nil { + return "", err + } + + // Cleanup pipes + return string(output), closeWrap(stdout, stdoutPipe) +} + +// TestImages checks that 'docker images' displays information correctly +func TestImages(t *testing.T) { + + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) + + srv := &Server{runtime: runtime} + + output, err := cmdImages(srv) + + if !strings.Contains(output, "REPOSITORY") { + t.Fatal("'images' should have a header") + } + if !strings.Contains(output, "docker-ut") { + t.Fatal("'images' should show the docker-ut image") + } + if !strings.Contains(output, "e9aa60c60128") { + t.Fatal("'images' should show the docker-ut image id") + } + + output, err = cmdImages(srv, "-q") + + if strings.Contains(output, "REPOSITORY") { + t.Fatal("'images -q' should not have a header") + } + if strings.Contains(output, "docker-ut") { + t.Fatal("'images' should not show the docker-ut image name") + } + if !strings.Contains(output, "e9aa60c60128") { + t.Fatal("'images' should show the docker-ut image id") + } + + output, err = cmdImages(srv, "-viz") + + if !strings.HasPrefix(output, "digraph docker {") { + t.Fatal("'images -v' should start with the dot header") + } + if !strings.HasSuffix(output, "}\n") { + t.Fatal("'images -v' should end with a '}'") + } + if !strings.Contains(output, "base -> \"e9aa60c60128\" [style=invis]") { + t.Fatal("'images -v' should have the docker-ut image id node") + } + + // todo: add checks for -a +} + // TestRunHostname checks that 'docker run -h' correctly sets a custom hostname func TestRunHostname(t *testing.T) { runtime, err := newTestRuntime() From 359ecf88de8b1ed4392ff82aac276353d5d664f0 Mon Sep 17 00:00:00 2001 From: Nate Jones Date: Fri, 3 May 2013 20:44:10 -0700 Subject: [PATCH 04/97] add doc for images -viz --- docs/sources/commandline/command/images.rst | 10 ++++++++++ .../command/images/docker_images.gif | Bin 0 -> 35785 bytes 2 files changed, 10 insertions(+) create mode 100644 docs/sources/commandline/command/images/docker_images.gif diff --git a/docs/sources/commandline/command/images.rst b/docs/sources/commandline/command/images.rst index c21762db05..5bcfe817f4 100644 --- a/docs/sources/commandline/command/images.rst +++ b/docs/sources/commandline/command/images.rst @@ -10,3 +10,13 @@ -a=false: show all images -q=false: only show numeric IDs + -viz=false: output in graphviz format + +Displaying images visually +-------------------------- + +:: + + docker images -viz | dot -Tpng -o docker.png + +.. image:: images/docker_images.gif diff --git a/docs/sources/commandline/command/images/docker_images.gif b/docs/sources/commandline/command/images/docker_images.gif new file mode 100644 index 0000000000000000000000000000000000000000..5894ca270e002758b8f332141e00356e42868880 GIT binary patch literal 35785 zcmd3tS6CBW*sj+UAPFD|HS{DP9Sl`K#LzngL_|~wh=>J0Kv6{0(5o7H5isItu^ai_j`|%i=(+kus7_0yaNA47>mV; zii%50N=ZvgDgGCEGFeScO+{N+4I~zJ0sDe?UNh-Oink491>4dt5?8J@)TEc<>-SB4TS))R7}c z0#5#yKqfOd`M>PV%*@Kl3O{@HU~X<+Ufw^&#nBi3%dyJJnwpvuwY8_3nwp!NnU^mo zUHLC*U0uDsy_wgqpX=`*930FU9?l;hpO}~^zIX4!ga2}IX6DJ0Csj|M*3QpAfByX4 zyLXpAd{|yy{`>du|2*{nFBkF;6bpbi(EIO({u>h@vdW|Ct&fR;w$@>qcrXN0&a7xY`c7GB^N@v$7r*be2ujX3j+*UQ5MKnGzH`-P`a-L+DspQgL zGnS{dwQ1K_d+kIKblEGe4Lvr5kv=tL^y+OUu+{sPO3R{P3jf$h&{<^j`b)jGH8?>Uq8E^YhU>-JrYIyO&?{s@*Pn_VvWv z_lWH>Csg%*|HL26QuVrV{m1f~v5S%Y)nhTK7CHdXxe$t5S$_BI-p!k_Hdo!P=s$)1P1V~&Y2{vtmfPxsbkVUvk#Wq3)V zblS@z#%UJ_atYsYPfC=^u`fQ`zwwmz6AA*M20sQ|*mJPFTPjiA=v9PmEZ&PE&>0@3;FcQ8sU0l3ENxQ{Cq1 zaEfgMXRxW=ufe|AwB>E3q;9&P>GJiT0rTrL*vz-B!w1XF7kw0(XM$6gn=kP)Y5&OIDE7?Z1{U%Lq8-P{eJ$w=le*56bfS^m>{D7G4J^t zpKF~q-0t7;hmi^bB z(Fds=Qkkvv%5_?0W%J_abw0*CPgTTnId)vCt+;eRAo-Q!Z&5YKkh}TZoH8W}05v+W zOtGW#lB6~O?oqyi9?b^8dOLr8=r%47pYa~_oEI?$IsL$Tr}-cxP&w1GBzJ?vY}MU$ z$R5%+58k|*;xp1>j>^k164M%*4c5I=9zVDtUoGpsJ=A-|HH!b;h_U6bCvO(kNk%~O zKSJ+BNkAp#ngj|!|5iTG_T>t8W8ibuJjk9_DH`j-L`~RI)&Qhf^rJIR=)dMf)~&d_ zL)7&^MU9yrH%cc!4Pi=#`;)ni0H|{NaYT0euiZ4%*@|{w=PG>C`%RjSQV2o+C?f7Z zijk5WfRBkdkR?C+iKhFf1U{;iY3C^#OyR{e8q7DC<^-9cU|l&C24Mc|=DFQd?hN>K zgcmmY3tf1qYs5N?aZ*-cK!}=*5Po}{A2m{^kf}g4hZ?6~^jy9>=a!faX*D5eD~#)t z9kDQr!dBl_OU(peM4!x1iL+V1VQ5^;#~G_Yl}&MvG}HulZ98i47{G1qky&R~I#*+6 znL&mkS&6325U6-N@mna@R=TU$r18f@;d-{M5RaG=8hQKlhkLeSD|PF=Ww~TdZp8Eu z-BNcdY4}vO-I%Y2S@DTViQ5rHi&jiQc{>n*Tm$rg(I~7HD^OVEh>e+ z%|dv0hK;nn7Z%TZDXDe9Ta(VL)(?kp1{x?xJ}QC|xsvgw_cH@>6NE@RQtsoJ65R`1D9r z!z9=<>Q{YQGS_qWC6^*;o!9ysc2+9RxZ4#^O~0`IMY&yf_Nu(?CERkUq6R)e#4H^W zb>>52_5czlfq>%^PyYQ`cnu~2SQ?)op3Fz}*f#6pBhl|GzU|eShl^vIN;Q=dL`G_%I0; zp3#SOXz(|a7NYoRPWE2u;JZrw!rEMJ(yKj8TryY4f|Ph9t3R2(heLdQR6XD0x2@XjFqUQ1P$@RZSmoeiMSdfK)E#B?4YQY+mAaXq55GJcvRMx zHX%-e#M8m62$Qu@6L8NF_z6Ui{bqtO+i9^rRZWnkn>dXepsBe8{+k7Q(nhbltK3q)wW^O=PidsGa<}juJEZ>PLjLukVzih0RC&hw<~g0FI1&Z)|Rb-^Mc{+%fX)QLmJsaQ>(zHl`DBa_-MGWpV zz)>qybLqyz!x(p#EdYDF?Wu`5NewSfkQ3u|BC9g2`h|q?Tx`g{kpV~E2UrX6}j6qE<=?HzvptZQjSK{nGcQe<;f`OP1QGgx5a`HeWu4b*OLa zL4k|82leWhV&+Rjx?tSLK!=A6Fzp#rzO0fknCsYQwa+0unRsPA9 zD93a(*@^F$xKbt=*QXPjxKqg5(YZBC6y-4;qU;Z21gEBh^Pwme{m5U>r}{F3DxX)= zMMxJ&cvpY&tEBgV{|>Y~-MoL|?m)XO`y#$w>*&cdqN&&&D%TxL`4A`r(uBE$3BZw9na5X5Zh0Q>~Yw;d}>D8(3J&k+H-Vk-RYAA6vacHvXWLbAVZ5O z#-F*h@x&v0r(;)-DLRvXZnXXFbV_>Zj1q+@Q{_A>dGyT}4^<{?N==d`0aNxr@1L(S zPxK=&Lq|Oa#jyb~9IG%BuN3YII2u~TMuH`1{hKr7F0xLzdG}Q^L}Mz1I!NO^=j|Cn zT$(tBXitjKnncBUKui<6SO@)aJfmFf@oQf)#o2vgIeGI`Du^X-U4h<*q7CcexG%1I z9mwmsOp6sc59WbN0UFvFnIu5X@EN%?M>FaBRJaIBD+I4hZ)q7{3_2F9>fQwvzAFg4sW4CzaOvS{f0 zQ5evWcUl-tKI9&WDsrGS4m4$pvEl*1Ot1%_rDIR;w*3Zl$OVt$Nb%};_+c%uEXJ;F zdbbL(i^7^cTM>C0q%q{{DYQ+dDo2EyqtcCFmLNwef)>)g!ceSju0|vEe$6pw4s0?A zwMP5(?%hIh(#6wZT`FAq^=#M1oTHN{U?7!D$n`!-2B5ATvIaW{-26-?ROKJL%>uI> zD}=laeH5LONJB8Z^NGB`4^A;&yoeJo5p_X`H4`Lq5kTK>LIa}0hIjWC4SdHE=}<>_ z%*}RkX@Lbb+P%Ou*RFjPE$AqA$UPZhV^Yb?OyZ(|4HB8Lt`^Bj3{ZtBWD{9*V0bIF zBp)N>LW7}w7;Rc<3Jp%o-F^lu)S`_c@W6Qt$leYe)yBAvovY*)l~Rv%zGCE#$@OgG#87~u85m)Xmh0Oj zIzE{mc`iA!uzg0G=u$j_JuucS`y>X(K7o-Dm?cJ{o>-x}LezzxL+1LYQrSs-czrIc z!im=AXuWX3ymG;~^C3FBVo?qi=9B;yoJ>-s3rMW>sxSPpi{0e~jIv)AI)55v2f|iy z6kJ%j1qSC-rCC)`X;G@HS^3!+6XSS3+ybL$iIMqP`EEZ(od(B@T}b9dc+l2nN5<3j zxJQYx_6-+}ar;b5Wsg}TRN zeBI(caLX^pCbPy_&WFCOyMYtByY@I1n#Dg(6B%GmAtvWm1D&&4W6irXml>Amiji=^ z!2HT2Civ-(Av1u!E4FO5F37c|Gpr@Na!dU(u!mn3Fmf{{yX_O%XP{#v&JBV~2dQ#MqFUD)~t0 zDAp*G+(tQ?7J}+WdX8q~A4_O=8Nlwr?6RxNzc`9s?_}6;D8k@y61y{nkKhD*O5-pm zXHf(8P8Iwmmz|ie;TR`8^ezgPnu+$nt>TutuAC20SABgvbwPjLWRFIcYy|ddJ!5w| zEhwAYoyOf>%|h*nN1X8JvL*KY_nLB62hkSF)m1<6ogzo_R5a?PCiXgMcE*w`WmqkW zImm1O3+H#La}?KWX{#;OlgD(`?wrg|i9(lbU|msy4gIhttLK1u3vr=Cmusrafd$dm zM~yZ`UW?i7MB{Blv>3*l)~X-@yfN}d^mn7cE7!}HEnXlumoDs}GGX3!va{f(F`x2D z@#Y=b)YKdVN$rv&pihsnjYj&2xmp^&8(Z5`Rz6#zv2hoiQuL}I3KgJKGK~)qws*0} ziAE=KkN>HKsm%~-?HRc`pq)+MB!Fn#+hLu%T#D+ zQ#|#)k4BA|{4QhVs}>}!G}m2A?`aymjsN=t4g4Wf}S5ch9$0ceAg=rAk>J5P=k=?%}NDy;#U+ zoDGo=|0rAcI0^>)N3?k(P1GYoAnz50I@0Tgc4ZIDWVa4M9thB(nVK4-RdU&bTJHX_ z*}3n1+`v|&4s@F_YR5(~Rj@4;k>_Zv7wG6yZ9K%M*t_IgM$h9zyV^_F+0zg*^-zfm zYys}nQIAl;O$BDI+S)YvwzjHVjUMN8`J1AVh$D?6%GCBz1M_*0R9wjQ*H&SPP?+BC z7xB_(6KXTBhPEOA%cE;V0(~k%;n9;KY{ur~T^J9;BN8+j2={_`i0l<@eULg4)HUqgCas0)Kx}yavlp;lEOaHMVs>~pF*S@|4x6wUW7KC~K4^S9B~Nh-(MS#Xeb!Hy z)h@$?ZVi{d+#7hQt0kux$#F(>{&Q<=h4N$lhOp`@bg~cVU=MVYJ zS4N$;VeU3ifmHybJ7Z_J0}$?*$4l=*xt9i~fvU>FXiT6L4Xx=F-~9181AfsNb!nOR zGSG89BTx}$|w5O)yi`R4nI9~L-Vk;V!p+zXzSM}d|tN`5Vg$L#_b(PtWllE zug2q*^aK}{7-cz|UT5SUw;g!hmIHV9yg9M%bep1jPqTKnhg$ABMNLxuXHvDXhx)yV zR~P>(kilgEe$ST+H#CM#US8O-OXSAFg&q5fj{VBw{(JD9zy}F@-fN$B)F}kOavDN^qfb5pHr+nR5y&aStYL1A5oBa>q&K&rF>jZb9PJ~ zb2T}`=uK=+8@S^Kobv5Ygh$BkP$uxZ1nhiIue-~~HwbOc#YcbP{MJ!>ldSUWi#+_5 z?`yQRRae=cEL`l|EV4GuAv)}gy}e2n8hSW92?{JFa5N9Ra!N!H`CqZ`PprOQ8Tz8p zbw@7oJ528!SHS?xE^R{keex7g#IfBAb_4@Zcivpx5ZcI>0;r(|N=9 zqt1yc_kKm4`JSw9I@$i6AKLpOANW~)ym}>0!TtS;!?LMopT>p{X)nE|WQ^P=)z-eIrtg!z?{#_4t9b|k#2jE$jXXVV zfmX6uvL~|9Qy++JqG{-76(v)FJN^H~Y+F#njx}amO%^IHauWk0rt;EMPxDn#yaq|5 zhzi3Bn}pQt9s*TIV;Jx@0l3if{?x-6PwdTP5}>SCkal||1QfPpX@Ebxl4t03(w8-m zgPND&c~raMEtT_zpluAq&*{9R zB>^+z%iZLPg@NU}MIm8(W}2r*VwduN%r?gZteC&=8~~?Is_iK^1Qol+iqD8O@W+IJ zq!;l&X8T1;1E3c)e$^y06^i$G?OLs>>3lzGiuz(cj0r<%*7h}MyX$}?aLROA-=_cR z!)soz!4JU>={)g^Cv>0P?K^2e0lBf;OOv=jTDQ76=%Noc6GEf&mm)|jE5BBj({SxB zTgEiBF?gErhTjbP2#iJpKT*u=4L=F*s5PJD_*-Br3}INdQuQVYyD_srG#AnbegKfB z9`^+mC&oXx_rK8AX6x&R z-HQ_Z8($XgvX)PdZ8e#PfC{_eMM=2MUc21=nO*(D!=D*UR>Z^o^^PW($bF~((b3c> z3Hzx1hTSqm1FMz)e(f$V`%GSZQ(e`?1U2K@Jx`^`fMdhXKNhzShl{l=tDXD701!Q+ z=|HKb38!g)P4sv42$A@8@se6DSN?L<@NBsRhCTj{vy#Gu+R}c^K5YS0neVR*Yl~jm zasQYc3X?sW_4gBhcSHnC8=QoJ%*@XZ-TSxE;GTUc?{?#H7^Z0Nm)vPC_G|9TkFnY; z(XbmpwU&m1HgyMD?tuImlh3E@O-M{pZ+dlf89O4pWfAqA| z+G%ce$YUhC^uV^)OIJ@DV0eJAqbOc!Ciqc%$7vU5kI2UaRf5Wst7RX8#9%F&*&JBj zXYlXk(GNkAGJs?D>`UpW_I&e-KIQ2#wQ~l!TP!Ow7br%A6>N^1^`Ui#4pf+zRlLdo z4=W@t?0b8$c^nmfTzR*-^0L>uo~9d++^=Q>V&pIFe7WwpcfH5QZ5#UY?)T<=dA!}K z>XN~h@z0KN7?6Hf%GDN~PJ{b$ZPu0iDe+)M+TslC*-UY(7240GC%QlOl|0kM2A=)* z%9c&-(f8%gGF60XB;E!82R zz0X%EM8ZD_-=7pVCU@BUW@tL@fyjNu9SMB}bc=(T1aN#`|0XCadBdD%2qQFSWm@2Q zJKAqufm(;$rPf4p&7%&w+0cA!ooe3fGYO%u$x#K(83PucTg>YQQv0(q{ME(bpOyU! z9y#=FU$zss^{>Ih`LmJP)k?3OM=WQ(&+QrDDt{jxv6h_27}!6OClpXmT*{I5NVmy4 zIyCBNxs=1oCY%qU-`XdB-&>}4hFn5R+#kvo+Jv_yIJ%7clrH5rxYz1p-T58G+}63Dug>hG+{%1+5KW~w>eiR=N=XcPdG9&c>A?prH~Ny#g&z<)oQD< zEqi4}pG*0FXNAS(mHg&%sSeXO<&~R4EfjZ1m9Uxcd;SH%p*mA(mfJQ%%>+YZh5Mu# z6jkDCeimzp%Z>!7Z~NA%{2Wum!Yhp_1OC$|nGn$7Wm=D5gkwRw7)kBv*Or4$%gJb? z>~$#&10(C9eKeH>;>DYT3>9|WxvycaWmQ?`{zUmqZsq-oaYrf9ulY|xuHLIKJQuTp zLffqT2;f$}G`3YTn^R41$vL#`ccC-0IdYrScMCT$E{WnF8nqPmrPsKegq`-PZ@m`` z;NY_dcatLZ0{6G6magr6`+T>I=|>0~?idj7=}uR@&F7@+4^Vod{E6lvs|0ge|0csW zL^_<`CS{`+V%Urli`l}Kj}x$TAkVnX5Gwns7rvE(%J`?OHv-(X&B_?KckI__^7eL3 ztNBuc3M1xR`*o!1`~SXl(29X2aR6DL&uvVJ(ZK=1PI((S43;N?lLT}EhOd&w9TJ!1 zZnb4>@0}C6g|`RHrOvdL)+ZqkSpn5gT+ay&{J{z>oy01o1Gag%lbaDE8CDA>;JdgK z96iy24hXaoXW?TsDU9GKB<`7vV!V-ja~qKYm&;O)cu=Va4`wmAsR_ZD?{zOg_^Ii2 z5(nBfUNF&Z!}YIh!S+8(#|fLc$r!8mD4}ZxyRgb_!NMR9rWLQR-DUv%DC@YAiG4pw z(K!&~r6OREiW-(EoR&a?AR&0O)mFtME}}R@64E5C5o~4x&V&YGilrb7_3XNHdlXJq zNWs!PCXT^CRPaH#5ataMXlIz+YRgtx$E2bj?BISCkV z38#5}S=q%;47|@2HskcK-bDU>3hZGDMHU*Dt3~cByWtFk!-23Z=Z(^o!ftnt4d$U` zhxE{e=Mo#4u>aXN+v}M^e_p-hMzE5U@MXfnPF?k{cR|Z{daEVtW5r@4KU=>1sUxig zguZ1-F#Puroc$AoKN3JQo!OIx7r|wd*5QAXx>buIyb%>hsS$6FXtbMObx)VWuvJQ# z-20+}^BgB-pkmf9Qpi9t#XmIPz4Bxn`VZ2RMgZrykX0lsH#$4nX`NYn9*=tT@==ur zCaiRhiHT(jt-<}Xgs&*&1yhLJ0Qf>oX20BxNtc=52Bq8@vn|$3Gl|}B?YMB(vz|>` z9&G@*t0xnhFgsAGc@fC39W+RxDxUtfvDmi6GeijR94x`QImxwtc{|V=Sx=%JUKHf3 zYMuD|6-$tHOBT|YzqP5?Ee}=svOCo868K6^f)tK+HeOKU^&6Ziv-AgZM>HskR!ob zu^=-NyMebSnGHW-BDz#DC1=b=Hl!IA?@8OWftX~B0iRGv_YGjjxfx z8Ngtw;}L7XoXbJ2sZ5C(i9jh>vVr-uAGKh?M2;Vfhsb=`italXwTfEtVKN>v2mwSE zY{Btu_D;}?N-(WF6l@0c643b=xj#Z{)j(o08wJcAQvIj$_re;Sn0?2wcI<=606N`; znIcP9eNFH5m4F3M7CrfGp`^1p(S?S1G^7w{un8Uwe535@j68Zy5y!(Ow3LSnup-9~ z!};%!cWmbH$7|5Gy+Dx<+c+r17E7*iT+Jvf1!U??Vmzt4DaQ_J?M)xM7B7{ME**n) zU`7OvsR(I!HjZ zL0Uo@q{l~EwJ2VEM2`vm*_x`&%1&)t&6b+X_9P`*(g>}4k0Kn&d)oumsHbP&ph6^u zCkX*xVLObN7z-2&pc{KpJ09ZLb<_Zxm&^hWoG?|fGPPPV@~@#k%jZW&b#3@CUOrZZ zf%LzTb!R(2V*r)Ntmf!U6|Q&Qm;CVH94;s57c(8N=3c! zq_v8CvmG*%3%@QQ?EYEnpitM#awL$djHFPXBn#yn2Yr)!A$wF@%%I)V+zopRbpj zzam9Wv~`nn<>l<~W*nd2sqBa$6_#;ny$)HcjQmkXAfcm?ZuhUwxOZdpViyroc~Xjd-Yd=P z?K44y=AU3cy~Dm0#c9;0Vjg3g^TIxD>Z8;9bcA~;o9HB z<@XdrP|RbQ-~62cw*p(;Fp9sAx!wxhO}^d$HYxO5^d~4B`h4lOVWWzmuOd>3K0iRS zTYD)wbgq(*>tHz(QvHGz=We6(lFPY~>hI{rde)0sgli;Gi7O38WnIgghAJ3CZe^Et zWp!18nhM5nIdj;|eZM+abvbvnwE|D7B*#^-h}9MmMlw zYwwZcbMOgwj3X<;mI=Syi&qf9j$m|5bJXvZ3W6i_HAf_A^%mg=w)-8*iaezHySx|Z zD1Jr*2g&;t6%q#8B5rdm@HfkO?ZR%xjb&Tc3L8zliEriL!FDR=5ap?&4*G+pkH~Bk zlF2ZtzgX%?9^d2dXFCl(;D zODk4uz^>=4BDx8@)$56+to~9lv9N3I`R{jn$%W0O%V@;ox(5jr2|yccspNNF&q2jW zMBk`wD*N{PR1dDWf$RMx9oe9Zqj7Is)xh+DJN?0yY(NAsU<+N$}1O?>_xTGS27ZY61(M~jc zQci#JLU(7^=9@8_9q;VE>uWb%VkG-#!N)Dh@Af5|>k&7V1|F5OjV}U~cLPfQJ^**h zL~}Owrz=b4cwn+m#TT23=|=Gj`x4(eV-zAYO8PMCD)`mA$BH78w&YBb{Qb$|`2?M{ zmb%|{mRb7GIm+!#V5*%fVN> zC;oBoQ|FCucVKBpqRsLOmpq4`p4{n%Xqr&B`9|rDkhi|tI%L3$w%}2&6cSw6p6zTu z+>ofc;2NE^xS;6J8#uA7_`aO_e)RJD+c$_Z{H~H*r9*!|4E(Fdj$AZWg2r!Xkm+fA zmarO->*)0{ea*o26gzc*O%Nq_>r7DhmuHst!TqI0c&ac}dD0YB0JATKko6r9E z=9SP}$1$FU0y=eos@Q10%u6Q7xPiSI7Gvpg_aqA(0GOo zNTGOdX(O|LxI`-LNx1Xlk4>f5@|O6W{5B#={h&cnF)tj-{JHv~hvV^2pePjmw3Rav zp@fuw`oca#xhx@ciSRKfj56o9qcRlguM{7-|)PG<(w6pYvHm%wD-Wm zt^;qSDzQ`!N~Hr;3xQQFibTr4Y5n{DrexY>e@_||+8A+8Uv60fLlr_?J-3K$^32R6 z0mm#)X$}2+EqvB`Ro2<_S!&oUku9v6+DthW`u~cyEiS?Y4^HQ`W*6A7$qTkB%(VU>mx-FziG(1By+0K)qvN$nX~33CMWa2%}zJeJY9_P4}nh4`=8^Otbb!vT(B= zuE`zc(!;wB#w3EvNu8zkR2Eukw@v-@=J?fcz|2y!Ff04NUTkxUm+F>8;q525oN=c3 zwr9>QIy*kpTn`>T*A@pei<+b~FOo?2buchfO8FI@WZd{5NyVVJbTJgWrV}3sWdpm( zG(^ok9JG(>_+8p{Tp9@aN5vrEu9XPDfZCNE*N$C+*v%AmU17jy+T{p?bg3eYDgPgS ze~O`Zc<{i6ZCe{EEJN>enG$!d#cN^as}j8=GNdlj3u+*M>Z%QyzqVyEg$*#QwX)rm zIZd~{UYCRH%F{5RkalnTb?oC)db~*kmtvo}&C6CRodDa}6%#zW%==$#v*A^Wn`oU^ zDKzDO*r9C0^oLfxwJrsXREXDF9DZuQ_uB5`&)T9Z5)_9xka!cirqLRdtC5cX+jxKH z0m0Y%c?rZJ)__|xo{%W=Zs_$m7H}9L@!q@5wvo5S_~5b)Y$??g6WezCvjzq^&-XeD z(s$_S-dI|bu9N8@IquTwbScT6feO3VS96o{dE-x)Su-MqL=Ehhy*sd?A=G@YG*J5iMaNw}8;L+mc>w!P* z#a?vrbXFzC%J5;WqZ;q%Qkobxs2uN!*19z1ckn}1XW^_p4#urG&17mGGiG~{~ zbVXqId5e#|y&41pEH-d(zWLFs=1({3@fnL!2EJFJtKVd^btEJd_yZTO&$5B}guLG8 z5({6zgh>ik`Ztx1iyiE|MyL8Co6O4Ij|m6@vxLZR~T)~dHe1utjv7j<-Se7-hX}cC-bB|PC}H! ze&8o}JDC<~%7RUd&;Ox9Dl?pz*#)!UB$17}f@9OWlk~erXu1>jVnl8V4P(YWdwjLk zeFhJof{(sg8FI-_^+z-XT*33}*8PLn%L&gDdyPc?eJ&m?{_!Wp@hW?)`tR&JI~;)& z#uYYL!T!9w(8aNF0QRiJ$(ti%2v#D;Vq`4U=;s&tW1ac0)(xJO$)%CPu@CF5zxNz^b4Cjg>HwFXzxlZ4gNX}`)+5~PNhB2J+Cc10wrGQ?lA zP8UWgv_z>z@3<$u{kWd?WZ4#maHjLZI(ItaFmzbRHgaH{A1Be17%b$8hjG8#ktf&r z!bP`VXl!asSD>})8B=XE$&JoythXSO3*qRbYOmDj{7kJ{;MtAYy%bHLn*@Ex9YIxp z8?tPbb|6$dEX!kx*sM@~=HgKm9@~-_PWyGZfoXY*D^hfD&&kD+0db5^>6DVNL^{jU z*DwKd+TG^(PziBg1UFgBw_abP7)Th{0I~E?NQ=H|1Hx$p<2P9pI$#dW(WtsBKxl2U zYJz?UnSpjt7UeP>!|+)|m{KMvyk%a=&NL6VZT4Z>v$;ufLNv#Xvys8l?_r z-qwnaEHOF+6QZdq686m9H{SZ9R6ZnG!xb_lt&@nSC9A1ddW;yRl?}y}nXRA4*b5+u zXj&4nizK9;E?yh222gu0vM>e_C|Y#7en+EB>}zpxHWPD*glaNu{31Bhq*$32y3vT| zZSRK~UtKPlh3krX;uFQ~Y0zEgvsZMO+vVOAw4U1LfYFL93DpUKjGw;2dwNgOU8k@c z8!kz?1E75U7*wBLg+F}Z$>{2D5+Rep4WYdCd1MK7;)Q4|CSZEM@4R+X{3C^8`E*fd zpnGd24dZiA)bQ8?VUVQaT@;XhyX3C?~J$I@mR{bGpOcV3alHW@6pf4!tWLlaV9 zGI`_QxgxSm2v02+_n#ITn%r>4<_OI98SdttO-?CiA|RdHd0P9Uy>c+I=vX9SP`vFh zd2)W#+y=RGHYC-qFWd(aimZxjDND@TEau)i{9_|$sr10BmQW%T+Td2n)JvHfa=fh z-|WyGc*K=8t8HvYzr&mrt1SMk#3J=*yk(bS|h5$1~JRk_Pw zjW;KN34NhJ-hCM*JpgL}n|IO>@+6$5uSQG^m|2kR8m)Z{ldeh48ZfWPkv?uXyRh!C z8IY!PYxg|Ll8WOLiV}k%nAtz~WT~(#d@1!s*zYpcEw^P*l7Wqm19p5erry8j%k(SD z#qH7v2wigU?anzT*^#V#=Z0f>7t<}uMCOTTR7o}WyCzn?v|B7OhZ*b~ALQ~cMP}P~ui^ww>6;t#7BOd?9+Lnq$!0A7 zMe$j$M%?@K(_StGe64fau~Hva3Ocz3Pm2peTn;^HIS#ZDHvAsI2O#^RIR zxj}F9i(HEDn&;X4K!fW1+;eseK_wO4*;X$3T9lv$?GpR_`8L=ypJTF4mXvmNt=~3K zV&h`b z#}+)54+_}9vLOXMPRr|luK4;y!JXwm#3}FN^s3jLALmU3fv(*-2KXS!nLHv0dSsJt zm7`FsRTPU2^2uNc|IC+bC!Z{-faCJ#ONgAY+?ORp5!c+WjTK&RFQiu%^>tgY8u#Cb zx(I74Wqhu%FDd_3QqmZF!G4EQ22?heL42N(cOPH9UAszcPl62>u9yZig&JGS8vB46 z$G95j(i++UMc3IH&!08)0H;li>9XIK;9pG@vdlt}eW^ z?m$nS1_@T>)1;mQ!4Ut(c%&Y@^VXU;FIN@UGBvQ%V{-r85Ml-xDpZD$ z?8WOn&A3~Pny-u>+p1^lecfZ=NgaQjpMP3`f7Y~rd*DjZ@e=6S}ABI@a_P&i%$}!!6Kj>Fof|R_3 zFH9@Hm)VA+1uW$|-5&AOu->>Znrk)z_AXQs*`>f+X(oRQwJ&z?jc|Qp{^~F8W z9wdi4Z+k8=UvwpOLMy(l!ubel2ApqX03r?c7H(%7bJo1QWfoDGRQt#!&?MB(Tzc=^ z&rO@Z*zGg1zj4G!2H^Jgc4hm(FsHpQ8Z`m-noW)toNK322!2x)|22h}-S;1$K%It$ z(q;;2qEj>p`zs|B@rT1*oGdL7e$}^JYf)4V7IZ{mBkX|_f!T`=Z8_3wb*CEKlj^!1 zuz5IZ^Ny1m7;bBuVqei@aId1l=F$pbK4eJm^K(KC`OuIQq{ph4%(IY1Kv~}AkD8V> zNko~*Rm*W7B?ryEFK#koJm|!gjT`rXY@~Zy)?$Tf6f@XUO4$fo)vj;Y7=Og@%%Coj z8}PFo(c-|NhG%q1aHp)h8rRh}1g(>G)e-c2B*O}(_{EwuaKu=VRJ?ECT=D_^ct}n8+W?CPQ*FK1tf$tatMnp1WeaWh z?iS3v@GTY%B59sf;M)`{Z5>{u;F%kQSOX8+LBIi>U2(=|T1I9^M*g@M{cVBx_JCyZ5PLADFQEd2OZa>Le2Xid(I)3|dx^@(?JsH-KQ_ajPj%dpN%M$3K#g|HnMNC z0HmtNLwDn*#13z)`;SVObC9pcp`oMU8)z=Ghj6&vch)3Fne1pIN$)CmeG(O_wMGAA zH1>`n-AT~rLzJO`G{=_Y+Y5T@M&rH}ZV&$91UZeYsPaHq8O>Qwe#$pL`HJM?{@K84CI8bBMo zp_JR1HVi`*?%m6H9hZSeX$?9LA|JGHwOz#Sj=l5|;SI~-N8AL^#rkO$8`<)KUdf%o zcuc3_&s^BNHTu|@@ea%%tH%YGqD=7)@+`V2_aXkzSfu`=3k!Og*;!TIF}k@L2Ym_? z+aJ6Mp81i8s$dXPX7q(aXbK4x<=?Y*94@>%n-e@&nLk&PM0v$!$i!-+37xvS*P0Usoh=b&#OER z1MXbx!%@QUo#}yj2T3$6Y2xAvs>l{A?bpBZwD7ghGu;~-;y=4O0I-4mypBC&ED5g2 zyxQ|OWRvH=4?Um@N-yF+kJ+CN5C7|sl7d~5X{w|NmtUV5X=eG;|Q z(xRd|{CQC#_!%J{beZhJbI_!Xz&a6btw*~n+zJ!mxU_dYp5Xf7lg`}7+djy-Qh|)s zu8Z*iW?>u|2B8Va;N|1-lef$T(BL<0rJzS~-J3UL>8mF-tVt?;N*1N(q;oyrUgo!y z+@O*dU_r0cT(BM;m^wU9-J3ts%o;rYL41eMOeOCMMrv}=q~t?l=~;f+-;68{fZXC4k!{P+EH&K!(kn6Ym$gD|r1Tg=$WG9e|Z zu@p(NZ*61jOBy0OV@;G$WH;87(vU()Ly}5GdsLqJ-S_i6*Yn)htv8-o^d-=g;;jlv`l07B=BsIRsd;~a;C7&G;Q5AFw~fVV5D-yp^}xrG?RJFA(% zz5G_vts{zWEaz^qp_={Jwj4e_l%eEhi9K8Bj3rHWNkHRWaMK@HgQUf7myt{loa1954$ z2t1?cpa8iii3Pp9NHWaN9@rUYA!T8)M6jcT6;{!i(1Q58FRbAWV<~P*(%89 zVkA)3u;GeD+czCnN6|zTs3PgAT23R|^vf9Q2xL$J_7{GMoEKmz+v>1B-!ASsM`2PU zpJoLii67c;aO?5MJD081 zXj7s8q2T0daZuYe0dI|OClut(N4SZ*3;QCIZ8Z1t+T@=BxvAhjOMI=ooeH&1A)$`@ za85@ueKrFC=}xA_kFM9$=E{ndgOK%!8c@UO^!H zA48}O8{w|__2W#CU8g}n?e`#QX#BAUStm~Ye2kYh_FS6dG3U6Jb-eg6bO#3+Wf7nM z{Bz!NCIikHXRC_4$J%_hb~w22X5%wzk`Jlp>uTApS>3}EGd?Zg?%lpjB^mdqPF;BB z>d(cs{?ZI{oUiOXvFF7hXK#>GkCD4F>Rr*|QIPU{b zSi)v4iTwfAO3pZ9M-Gm9ghf4#=VXj1Z~5hr%ya`2kds5FMILYkwUHpVOa#8~SDUyQ zOA5r@78H`jL&{-;yk^l*%r24(DsD>qN|%#d@afFl&f^-fU6;`#Y=S`oG}oLy+R9Jj zU}d0zSPL}-2}1t-6N2XS$6q+0pluLoyuNl=h7l9E?pRI;ba#5V7vMW~x*?`I@2 z4JDUuo_H$lNcJ}#LJ6Gj*tB%YdhF_TA9mGWLK+^i7GkIGt^26XE z&aCIW|D^5=N&RTah_ehbk(9fB;W!c~N;DLsB!69J0H(sw!PctzTQLaepgvE~M7PC8 zx-z6@b0hd*)|`K}lDYAk0McvUi|08fe7Lm;-ZG_Vg&Ry>OJZUm=et*g0r2FGreYP( zUczeu(nfQj&`~l?F5orK9;0T1n2k}5PecHU8$`G4UwEyW*gI{L7+p?*B^X4Wd3Gq@ zNahh?HU;_N>3B&-&r|wXpk=+=sP@+H92giXQ zKk-nt5MVl6>Ojm;!I?6D1(9EkKq?4qWIx_5LgC@8RZkQ$-U_X=drc2TmcDX2zQ>}F z1!#TC-|cf)Vv`6$4fPp@&VEWVF5phWm1|r?{{D!q8Pi9u_1X_bH_r`o9=cn?bMfev z9gQWagcHreILyi^Eei4x7m>R;F&U(YaYLQ$!nmfyQz5^l3}QGEk+YX}v#&r9GRCxk z0bA@Tt$21mhH)z#0jO;Aoc(vEGLR7hJO+S&FB_x#4u6EsNdrsV-#W{QpU<3Qq;gqp zmi@_iPaC*zi7mO=Q9_rymi(NJk#pAC8G+ny*X?@HtO{`7NCbTl&|`E*vnK15f+Dd? zzh4_jZK2YS)c~t8w5#{BE}e`tc$1jZB@9rYkX|Yvk2K*Dz#)nSsjK}l#sR<0e4;NN zK+YZ_MQs25%E+Ru*;K|KH0(j6=q(gu3K`)?*-S5eDFh7dcQ{pwSdw7xz}2Qb3)0wR zVg7W3^f!kDdlPAwj=e_6#?kp!jPp zuRCe3JL1$Yu}6Cgso{Z#I4oPvrlSElX8P_)X9|LX03@__?8;)`$)ApT^z(A|ehv)l z`0(@B+Yc+UPA&Y5O)La9lrGE?BW?vItv`*!uw2B+@p~oWyi$DLdt7{!0$E{vU)?>Q zAa>Xhuyr`(S~wf}j@!rQUW(T!7_dzJlj`c8jy|v!*Y4>m+KE=8o3@Wi<}4|w5yR9E z`B^g|eoLAjJF#OF)+WneTn6_XN_%LBfGlyhM}kwLmHi6P9M;4p3_TZv{%jCptPHuY zCTZbpRc1rhMS%@2uk2MArXSFVjWW9(VHI|Yiv?NWU~D}MXl1$pk(_vkE*-oZlli(1 z;!cGfH_{J2MQF#M57wU<`Q;6B14ayZEqMo8;0BUsk9VYJb+5s_zY?a$&%7i8uUdV_ z?^&j=K~eVU%(oCV2*4B1$&Kq{H&i6TL`DO1yf$HTCM3byubu~q2YE}dBR07rO+Q1u*wFI(n6>*;^@l2!BPKyMLNzULaDO0`pPxdc@)jcEg&* zfcEcPd*Bl*0oRPu(=C(327!|woyR2 z?;G9@*^l$|VY|+BW+aXE|i>SOs z3kIndiqVqy(52UXo>Sn0E;O|(4x&hL#lop7$}nJMOuri0C19No{Z^5nt(Dy3#rS7{ z2umCm18Fc39r2}h3{Ct-PJ%CRgHR~-jF)^+Oay>3=Hrg%g6i>T>lj(@g5u2;pdlc} znT-e`C<6>YmI)ct@pm&k?hit;6qV3-kU6#B?mX<&D&O;4SrkV$z`zb#mv{ewIX(&l zx~gbcB{v!5@ZTg{rIHxPLyxEIhmKqY#IK|22cWh60A>U6($h!uu-N#4^388MAZLIS z_!H)w?QMz`FiVXOm2SMknRN~Ht=t5(3(KtA2cfQk5X zpAj+zgQ|89M9@WI`;qj@)ps?Nle_q|PQE4j%V17dUgxY(qkQqO91zL10X{k@Ys&QJ zZ#la8EnSf3ssktOi-APxe##jDwgt3rtaThcd`5q_0mDfe112Mc%8_PS8B6O`Edg6Gt(Iq% zY>BQ|CT%aDf@$d;rwOrrFYb`6;GtIFz5X4~Q?efqB)h}yY2WYoY_~sVw^L3j3>MbF zVL(u6M{rL^$cv89?T#?%yAh^$qrC6NltvXA*Lru;aNYG2cN$8& zZ@lPk-0o^Fz1<|;bH}@fV%*cQUD8?F)BBHd`uy|2Cdo1**IdiwjD`q#Y= zecJB-@}YavbilxE;792IBWqyGRO-)*0pL&jjxOaZe(^dDv( zW>CmyaGI}Q?A(Av*`SPCkMzRezwMntoY_$Ko*_jarqa0~)jx?MW<%<|Lt1711exK< zjUl~r!(9HuGB<{e7KSgH4x9HP&16QbvxjU#M(oN)$_z$CAC5Tw8Sz4lIIfS7eMZxW zqXOBZp8TVSvzbSFM;-o*!fIm_r;z}DW}wg5-mtINbT=)9FKZ9e(8qzWv5{yo%?4H_e*+_ zg$v`w{G-m=_mOw**Onox7RGBrM)BJBzZ`zhXm&$>cODwv(C=3q9p@jLydln7AD&?EJ1_l+e(q61?jz6M zW@7E5X8y?%ncaqplj9+i55A5S1At7{WXZzhXPFUK7R+(vSQ9zH6WE-MFk>PfXglmI zJl-yQZ14hp$p2W-=kd=QkGB^d@5r!>wOHmYk4Z$Br=0cD`<5})z9pdnMdke0Oeh|i zR(1e1dcexVb^;SksPX~8u|b1=YkNyuxu;pF{7FZ!{8K4#UiG%agG#B>-Fm?_-L-)6 z`+X|sjiqt77fq%_&%=?UurrmFHKmcY|LN%h`1tF`KQlXjcY~n{(;bcgiruU!Rb%yc zI%NWe-b0FE4H@tE?ftv+gn+TX_5dNc=jrJKPqV5VyiPx!OL~0tG=ky`>HqaGeRR$F zBed;xpORILFj4*4)$oW`tN-GGDOX#{bEovoC;A5}45)P><=qYp%{`X2rO#*7fLn$W zCkGHW)iAR~j&;bbQu`NotSN}LsLBKHOW$6fS9%8D55E@gIBRP1ylz)#>&ZgLkJSTt zFCvw98HCk1>@^~=O1gL9J{ecM`Q1SFzP3L9(n1)K*}L$~9`2d}K4bz~de4XcCXO6= z*;n1bS)Z(yS8((cpQ{=~k0X>!I_#J)juQyVEs|@_m$&c>6@&H^_NSYkF$O(?!iJ2!^$EcozOSdYzZ`RjV}aYL<0|)qt%GPsGrQ&ZY9N`LJkalQQ|wK4Cj>z zp3d3fl`nrF&R5 zwK!N7mZgMswILMKt=WNSG1yC{d;JOy2-lxMZI#bQRPR}J?#<>2d zqyIW)Wt4hZxv-LdaDAUkl z1)%837I|ZZmB5Pw9&CcpC_x? zh|}MnliAzM&(bf>U?$JVyi)k*7SV0a#$v;MgbbSHQPsI_zp&P>NDAATua?}z@ZAwa z0>s0H8m^MnwIU5ikD8wO?Cq8l8piLLclDWW3)}R1FY*fpzY58Rod1{OLR1$ zu_<^onUtldTv}dPWo?jBgd&?WQV39HSp~VMY7Xs0e^@*=J3lx)O1?VBCSOl+2AsPa zCGu8u&S4GhGLTYh7Ja_40w-F6JYs555P;tA8d5u66;zJP;O4Wz zJo_w{uM9~sBdZt+=KCRy{{OACm6Qz26-My^6BU?4^;Z-f3&@95rw^vf@d7Wi(I(w( zyQ@SvhzZ@R4fC|tGb=_^w8rvrEw|&S{n-vWD3%qWlKy_m3c(K{TfSmx_5z?{EpH?e zOFhz?z$*z5t7%l^dJcuPl1NmvOWXO{*0VC`v(T056Xd*tpp;J=}QAxzz)fqm~!=LY)ToA4PsQ(vpQMDGCUa?H|{)owzu2} zm62K->z|z5so>{KFtU1Jbeg1;8N`gMMT@Y|0YHW#g-S&`vk)m4Tr$522y!3EYUW_#9_btO9Az%ajVzg?o-q&7Usjx1Tu@cRibjrqB5%$^VNkS zd?FlhQmgW=5|rAE3$`Y2YX>E_l+IJw6X%elRkVi z?hTvwMzTlLNci>hZ6M!It)UNB--|>zre${?MnXw?;4Zq}0m^j6+=(W2LQb5cFxq(< zV73pY{|veFM58SjFRQneox-xjtE*Pi$stXb#!NNoq@ou$dXhabAucR(Pr8EjlC{-P zb@I=}E7kf@XYdjdlvXMrL3!4S^%tVyf#i)VcV!Ud3O|JO@R_$=?+#uhwDq zM4B0o*yiCaES$nM{RI}RA9$CXRpYMl3_B?#7Cv;0p47Xy5$}w|Os5m?(W6UZXRVWg z=w~;6EsGoY0$sY;4S79oNp?Lu%7z>_H|CqS!Rzd&;kNYhqf-n!$;!b*(co{`-NON@vYphk<{DQbk^A8rHTPNkJrLvRw zb|f@4zmPX5R(5JIU+>0rOnx%3%14!Uw#KxWy#6*@`2-7x?R7`?v~EOUgiU$2k%>YJ zD?2%{efu8qW-H3Y+_-f<$sXdBe^UFED_@n5m^$FtFkz0m5x_}>le6}bk%Q>SPAka+I4(6N(W8i*xW=+cH=4oY z*YQB^CH2&eD&P2X4C-Othe`$*=AU)DF`Z?^MXb>kjp-WdqAwGtTqGynC!r?RfdMqv zMLAUX$_42q0uN`^Z*ismXA;{d}GRnmyHwySgbHUeq}9U%!{OY4{qqz4^b4k{)Z)nBo( zBF;RD=+>2B8Ix3AIXw0f0!WepL|WtQQR)<09SAOW3MenhMQW0<*8x>4A>U& zwO7Ji7|mbUYs9bFYY!`c=J1y;HMqKBbkH4;mU`!ec~}ADVSh!T!7f5>FCjIZ2tgj3 z#X65pKKfH7Dn|6UCgBM21to!=@jBdUU(haz-6aaORDp1<4f?IvCEYZS5p`maj_cS$SC$F9+RS}pX2iJ(S`AbAN<`|hhCT* zv~t}TJd@}d5!Unm;u4waz)OO&GSH#>uMMZZqx*lKpT#?R( z1hvwEkbyi2VkjwB?6=cA&lNXsSbtc;ioD_NCi#^yc(R@s} z<1L6f#f4C2kT5qU9^+VS*wbx;dVYf)vGsF=Fqx^hV;Dx0*kBZ%2pPi0VUmPHfI@X^ zApK;^D}L#&t9Gn$ZjlyQGgSbg7?y~(b=s!{4{~cX2wk}?Pv!cF?bCS~&VOfN<%R@X z1J9^xPN&2Fp0Dsa;Anyo%zGFL16+CY<%$EvUrX-y&FhlDTeHWwe+9|{`WUi}Y)PLFHumC@s~Z%ITNeaq3Y1vFQe5;1H#mj-(mTI@uU z_dJvP_G&rbAv#T%V^ZK;S><&;TEd)9Y()y`+GX|$qUnor2c)Hd+Q&o`Tk`1%Pa@(* zJ1B@z;L=Es=0D+l%KNp+tTrePYhEI-v6kRWkG|_YhtT{rD4es=i(xE1 z&jFeSK0;D@TT_oFOroQW3k(>q!JzHMrqe|_Nc*0TxP_l&f@4}p(S^Wuv??mX^O z3EPQX{P6E5dE@V5ir~5D!aJYtX(Sa3{rUe@d63lKP07C-R(6k?7H)v!c_M+CNjS z9ZdBKPh}N~{z*%%daDX-q*m}KO#x{$foZm)33*pxyZ`6fdvVD;#V9?|9=4;y^whLV zvuPJ4(vuFQrxhVG)(}~9>6hnXI5Vt22Enpnfv{bOWL)O8xQND7aM2)KG%$(&E3i@` zbKN$Ro08d3loeZ>89SA!O$@gNvK|+})h7aHJF>>cGL?Z-d^LMW%o9!&$A)k48`7nQ z8F)$}kXD4~>^Kz;8zL#0fvKSUlataCS;02hNZ;&DTj2hj+6$C*{7!j7(qa zbmb|g^Tpetz3c_&Pn^}s6@3Sa6V8Y-a%CJ`=MEh69+NE@hTN61=lzZwFd{OhXn3?ONV{;+K-;FwU-x5X`g|WPDrmGKPqmJzoJ(l z>Rx~sgAWl3ye0GA98e1Mvf#)Zx(l?2hx|O+eUY|D+m0hNd9O-+xf(uyly~at=YYc3 zqTYB$;rt97=5Q3-qDtd+yZ2q@L`>1;`-P2FMSre{Uiw{dS+v+F2*`J^wPQj?4Z0go z5BDD^8T8*Np3^C*yYFLrwWOr0ME|u%s3;_p7|!uF7GL+ck%8|h*6x(dEo?9CD~2#} z#cHjGuDuIMAIpn~6}w^;B*hNs2PvMJ;8O$^)&xjWmr$Fzfw9Ln#o%h$C@OJ44g zhbjzm5o!_A-h~L~8DKV`q8+FNaFvJbDmTI_jeHSzu+V+pD2+N%|Mbe6I0oLGVG+wX z=qIa|=UVYg1kJ8M1)K1+c z!@A$bc)GF%5L{74O}=oi?skj$yKh30;irNgs7fwDp>2C5GUc>etX!!+wQO%r2g^K9O$Yrpy9Z`Yc~%5O0m+QI_C%+s^-;;q`!37LLP8@jzSw(cuArt-i_3XLjq z8l`2x8Qq%EvD_@@m3KyctKb#SC~G>OMQd$cn7F-(Oly*Gj+R!3i2~sXtPDtGF@=gx zddvr-Vy#W?I|6#kDYy5ZX1F0=WH#~nUiBT7nnCe#szIXP9+BTPhBkJbRNsne4kODL zf)q<3PkEFvMZlbLFc(l-$H?v{<*c_E=3o@>VMU{AFhkM2h9FuXGxfJ4egvz=wGYX| zhrLEob)ZVC+2?u6X+Px4{1k^y? zc9L5=Zn!OvVR4!dKt%2fR_yo;GS;ZPLkjI4+k)rL;!m2!MH%rO59d|Bhjb;0mu;Av zlg+oNHK~{!&ky|l-_7emB{eafR4r=qhA>H6M45yb(cU@Lm@jmo3Q58lNwXsq^E;hY z&P93R2mQJpN9bphPzt`tyIFUqG2T9d`>%U@VM&;R6~R#OZZalk>=d@ZhA5R=dYg|= zy-R9Q+h*JN=ywyDPhV8N`656r^E*pI&b3HSjxyl|E(;_3&qnFxl)Rju^k*xWb6ndz zlOD(=u~_2xZ4|1wYQ9w1i7D5)-D@eV#pi_Vn?UmX(cw8KG_(&nnV58)Hqg*=tSDT( zM0?lcX{NJ??{DvbJd)(qTw%l?R83s*K56U2nWCdi`AZyOA5*P}BIdfO*cs3+wKi<~ z-G-89G*TrfCnTRr#x6A~9FeZmAIy||K4d(&Bg6YpH?~v0_mryGLtKj0sAvdE&rR8} zR}N)K6r7edT@YE;9!sO(4u%Mm|Dd#q=+k&r=OqojFEGnjM*oYEM|JQQ%P`KI|A+jp z8^MEuK1j1LLl)i0*#k+HSBV-NOKT%UyWw4W_i5)=MXKFwWY)tTgSUNwB%kpt`gj(T zbEZ5HajH}HCA7LpxK=9_2`SQ?#v$CUkaNnX6NMTd;}sO^8oc}!}|JHVW%*? z6uMr~4b-0}*eCp45L?oj4ZgwK7VZ-?LEO@OIOcXmV3j;&Gzi1w`k7)D-(Hkqg7%dI zDp_Gu*KUkvQKfXxYp?r0$-*EChsJh4ao7|2gqwv8G1ECcIIYo#a%BJ(hlF30$%{c; zvUITuE3C6+n1(N62HPd_TKT2lm=o4Xk=-jwlg<;?yUHA^wlbMHJQnjr;7B81jCzd&5s+Ztufthp;YM#* ziSStyi6}0bFlgR#kRZGRiym*EEVUA`vg#RzP7F?W<{s*2Srjg-ZZSy8gCrej?$-wV zqP@ZT(0rkKZt^bv>(1>5gw0>;c6%Hi6|yG1|2eey1m;*|-?=y3zw{#Z#@>amCkBu3 z*-?O#0mAtsNOLhnYw5_@8M{=fs8_45^~Nb?uBj-XV){g*;e>!4tLUn&c*u%v?#8v4 zryXJs?GuwLxLFIj;SQXRgI1%(H|K@V?R_=ti!|FPf5OM*5nv;+e^AZZzAQmVo6J@O zZLF<{xwZBH=~a2~av{rlTk;7eBnT6VybI6tczEB^Q|w(48pzw9$IW+Vc%OKo)9LV< zCmuJm=&!dDY)QnOvcPxg&8&=%ECY&v> zzG*9BL3AiMwAh)wMgZQ~jlmq$wP^~rGfhi3iIS|)RwKWg;eDUg3f2fj)%_D|_=NiR z+tUOUbi7^S&&jn8tq;oN4;N!TY)j7HzeA%l)h2v7^T`KF|3b*Rnl&8T@8g;mmXY3Yryg#Ma+nl@cK0YqsjITP!52 zP1$o?PnhpE3zp8odo48C?^yXM#y zHf?LUW2=4a(%TNxTa{4jWz{=!TX!#qxUC|3&P3%uM(lcfifOQ&j*HkGxXl+EVf27c zly$n9J5)pbcdzEL`GLO!9pIvC*j=NDB z>iQ!4KQ&~@%WvNozWczbA!B~r{8pF$eTUOS*8FvQ^Y1?hB@=X+LZosFtNw{Su7j3R4(nTairqR=l3arvA1e_FCYp(^c1$RhuisI9mww7+%p!5=OC|+5#AH| zx(B&#{Kl?MGPKVdn3~)<`MAtgS@mYOzWlRezCE6Q71KV-VC5GkHv>A_X8M!TzJLr) z4ViiJp1hmAx2do1ap@gHpX>g<56_Sm+zH=>!a{705(ZNNM;;=>n_CYH%DZ>wgl&p9iqEXi}r1QO4S1|1wsmXXgJ z3Mb42CdC*;1WMZw)y9Uj^$k0+Hmq9zH!+fH(V%}PYYy2r34mVywDx3oz{(Kb) z*eeGdcAg7^xY5@CrL;Zyrr?mH@zJY3G4GD&OO#Qpk`Er|0b=)MiY8y;RNxFEl|~3} z%c7p0d|MtjMxY_%f4qBJsg%tG5u>Lu@PQ9qg-V_d93~CcRa+XV&G!ucj~$`$U42>1 zd&0GS4*?L6Q1pEw?5q_I?_f!H-X{r~ZP16~j7PSDn@E^&SlGF@v%y6V*}@d46g#!v zIUF7A%!HJEjogP&Y|J@)prxk@5MV-fY=BG{7!$7BcH3T5=%|%~2nyv6u*~ z6SH(e*aR$rCf5oj4IhyKv2k=7+B~R_ zgN@J8X*)dFg@ZDXbS-^2)4S9P0qCTi{dL^Jp7pVn`hJoJW z;JOh;+eX)7Onwc9FsZ)*>s^SbRg_#Z!xFi%_~(S1v=ICVz>`IGqNd~};vOK6_RI){ zl4*{TzTCKE(vng;$7)JRE`+L3fz%-fUJ}>J?HO)U`|Co@0P^P+i1MT)@3;&C0AH;Q z#*)L$bg~nijcI&8tf-iVGqmhxyzqYp_B96Unf)tW%-V9KK`RCFc@rmADcOJspvuBQr|M2M~bM6z|#u zrS$?pbr%DH?vWXYJ0MFTnQEp?toF(YgEo_i#ko#g+CPj&Kb0iAMotlFHA2wHtH%G6FAvm=Yp#oJrik0S)t}|2hUX5s4P6oi_ za4r=lzE1DK)BzKYV4+AJMZZVDT2sUvTK02QNEG1$f)F|#s-)_K(-YF5YX2Gv<{;nO-4Sic?+zN-B@dt{@$a>jMG( z?~=J{pYryGEE(BkO!Tm<+?iBpEbG=T)*MI){=+H6} zg`-;N`y(I~prqjC$nm`eWaVu2)WnbLwW^1c95Q~C8kruiKMH`3B<9^1vun{GjjyMf zfV@07fIpZu_Cytd#aA>UkwF%!rzajCbqnhd23zKxBoB&m z9c~G>jl%6ZMY~tpxD%=_XtG2Xz}>NF-kn~+I^x&|^tsq`c-~VkQO%MM3LB3*d=K$d z{v`=SehRf2odANTjYOCCd{rsoIQx#=m^QPw7Lf9=5K|oD(ZIbI4k5mG3p|X;@A!r< zVV|_rs+tkd9l`n$5ejuBhADB;*Let;5b4>&Tw-MX7_XnQ3=tU7p~ZzY(9AmSPP$Ov zki-E1vqs?(Z^?)4PXz(M^tgP{a_g18b-FAt-mV7lhi!n8PMOc2js2*7_-=hS5_o=8 zKuh&k3`$-Fhja}~gtGHtc>FOp_pk4?2Tq*0IOQplK@zUeOUIIwEB|qKGKC50=#?~# zEE$X6DZxPeoocT^IiqnKm&`k$wD3^Bh(NCqAW=4rOjf0Gs{n~+(>}05k#?{q`H+PD z1Ei3O06g-+1rSMaQ$-R>J+%NV8}l~hW&C61?@5BVXKq?WFHf2lf3wRF=i&XuM0HGF z*R1rjmf=Z((s@(3Ly#y5qOiz@XS%YH<8-NMau>HR;1|l|>CdEEyfsJmh;zv5U!Az3 zmc)!rj=cd#6cnwb^M&$4w_OflDY-nY$I93tD^hPgZ0H|5_0G z{hz&8i39S*qgx=7TKJO-yr5F_N>l>Jo^Y3AZasPUu~v(D$xgfTO`ar54FGn3fgji) zjWGyyC{Az4JV{SeXC(R(~Vhs_%tDqDtW0!St&KNyqGKd|AOADz}%Er??K3UYs3idT{1GdmRe9!(vY6cTrF%NB(q zdes1OfAO6Acw&Q?9+qe(i@{r}AaknqbA}MxBqZOmrUK6n5MY^>6qd##1mE;bN#4M7 zWAITDc+S@Yq^14&NqU@720+eephHitr8c~!oi55u_?BA3se`gdnJ$BSZ_B+OhVb-V z+!RHPu;7d7vA67H`VoKw64$f5;;S5C_Z^oHPf6Ug_EYA=jbq_jf^I*z}>#m3PO;vmK<3JsonZ{%tI_Tr}R$ow`!LSYc7VyHl~7NpV0)I1&}aZ0w% z5c%_K+^ry_Z#?obG6g0oiIOu_ypp01W~N}^Xa|&~al!%K!h^*Hsm26v0?LjEe0hs> zWrNZbV;()E8nciVc2!V0!vXKD5a)gue0`wPp#Q7=>8hA&z9p%b%m{4e4Q@C7H z3DqjRV_e_BKP#5J1OW@bt1~e*Wz&HrZ9wl`g8MVV=^u{#2?W!Yn##7iQJdQ9L>SA` z)NmAc($d6IDxDILxD12|B}io>O5Vn8fF~f_hF}jMRJ0H$3a*_E_B_yNMO<}J*7wy1 zb^>%A3043?fgEV6B`aY=Frn0Byt#HR_z3;ErIjO}F-1#}Tor}CpzL13*&zsG$}n$i zq!^&Q7QkrN40+@EUjG2da#3YKn06qq8eFS|yQL28{tm)KJL{j$;kyrbJayrsduv#8 za>Nc?bUPxahpGjk^PM9?fhDzoTW_0sGF_e=zNu4irh|geiHJ*BLKsv7^#>a>$>3-z z;j#|FNI`e)wqwijqaR0r6Hg)dFub;M>k8CzO$07-b`OPUQ)@vc!5_bRVQV`Iapr?> zJANH-jlUBRwT}=p;rQBwU_?1s`@P$SyEEUi$JMj5_)Citk@1PET|AZ0w(WeS=h!iMD$Mby%7)2t6CnZK?p)5H8&5dXb3DFwC-7YA?WQJp&l^V!(`I>(-5u{c z7gKwR@)`!B4);ws@H!H7DDaz!&ePrmC0zVHhPC9h{nMIC>D;||h5>_@IkU}a1O!Ut z56dv40cw*j`?N%=!`(aG*@p?cF;?nC2g%+ekRy0vq!VuzTJ90*0zf&d9FiZ{#l4Rc zr8XsU9tU_I%hpN@R(huNEEtNP8FXnGY>ytkd5{3N)Q)e!=5%Pl3b-*KxpvLj6^E%;xUG=2X)_@@ew!RB2iA1{}jMi!I7Qr0|PO;OaM7ykxg_$7`0HAvo8DLn4aqQ1?__Hx zylC6PsT*_h4`N3^_hH!;zu3(*4P}N#y~8OzBBIvzvH|Of9tMH;S68O0OH)CeBRbE0 zftY(N)H@BlI={j+lYmY-I@63A)U(r4*Q3nX-&FU_05o}JUx#Zs=*Y4+^NLcv)^Na64ihOvvsJXPbQO>u#>($)eSKpjp{y6`t zr~K8H`uv}luVVhb0%WHlbM`YIHh0A+hoGJ5Ve>b42>fGzl3Eh#uNOJGq?xcJQBf+@ zzjR=JN%qqvoa}P2(X!%&1f>hh%&W_)&HieO%TwQ%wVFM2%wJD=yf)As1fPF0YW6v=01Z%R#$F%HI4emuKNw zNBbAY&G!0FvQ7PDs^V9-oE1mG>5vPH{?F&aE7)9LGNJ|8g4OBqvJRZ|QQX(vH1nOc z9T*-_=)I7mb~Qw>!cD(VPkvgL!~uThAEmVm=Ffec6aba(f7Ic`*10}S=D^O@pS1Dj z|FS=I9^Gd%`AHTFz7pV=Dzd+62=P^){WbaC|H<0+CyXMcbDV}yVShgN6w+j#)x||M zH(4%tS}I#xDln~ju35@^5RkXdnj~))=^*w{Esb!Ssgb}-qYddqV77Ee&7Kj;v8Zgq z`{jtB`-6Rk!7gLDLw*4P0r@dMtICcB$ec%g)j3As4Dc`JALAv~m>iQK{LxpU7pvaF z3RagrfI#gDx6(@#nMLN6~XAFw?bM2Eg)LI>B#CvQ5F@c zXXP~!1@+>^`jq1OlY&z^#Z$uEwzlu1liC8lr!1ny(xO@w7wKl?)vut|m${^PH7WS_0u z3J5|@gM@{KhlKz

hX^kC2g)la!T~mzbHFo1C4VpP-?lqok##r91$LtE>yEgg*cx zMgR*+PD2C&MuQqiPD=nShd>7aD1*Vsg*wIn8iytT1h=k*u_A;k001zAAKin}BZkGt zgDWP3v=prFhl;22^Yr!h_xSnx`~01$?*N@Zcr_>%iW>mb4VsbA$%-4vgsm!&AmTDZ zMqJ@I)B->afdrix95jT%3jz(DSR-T%5ye3TE{K?UfI&cj@%ri9|H-qb&!0epq6&HQ zkcN;rJnoRV4nT&IHKNJ@@Sz4TLm~oLTqPuBAtdLH7E1cjNr8_@7y$Soa6ln&g0N~; z0svysn>mH*-OIPH-@kwx{kw|@1%M%h7!INJA<>5&1tLcVAfUiTLoF6;ojB~!0EmVW zFFfpI>5`@;oPYpu)rsNOgfas=M;Pa#z_@ek-p#xBqrtTc(egHEgk~oc4iFj=&Jfc> z5ELp~r6hppLOTZB8cxXgppLB>K;Qxb8S*FJZTaP>V#bSO{&-ylr=AOHXm zlyK1x7XVNI45z&yfB+w$P+%_r3?RY?0TL4AkVWEXx6hEPhglJThMEbX zo0e+osi?*`pQx<1>MEe5rYh^Kw4S=^tGMQ>>u#~u>g%t-x^wHU#1?C8PrU}K?6S2E zd+f8&Mr%s4%vNhHpUz6F?Y7)jLhZHSh8s|}-Ii;v|F++XtM0mlkbCaC@amc_yYzzV zZoK&B`{}&y2%*Ad4%OHputTWuB`ZtZkeYx71Ej+U09$*nz7$uiBfqOG*qlnoIS9e8 zMu@QSDlm@bbcJ*wZq^2wZB3VLc3{-vpo0@Lpg`#kD4>v( z)J8~M(lslfz+ypp1AM~@-Yn6=3S9q?H;cn50fXCxDDi>`HDvws434YNS`AWry&Mc9 zaG1d}FI=Sq&WSpK#Nsm`w!@Y$*tN>bXU8u4|2@4bag+uch%f*(;(V}#5d)?CLI4Ol zp@Is8n4pf(g30t8rq zTM-i1rV~aKFw7D#08qjXw|p=I5E&SNAe69x(`djvKybhSWF#!5&F+B^yoV0}P&!0F zX%Gm3NK#JYg#1Yj0>xv1Xll?NCuG1TFc1I-nF9?ImT&g9Xz9gQ+!8Pj-bU#S~3(sV}%WCI1DZ=CzD4+(2&}NCnh~{5FA*G3kl&y z`INGTd4#1ySaTlqxo!ji*juJHh652q4w2!Yi|Y!3O950UlI~#0C6~#}OhkZ<7W5%4 zh5>*Nm2nVf90UaR2E;??q7b23AOg({s6Zj%0{Fd7ni5F>01S1_P&z>g#dCr}uVbDnmFWWT(||!+Inl(( zQkFH4P%msr2nw81p5Z700O+J4|6mfwWbQFSg&@$Ley(BwE*Pjz1=@s_t|Ay5_!;sT zl7T@OMg)n2{Kp^_*qCv4B zjuHmIA`rCD3FOIRS0J#29e{5MOQ1_*fB+*I(Z~rUxQuL^AhID0A)bXWUlVr9SZ0!d ziiB8#$Ov!)BUEVtY*9e~fVcuNu_&7qAYUO;NEw!vrT`?jS!d`LYW%JimFlPG>BaXVe$)KLwM12 z`NiuhN+19xnD@Y0qyYd9#wQAZpk5Fr%u;B8;0i}@0|T%WVM@@T1aK(AAZEo72CxAO zn|O9lXg~opTomSV;FTecF-Z#0fG3>z#%F6n0|>xi8T*(|c5T37aZKc`LU9N zV;BtBxX4gunhON*+9g{#AT=OsPNU4_r6pkl=QDw5uS{mRn!o@6*gz7x%;u#X;RI-& zh5-&h=Q`W@&UntVp7+e>KKuF4fDW{s4S*5=7~lkyY_p=hoP!v+fCZ3_w4^6Z=}KGr z(wNS)rZ>&$PD|PaCNaptnioy#Qk(kJs7|%2SIz2HyZY6zj Date: Sat, 4 May 2013 19:47:57 -0700 Subject: [PATCH 05/97] First draft of new README. Feedback and contributions welcome! --- README.md | 91 ++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 73 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 13ec817e2b..7b1e18f390 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,92 @@ -Docker: the Linux container runtime -=================================== +Docker: the Linux container engine +================================== -Docker complements LXC with a high-level API which operates at the process level. It runs unix processes with strong guarantees of isolation and repeatability across servers. +Docker is an open-source engine which automates the deployment of applications as highly portable, self-sufficient containers. -Docker is a great building block for automating distributed systems: large-scale web deployments, database clusters, continuous deployment systems, private PaaS, service-oriented architectures, etc. +Docker containers are both *hardware-agnostic* and *platform-agnostic*. This means that they can run anywhere, from your +laptop to the largest EC2 compute instance and everything in between - and they don't require that you use a particular +language, framework or packaging system. That makes them great building blocks for deploying and scaling web apps, databases +and backend services without depending on a particular stack or provider. -![Docker L](docs/sources/static_files/lego_docker.jpg "Docker") +Docker is an open-source implementation of the deployment engine which powers [dotCloud](http://dotcloud.com), a popular Platform-as-a-Service. +It benefits directly from the experience accumulated over several years of large-scale operation and support of hundreds of thousands +of applications and databases. -* *Heterogeneous payloads*: any combination of binaries, libraries, configuration files, scripts, virtualenvs, jars, gems, tarballs, you name it. No more juggling between domain-specific tools. Docker can deploy and run them all. +## Better than VMs -* *Any server*: docker can run on any x64 machine with a modern linux kernel - whether it's a laptop, a bare metal server or a VM. This makes it perfect for multi-cloud deployments. +A common method for distributing applications and sandbox their execution is to use virtual machines, or VMs. Typical VM formats +are VMWare's vmdk, Oracle Virtualbox's vdi, and Amazon EC2's ami. In theory these formats should allow every developer to +automatically package their application into a "machine" for easy distribution and deployment. In practice, that almost never +happens, for a few reasons: -* *Isolation*: docker isolates processes from each other and from the underlying host, using lightweight containers. + * *Size*: VMs are very large which makes them impractical to store and transfer. + * *Performance*: running VMs consumes significant CPU and memory, which makes them impractical in many scenarios, for example local development of multi-tier applications, and + large-scale deployment of cpu and memory-intensive applications on large numbers of machines. + * *Portability*: competing VM environments don't play well with each other. Although conversion tools do exist, they are limited and add even more overhead. + * *Hardware-centric*: VMs were designed with machine operators in mind, not software developers. As a result, they offer very limited tooling for what developers need most: + building, testing and running their software. For example, VMs offer no facilities for application versioning, monitoring, configuration, logging or service discovery. -* *Repeatability*: because containers are isolated in their own filesystem, they behave the same regardless of where, when, and alongside what they run. +By contrast, Docker relies on a different sandboxing method known as *containerization*. Unlike traditional virtualization, +containerization takes place at the kernel level. Most modern operating system kernels now support the primitives necessary +for containerization, including Linux with [openvz](http://openvz.org), [vserver](http://linux-vserver.org) and more recently [lxc](http://lxc.sourceforge.net), + Solaris with [zones](http://docs.oracle.com/cd/E26502_01/html/E29024/preface-1.html#scrolltoc) and FreeBSD with [Jails](http://www.freebsd.org/doc/handbook/jails.html). + +Docker builds on top of these low-level primitives to offer developers a portable format and runtime environment that solves +all 4 problems. Docker containers are small (and their transfer can be optimized with layers), they have basically zero memory and cpu overhead, +the are completely portable and are designed from the ground up with an application-centric design. + +The best part: because docker operates at the OS level, it can still be run inside a VM! + +## Plays well with others + +Docker does not require that you buy into a particular programming language, framework, packaging system or configuration language. + +Is your application a unix process? Does it use files, tcp connections, environment variables, standard unix streams and command-line +arguments as inputs and outputs? Then docker can run it. + +Can your application's build be expressed a sequence of such commands? Then docker can build it. -Notable features ------------------ +## Escape dependency hell -* Filesystem isolation: each process container runs in a completely separate root filesystem. +A common problem for developers is the difficulty of managing all their application's dependencies in a simple and automated way. -* Resource isolation: system resources like cpu and memory can be allocated differently to each process container, using cgroups. +This is usually difficult for several reasons: -* Network isolation: each process container runs in its own network namespace, with a virtual interface and IP address of its own. + * *Cross-platform dependencies*. Modern applications often depend on a combination of system libraries and binaries, language-specific packages, framework-specific modules, + internal components developed for another project, etc. These dependencies live in different "worlds" and require different tools - these tools typically don't work + well with each other, requiring awkward custom integrations. -* Copy-on-write: root filesystems are created using copy-on-write, which makes deployment extremely fast, memory-cheap and disk-cheap. + * Conflicting dependencies. Different applications may depend on different versions of the same dependency. Packaging tools handle these situations with various degrees of ease - + but they all handle them in different and incompatible ways, which again forces the developer to do extra work. + + * Custom dependencies. A developer may need to prepare a custom version of his application's dependency. Some packaging systems can handle custom versions of a dependency, + others can't - and all of them handle it differently. -* Logging: the standard streams (stdout/stderr/stdin) of each process container are collected and logged for real-time or batch retrieval. -* Change management: changes to a container's filesystem can be committed into a new image and re-used to create more containers. No templating or manual configuration required. +Docker solves dependency hell by giving the developer a simple way to express *all* his application's dependencies in one place, +and streamline the process of assembling them. If this makes you think of [XKCD 927](http://xkcd.com/927/), don't worry. Docker doesn't +*replace* your favorite packaging systems. It simply orchestrates their use in a simple and repeatable way. How does it do that? With layers. + +Docker defines a build as running a sequence unix commands, one after the other, in the same container. Build commands modify the contents of the container +(usually by installing new files on the filesystem), the next command modifies it some more, etc. Since each build command inherits the result of the previous +commands, the *order* in which the commands are executed expresses *dependencies*. + +Here's a typical docker build process: + +```bash +from ubuntu:12.10 +run apt-get update +run apt-get install python +run apt-get install python-pip +run pip install django +run apt-get install curl +run curl http://github.com/shykes/helloflask/helloflask/master.tar.gz | tar -zxv +run cd master && pip install -r requirements.txt +``` + +Note that Docker doesn't care *how* dependencies are built - as long as they can be built by running a unix command in a container. -* Interactive shell: docker can allocate a pseudo-tty and attach to the standard input of any container, for example to run a throwaway interactive shell. Install instructions ================== From 424cc678ebac93c69c288fcdbc2e1a4bd0493328 Mon Sep 17 00:00:00 2001 From: Barry Allard Date: Sat, 4 May 2013 21:20:41 -0700 Subject: [PATCH 06/97] closes #522 --- contrib/docker-build/docker-build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker-build/docker-build b/contrib/docker-build/docker-build index c82377e107..654cc89793 100755 --- a/contrib/docker-build/docker-build +++ b/contrib/docker-build/docker-build @@ -89,7 +89,7 @@ def main(): # Skip comments and empty lines if line == "" or line[0] == "#": continue - op, param = line.split(" ", 1) + op, param = line.split(None, 1) print op.upper() + " " + param if op == "from": base = param From 09d4b9452d80b70bf4d510891cf3545b3350cdf2 Mon Sep 17 00:00:00 2001 From: Ken Cochrane Date: Mon, 6 May 2013 13:38:51 -0400 Subject: [PATCH 07/97] added new sphinx contrib extention for better REST API docs, and changed the index search API so that it uses the new docs, as a test to make sure it works correctly --- docs/requirements.txt | 2 ++ docs/sources/commandline/index.rst | 2 +- docs/sources/conf.py | 2 +- docs/sources/index/search.rst | 38 ++++++++++++++++++++++-------- docs/sources/registry/api.rst | 6 ++++- 5 files changed, 37 insertions(+), 13 deletions(-) create mode 100644 docs/requirements.txt diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000000..9d1ce18f5c --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +Sphinx==1.1.3 +sphinxcontrib-httpdomain==1.1.8 \ No newline at end of file diff --git a/docs/sources/commandline/index.rst b/docs/sources/commandline/index.rst index d5ea959f6e..d19d39ab60 100644 --- a/docs/sources/commandline/index.rst +++ b/docs/sources/commandline/index.rst @@ -9,7 +9,7 @@ Commands Contents: .. toctree:: - :maxdepth: 2 + :maxdepth: 3 basics workingwithrepository diff --git a/docs/sources/conf.py b/docs/sources/conf.py index c05c7d10a4..4c54d8bb62 100644 --- a/docs/sources/conf.py +++ b/docs/sources/conf.py @@ -25,7 +25,7 @@ import sys, os # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = [] +extensions = ['sphinxcontrib.httpdomain'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/docs/sources/index/search.rst b/docs/sources/index/search.rst index 0c7624de72..498295fa2b 100644 --- a/docs/sources/index/search.rst +++ b/docs/sources/index/search.rst @@ -5,16 +5,34 @@ Docker Index Search API Search ------ -**URL:** /v1/search?q={{search_term}} +.. http:get:: /v1/search -**Results:** + Search the Index given a search term. It accepts :http:method:`get` only. -.. code-block:: json + **Example request**: - {"query":"{{search_term}}", - "num_results": 27, - "results" : [ - {"name": "dotcloud/base", "description": "A base ubuntu64 image..."}, - {"name": "base2", "description": "A base ubuntu64 image..."}, - ] - } \ No newline at end of file + .. sourcecode:: http + + GET /v1/search?q=search_term HTTP/1.1 + Host: example.com + Accept: application/json + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Vary: Accept + Content-Type: application/json + + {"query":"search_term", + "num_results": 2, + "results" : [ + {"name": "dotcloud/base", "description": "A base ubuntu64 image..."}, + {"name": "base2", "description": "A base ubuntu64 image..."}, + ] + } + + :query q: what you want to search for + :statuscode 200: no error + :statuscode 500: server error \ No newline at end of file diff --git a/docs/sources/registry/api.rst b/docs/sources/registry/api.rst index 1cca9fb244..ec2591af4c 100644 --- a/docs/sources/registry/api.rst +++ b/docs/sources/registry/api.rst @@ -84,7 +84,9 @@ It’s possible to run docker pull https:///repositories/samalba/busyb Currently registry redirects to s3 urls for downloads, going forward all downloads need to be streamed through the registry. The Registry will then abstract the calls to S3 by a top-level class which implements sub-classes for S3 and local storage. -Token is only returned when it is a private repo, public repos do not require tokens to be returned. The Registry will still contact the Index to make sure the pull is authorized (“is it ok to download this repos without a Token?”). +Token is only returned when the 'X-Docker-Token' header is sent with request. + +Basic Auth is required to pull private repos. Basic auth isn't required for pulling public repos, but if one is provided, it needs to be valid and for an active account. API (pulling repository foo/bar): ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -426,6 +428,8 @@ You have 3 options: - X-Docker-Token: true In this case, along with the 200 response, you’ll get a new token (if user auth is ok): + If authorization isn't correct you get a 401 response. + If account isn't active you will get a 403 response. **Response**: - 200 OK From 2421838b0ac6a496503bc896da0e60d195132576 Mon Sep 17 00:00:00 2001 From: shin- Date: Mon, 15 Apr 2013 03:28:54 -0700 Subject: [PATCH 08/97] Support for the new registry/index API (wip) --- auth/auth.go | 6 ++--- registry.go | 64 +++++++++++++++++++++++++++------------------------- 2 files changed, 36 insertions(+), 34 deletions(-) diff --git a/auth/auth.go b/auth/auth.go index 2eb63b1d4a..904198da5f 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -16,7 +16,7 @@ import ( const CONFIGFILE = ".dockercfg" // the registry server we want to login against -const REGISTRY_SERVER = "https://registry.docker.io" +const INDEX_SERVER = "https://index.docker.io" type AuthConfig struct { Username string `json:"username"` @@ -113,7 +113,7 @@ func Login(authConfig *AuthConfig) (string, error) { // using `bytes.NewReader(jsonBody)` here causes the server to respond with a 411 status. b := strings.NewReader(string(jsonBody)) - req1, err := http.Post(REGISTRY_SERVER+"/v1/users", "application/json; charset=utf-8", b) + req1, err := http.Post(INDEX_SERVER+"/v1/users", "application/json; charset=utf-8", b) if err != nil { errMsg = fmt.Sprintf("Server Error: %s", err) return "", errors.New(errMsg) @@ -134,7 +134,7 @@ func Login(authConfig *AuthConfig) (string, error) { // FIXME: This should be 'exists', not 'exist'. Need to change on the server first. if string(reqBody) == "Username or email already exist" { client := &http.Client{} - req, err := http.NewRequest("GET", REGISTRY_SERVER+"/v1/users", nil) + req, err := http.NewRequest("GET", INDEX_SERVER+"/v1/users", nil) req.SetBasicAuth(authConfig.Username, authConfig.Password) resp, err := client.Do(req) if err != nil { diff --git a/registry.go b/registry.go index 74b166906f..73bf6a1f03 100644 --- a/registry.go +++ b/registry.go @@ -13,8 +13,8 @@ import ( ) //FIXME: Set the endpoint in a conf file or via commandline -//const REGISTRY_ENDPOINT = "http://registry-creack.dotcloud.com/v1" -const REGISTRY_ENDPOINT = auth.REGISTRY_SERVER + "/v1" +//const INDEX_ENDPOINT = "http://registry-creack.dotcloud.com/v1" +const INDEX_ENDPOINT = auth.INDEX_SERVER + "/v1" // Build an Image object from raw json data func NewImgJson(src []byte) (*Image, error) { @@ -48,10 +48,10 @@ func NewMultipleImgJson(src []byte) ([]*Image, error) { // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) -func (graph *Graph) getRemoteHistory(imgId string, authConfig *auth.AuthConfig) ([]*Image, error) { +func (graph *Graph) getRemoteHistory(imgId, registry string, authConfig *auth.AuthConfig) ([]*Image, error) { client := &http.Client{} - req, err := http.NewRequest("GET", REGISTRY_ENDPOINT+"/images/"+imgId+"/history", nil) + req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/ancestry", nil) if err != nil { return nil, err } @@ -78,29 +78,26 @@ func (graph *Graph) getRemoteHistory(imgId string, authConfig *auth.AuthConfig) } // Check if an image exists in the Registry -func (graph *Graph) LookupRemoteImage(imgId string, authConfig *auth.AuthConfig) bool { +func (graph *Graph) LookupRemoteImage(imgId, registry string, authConfig *auth.AuthConfig) bool { rt := &http.Transport{Proxy: http.ProxyFromEnvironment} - req, err := http.NewRequest("GET", REGISTRY_ENDPOINT+"/images/"+imgId+"/json", nil) + req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) if err != nil { return false } req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := rt.RoundTrip(req) - if err != nil || res.StatusCode != 307 { - return false - } - return res.StatusCode == 307 + return err == nil && res.StatusCode == 307 } // Retrieve an image from the Registry. // Returns the Image object as well as the layer as an Archive (io.Reader) -func (graph *Graph) getRemoteImage(stdout io.Writer, imgId string, authConfig *auth.AuthConfig) (*Image, Archive, error) { +func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, authConfig *auth.AuthConfig) (*Image, Archive, error) { client := &http.Client{} fmt.Fprintf(stdout, "Pulling %s metadata\r\n", imgId) // Get the Json - req, err := http.NewRequest("GET", REGISTRY_ENDPOINT+"/images/"+imgId+"/json", nil) + req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) if err != nil { return nil, nil, fmt.Errorf("Failed to download json: %s", err) } @@ -127,7 +124,7 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId string, authConfig *a // Get the layer fmt.Fprintf(stdout, "Pulling %s fs layer\r\n", imgId) - req, err = http.NewRequest("GET", REGISTRY_ENDPOINT+"/images/"+imgId+"/layer", nil) + req, err = http.NewRequest("GET", registry+"/images/"+imgId+"/layer", nil) if err != nil { return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err) } @@ -139,7 +136,7 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId string, authConfig *a return img, ProgressReader(res.Body, int(res.ContentLength), stdout, "Downloading %v/%v (%v)"), nil } -func (graph *Graph) PullImage(stdout io.Writer, imgId string, authConfig *auth.AuthConfig) error { +func (graph *Graph) PullImage(stdout io.Writer, imgId, registry string, authConfig *auth.AuthConfig) error { history, err := graph.getRemoteHistory(imgId, authConfig) if err != nil { return err @@ -148,7 +145,7 @@ func (graph *Graph) PullImage(stdout io.Writer, imgId string, authConfig *auth.A // FIXME: Lunch the getRemoteImage() in goroutines for _, j := range history { if !graph.Exists(j.Id) { - img, layer, err := graph.getRemoteImage(stdout, j.Id, authConfig) + img, layer, err := graph.getRemoteImage(stdout, j.Id, registry, authConfig) if err != nil { // FIXME: Keep goging in case of error? return err @@ -162,7 +159,7 @@ func (graph *Graph) PullImage(stdout io.Writer, imgId string, authConfig *auth.A } // FIXME: Handle the askedTag parameter -func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, repositories *TagStore, authConfig *auth.AuthConfig) error { +func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag, registry string, repositories *TagStore, authConfig *auth.AuthConfig) error { client := &http.Client{} fmt.Fprintf(stdout, "Pulling repository %s\r\n", remote) @@ -170,9 +167,9 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re var repositoryTarget string // If we are asking for 'root' repository, lookup on the Library's registry if strings.Index(remote, "/") == -1 { - repositoryTarget = REGISTRY_ENDPOINT + "/library/" + remote + repositoryTarget = registry + "/library/" + remote } else { - repositoryTarget = REGISTRY_ENDPOINT + "/users/" + remote + repositoryTarget = registry + "/users/" + remote } req, err := http.NewRequest("GET", repositoryTarget, nil) @@ -198,7 +195,7 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re } for tag, rev := range t { fmt.Fprintf(stdout, "Pulling tag %s:%s\r\n", remote, tag) - if err = graph.PullImage(stdout, rev, authConfig); err != nil { + if err = graph.PullImage(stdout, rev, registry, authConfig); err != nil { return err } if err = repositories.Set(remote, tag, rev, true); err != nil { @@ -303,7 +300,7 @@ func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, authConfig *auth // push a tag on the registry. // Remote has the format '/ -func (graph *Graph) pushTag(remote, revision, tag string, authConfig *auth.AuthConfig) error { +func (graph *Graph) pushTag(remote, revision, tag, registry string, authConfig *auth.AuthConfig) error { // Keep this for backward compatibility if tag == "" { @@ -313,10 +310,10 @@ func (graph *Graph) pushTag(remote, revision, tag string, authConfig *auth.AuthC // "jsonify" the string revision = "\"" + revision + "\"" - Debugf("Pushing tags for rev [%s] on {%s}\n", revision, REGISTRY_ENDPOINT+"/users/"+remote+"/"+tag) + Debugf("Pushing tags for rev [%s] on {%s}\n", revision, registry+"/users/"+remote+"/"+tag) client := &http.Client{} - req, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/users/"+remote+"/"+tag, strings.NewReader(revision)) + req, err := http.NewRequest("PUT", registry+"/users/"+remote+"/"+tag, strings.NewReader(revision)) req.Header.Add("Content-type", "application/json") req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := client.Do(req) @@ -337,15 +334,20 @@ func (graph *Graph) pushTag(remote, revision, tag string, authConfig *auth.AuthC return nil } -func (graph *Graph) LookupRemoteRepository(remote string, authConfig *auth.AuthConfig) bool { +func (graph *Graph) LookupRemoteRepository(remote, registry string, authConfig *auth.AuthConfig) bool { rt := &http.Transport{Proxy: http.ProxyFromEnvironment} var repositoryTarget string // If we are asking for 'root' repository, lookup on the Library's registry if strings.Index(remote, "/") == -1 { - repositoryTarget = REGISTRY_ENDPOINT + "/library/" + remote + "/lookup" + repositoryTarget = registry + "/library/" + remote + "/lookup" } else { - repositoryTarget = REGISTRY_ENDPOINT + "/users/" + remote + "/lookup" + parts := strings.Split(remote, "/") + if len(parts) != 2 { + Debugf("Repository must abide to following format: user/repo_name") + return false + } + repositoryTarget = registry + "/users/" + parts[0] + "/repositories/" + parts[1] } Debugf("Checking for permissions on: %s", repositoryTarget) req, err := http.NewRequest("PUT", repositoryTarget, strings.NewReader("\"\"")) @@ -368,7 +370,7 @@ func (graph *Graph) LookupRemoteRepository(remote string, authConfig *auth.AuthC } // FIXME: this should really be PushTag -func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId string, authConfig *auth.AuthConfig) error { +func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId, registry string, authConfig *auth.AuthConfig) error { // Check if the local impage exists img, err := graph.Get(imgId) if err != nil { @@ -377,12 +379,12 @@ func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId string, a } fmt.Fprintf(stdout, "Pushing tag %s:%s\r\n", remote, tag) // Push the image - if err = graph.PushImage(stdout, img, authConfig); err != nil { + if err = graph.PushImage(stdout, img, registry, authConfig); err != nil { return err } fmt.Fprintf(stdout, "Registering tag %s:%s\r\n", remote, tag) // And then the tag - if err = graph.pushTag(remote, imgId, tag, authConfig); err != nil { + if err = graph.pushTag(remote, imgId, registry, tag, authConfig); err != nil { return err } return nil @@ -390,16 +392,16 @@ func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId string, a // Push a repository to the registry. // Remote has the format '/ -func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Repository, authConfig *auth.AuthConfig) error { +func (graph *Graph) PushRepository(stdout io.Writer, remote, registry string, localRepo Repository, authConfig *auth.AuthConfig) error { // Check if the remote repository exists/if we have the permission - if !graph.LookupRemoteRepository(remote, authConfig) { + if !graph.LookupRemoteRepository(remote, registry, authConfig) { return fmt.Errorf("Permission denied on repository %s\n", remote) } fmt.Fprintf(stdout, "Pushing repository %s (%d tags)\r\n", remote, len(localRepo)) // For each image within the repo, push them for tag, imgId := range localRepo { - if err := graph.pushPrimitive(stdout, remote, tag, imgId, authConfig); err != nil { + if err := graph.pushPrimitive(stdout, remote, tag, imgId, registry, authConfig); err != nil { // FIXME: Continue on error? return err } From 2f082510a70a19fa83a7214485e3838e15bde22f Mon Sep 17 00:00:00 2001 From: shin- Date: Mon, 15 Apr 2013 11:17:03 -0700 Subject: [PATCH 09/97] Implemented new version of PullRepository. Missing support for whole repository pull (= no tag specified) --- commands.go | 8 ++- registry.go | 171 ++++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 144 insertions(+), 35 deletions(-) diff --git a/commands.go b/commands.go index 4be282bce2..dc12935265 100644 --- a/commands.go +++ b/commands.go @@ -553,6 +553,8 @@ func (srv *Server) CmdPush(stdin io.ReadCloser, stdout rcli.DockerConn, args ... func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "pull", "NAME", "Pull an image or a repository from the registry") + tag := cmd.String("t", "", "Download tagged image in repository") + registry := cmd.String("registry", "", "Registry to download from. Necessary if image is pulled by ID") if err := cmd.Parse(args); err != nil { return nil } @@ -563,14 +565,14 @@ func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string } // FIXME: CmdPull should be a wrapper around Runtime.Pull() - if srv.runtime.graph.LookupRemoteImage(remote, srv.runtime.authConfig) { - if err := srv.runtime.graph.PullImage(stdout, remote, srv.runtime.authConfig); err != nil { + if *registry != "" { + if err := srv.runtime.graph.PullImage(stdout, remote, *registry, nil); err != nil { return err } return nil } // FIXME: Allow pull repo:tag - if err := srv.runtime.graph.PullRepository(stdout, remote, "", srv.runtime.repositories, srv.runtime.authConfig); err != nil { + if err := srv.runtime.graph.PullRepository(stdout, remote, *tag, srv.runtime.repositories, srv.runtime.authConfig); err != nil { return err } return nil diff --git a/registry.go b/registry.go index 73bf6a1f03..07e79a5a5f 100644 --- a/registry.go +++ b/registry.go @@ -48,14 +48,15 @@ func NewMultipleImgJson(src []byte) ([]*Image, error) { // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) -func (graph *Graph) getRemoteHistory(imgId, registry string, authConfig *auth.AuthConfig) ([]*Image, error) { +func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([]*Image, error) { client := &http.Client{} req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/ancestry", nil) if err != nil { return nil, err } - req.SetBasicAuth(authConfig.Username, authConfig.Password) + req.Header["X-Docker-Token"] = token + // req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := client.Do(req) if err != nil || res.StatusCode != 200 { if res != nil { @@ -92,7 +93,7 @@ func (graph *Graph) LookupRemoteImage(imgId, registry string, authConfig *auth.A // Retrieve an image from the Registry. // Returns the Image object as well as the layer as an Archive (io.Reader) -func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, authConfig *auth.AuthConfig) (*Image, Archive, error) { +func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, token []string) (*Image, Archive, error) { client := &http.Client{} fmt.Fprintf(stdout, "Pulling %s metadata\r\n", imgId) @@ -101,7 +102,8 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, aut if err != nil { return nil, nil, fmt.Errorf("Failed to download json: %s", err) } - req.SetBasicAuth(authConfig.Username, authConfig.Password) + req.Header["X-Docker-Token"] = token + // req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := client.Do(req) if err != nil { return nil, nil, fmt.Errorf("Failed to download json: %s", err) @@ -128,7 +130,8 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, aut if err != nil { return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err) } - req.SetBasicAuth(authConfig.Username, authConfig.Password) + req.Header["X-Docker-Token"] = token + // req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err = client.Do(req) if err != nil { return nil, nil, err @@ -136,16 +139,16 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, aut return img, ProgressReader(res.Body, int(res.ContentLength), stdout, "Downloading %v/%v (%v)"), nil } -func (graph *Graph) PullImage(stdout io.Writer, imgId, registry string, authConfig *auth.AuthConfig) error { - history, err := graph.getRemoteHistory(imgId, authConfig) +func (graph *Graph) PullImage(stdout io.Writer, imgId, registry string, token []string) error { + history, err := graph.getRemoteHistory(imgId, registry, token) if err != nil { return err } // FIXME: Try to stream the images? - // FIXME: Lunch the getRemoteImage() in goroutines + // FIXME: Launch the getRemoteImage() in goroutines for _, j := range history { if !graph.Exists(j.Id) { - img, layer, err := graph.getRemoteImage(stdout, j.Id, registry, authConfig) + img, layer, err := graph.getRemoteImage(stdout, j.Id, registry, token) if err != nil { // FIXME: Keep goging in case of error? return err @@ -158,58 +161,162 @@ func (graph *Graph) PullImage(stdout io.Writer, imgId, registry string, authConf return nil } -// FIXME: Handle the askedTag parameter -func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag, registry string, repositories *TagStore, authConfig *auth.AuthConfig) error { +// // FIXME: Handle the askedTag parameter +// func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag, registry string, repositories *TagStore, authConfig *auth.AuthConfig) error { +// client := &http.Client{} + +// fmt.Fprintf(stdout, "Pulling repository %s\r\n", remote) + +// var repositoryTarget string +// // If we are asking for 'root' repository, lookup on the Library's registry +// if strings.Index(remote, "/") == -1 { +// repositoryTarget = registry + "/library/" + remote +// } else { +// repositoryTarget = registry + "/users/" + remote +// } + +// req, err := http.NewRequest("GET", repositoryTarget, nil) +// if err != nil { +// return err +// } +// req.SetBasicAuth(authConfig.Username, authConfig.Password) +// res, err := client.Do(req) +// if err != nil { +// return err +// } +// defer res.Body.Close() +// if res.StatusCode != 200 { +// return fmt.Errorf("HTTP code: %d", res.StatusCode) +// } +// rawJson, err := ioutil.ReadAll(res.Body) +// if err != nil { +// return err +// } +// t := map[string]string{} +// if err = json.Unmarshal(rawJson, &t); err != nil { +// return err +// } +// for tag, rev := range t { +// fmt.Fprintf(stdout, "Pulling tag %s:%s\r\n", remote, tag) +// if err = graph.PullImage(stdout, rev, registry, authConfig); err != nil { +// return err +// } +// if err = repositories.Set(remote, tag, rev, true); err != nil { +// return err +// } +// } +// if err = repositories.Save(); err != nil { +// return err +// } +// return nil +// } + +func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, repositories *TagStore, authConfig *auth.AuthConfig) error { client := &http.Client{} fmt.Fprintf(stdout, "Pulling repository %s\r\n", remote) - var repositoryTarget string // If we are asking for 'root' repository, lookup on the Library's registry if strings.Index(remote, "/") == -1 { - repositoryTarget = registry + "/library/" + remote + repositoryTarget = INDEX_ENDPOINT + "/repositories/library/" + remote + "/checksums" } else { - repositoryTarget = registry + "/users/" + remote + repositoryTarget = INDEX_ENDPOINT + "/repositories/" + remote + "/checksums" } req, err := http.NewRequest("GET", repositoryTarget, nil) if err != nil { return err } - req.SetBasicAuth(authConfig.Username, authConfig.Password) + if authConfig != nil { + req.SetBasicAuth(authConfig.Username, authConfig.Password) + } + res, err := client.Do(req) if err != nil { return err } defer res.Body.Close() + // TODO: Right now we're ignoring checksums in the response body. + // In the future, we need to use them to check image validity. if res.StatusCode != 200 { return fmt.Errorf("HTTP code: %d", res.StatusCode) } - rawJson, err := ioutil.ReadAll(res.Body) - if err != nil { - return err + + var token, endpoints []string + if res.Header.Get("X-Docker-Token") != "" { + token = res.Header["X-Docker-Token"] } - t := map[string]string{} - if err = json.Unmarshal(rawJson, &t); err != nil { - return err + if res.Header.Get("X-Docker-Endpoints") != "" { + endpoints = res.Header["X-Docker-Endpoints"] + } else { + return fmt.Errorf("Index response didn't contain any endpoints") } - for tag, rev := range t { - fmt.Fprintf(stdout, "Pulling tag %s:%s\r\n", remote, tag) - if err = graph.PullImage(stdout, rev, registry, authConfig); err != nil { + + // FIXME: If askedTag is empty, fetch all tags. + if askedTag == "" { + askedTag = "latest" + } + + for _, registry := range endpoints { + registryEndpoint := "https://" + registry + "/v1" + if strings.Index(remote, "/") == -1 { + repositoryTarget = registryEndpoint + "/repositories/library/" + + remote + "/tags/" + askedTag + } else { + repositoryTarget = registryEndpoint + "/repositories/users/" + + remote + "/tags/" + askedTag + } + + req, err = http.NewRequest("GET", repositoryTarget, nil) + if err != nil { return err } - if err = repositories.Set(remote, tag, rev, true); err != nil { + req.Header["X-Docker-Token"] = token + res, err := client.Do(req) + if err != nil { + fmt.Fprintf(stdout, "Error while retrieving repository info: %v ; " + + "checking next endpoint") + continue + } + defer res.Body.Close() + if res.StatusCode == 403 { + if authConfig == nil { + return fmt.Errorf("You need to be authenticated to access this resource") + } else { + return fmt.Errorf("You aren't authorized to access this resource") + } + } else if res.StatusCode != 200 { + return fmt.Errorf("HTTP code: %d", res.StatusCode) + } + + var imgId string + rawJson, err := ioutil.ReadAll(res.Body) + if err != nil { return err } + if err = json.Unmarshal(rawJson, &imgId); err != nil { + return err + } + + if err := graph.PullImage(stdout, imgId, registryEndpoint, token); err != nil { + return err + } + + if err = repositories.Set(remote, askedTag, imgId, true); err != nil { + return err + } + + if err = repositories.Save(); err != nil { + return err + } + + return nil } - if err = repositories.Save(); err != nil { - return err - } - return nil + return fmt.Errorf("Could not find repository on any of the indexed registries.") } // Push a local image to the registry with its history if needed -func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, authConfig *auth.AuthConfig) error { +func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, authConfig *auth.AuthConfig) error { client := &http.Client{} // FIXME: Factorize the code @@ -225,7 +332,7 @@ func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, authConfig *auth // FIXME: try json with UTF8 jsonData := strings.NewReader(string(jsonRaw)) - req, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/images/"+img.Id+"/json", jsonData) + req, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/json", jsonData) if err != nil { return err } @@ -252,7 +359,7 @@ func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, authConfig *auth } fmt.Fprintf(stdout, "Pushing %s fs layer\r\n", img.Id) - req2, err := http.NewRequest("PUT", REGISTRY_ENDPOINT+"/images/"+img.Id+"/layer", nil) + req2, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/layer", nil) req2.SetBasicAuth(authConfig.Username, authConfig.Password) res2, err := client.Do(req2) if err != nil { From e639309a7a9c12fe713857a988806285889b580c Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 17 Apr 2013 10:21:32 -0700 Subject: [PATCH 10/97] Reimplemented feature: downloading all tags on a repository using docker pull. Temporarily commented out CmdPush --- commands.go | 102 +++++++++++++------------- registry.go | 207 +++++++++++++++++++++++++++------------------------- 2 files changed, 157 insertions(+), 152 deletions(-) diff --git a/commands.go b/commands.go index dc12935265..8671243f9f 100644 --- a/commands.go +++ b/commands.go @@ -492,64 +492,64 @@ func (srv *Server) CmdImport(stdin io.ReadCloser, stdout rcli.DockerConn, args . return nil } -func (srv *Server) CmdPush(stdin io.ReadCloser, stdout rcli.DockerConn, args ...string) error { - cmd := rcli.Subcmd(stdout, "push", "NAME", "Push an image or a repository to the registry") - if err := cmd.Parse(args); err != nil { - return nil - } - local := cmd.Arg(0) +// func (srv *Server) CmdPush(stdin io.ReadCloser, stdout rcli.DockerConn, args ...string) error { +// cmd := rcli.Subcmd(stdout, "push", "NAME", "Push an image or a repository to the registry") +// if err := cmd.Parse(args); err != nil { +// return nil +// } +// local := cmd.Arg(0) - if local == "" { - cmd.Usage() - return nil - } +// if local == "" { +// cmd.Usage() +// return nil +// } - // If the login failed, abort - if srv.runtime.authConfig == nil || srv.runtime.authConfig.Username == "" { - if err := srv.CmdLogin(stdin, stdout, args...); err != nil { - return err - } - if srv.runtime.authConfig == nil || srv.runtime.authConfig.Username == "" { - return fmt.Errorf("Please login prior to push. ('docker login')") - } - } +// // If the login failed, abort +// if srv.runtime.authConfig == nil || srv.runtime.authConfig.Username == "" { +// if err := srv.CmdLogin(stdin, stdout, args...); err != nil { +// return err +// } +// if srv.runtime.authConfig == nil || srv.runtime.authConfig.Username == "" { +// return fmt.Errorf("Please login prior to push. ('docker login')") +// } +// } - var remote string +// var remote string - tmp := strings.SplitN(local, "/", 2) - if len(tmp) == 1 { - return fmt.Errorf( - "Impossible to push a \"root\" repository. Please rename your repository in / (ex: %s/%s)", - srv.runtime.authConfig.Username, local) - } else { - remote = local - } +// tmp := strings.SplitN(local, "/", 2) +// if len(tmp) == 1 { +// return fmt.Errorf( +// "Impossible to push a \"root\" repository. Please rename your repository in / (ex: %s/%s)", +// srv.runtime.authConfig.Username, local) +// } else { +// remote = local +// } - Debugf("Pushing [%s] to [%s]\n", local, remote) +// Debugf("Pushing [%s] to [%s]\n", local, remote) - // Try to get the image - // FIXME: Handle lookup - // FIXME: Also push the tags in case of ./docker push myrepo:mytag - // img, err := srv.runtime.LookupImage(cmd.Arg(0)) - img, err := srv.runtime.graph.Get(local) - if err != nil { - Debugf("The push refers to a repository [%s] (len: %d)\n", local, len(srv.runtime.repositories.Repositories[local])) - // If it fails, try to get the repository - if localRepo, exists := srv.runtime.repositories.Repositories[local]; exists { - if err := srv.runtime.graph.PushRepository(stdout, remote, localRepo, srv.runtime.authConfig); err != nil { - return err - } - return nil - } +// // Try to get the image +// // FIXME: Handle lookup +// // FIXME: Also push the tags in case of ./docker push myrepo:mytag +// // img, err := srv.runtime.LookupImage(cmd.Arg(0)) +// img, err := srv.runtime.graph.Get(local) +// if err != nil { +// Debugf("The push refers to a repository [%s] (len: %d)\n", local, len(srv.runtime.repositories.Repositories[local])) +// // If it fails, try to get the repository +// if localRepo, exists := srv.runtime.repositories.Repositories[local]; exists { +// if err := srv.runtime.graph.PushRepository(stdout, remote, localRepo, srv.runtime.authConfig); err != nil { +// return err +// } +// return nil +// } - return err - } - err = srv.runtime.graph.PushImage(stdout, img, srv.runtime.authConfig) - if err != nil { - return err - } - return nil -} +// return err +// } +// err = srv.runtime.graph.PushImage(stdout, img, srv.runtime.authConfig) +// if err != nil { +// return err +// } +// return nil +// } func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "pull", "NAME", "Pull an image or a repository from the registry") diff --git a/registry.go b/registry.go index 07e79a5a5f..edaaeed502 100644 --- a/registry.go +++ b/registry.go @@ -139,6 +139,79 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, tok return img, ProgressReader(res.Body, int(res.ContentLength), stdout, "Downloading %v/%v (%v)"), nil } +func (graph *Graph) getRemoteTags(stdout io.Writer, registries []string, repository string, token []string) (map[string]string, error) { + client := &http.Client{} + for _, host := range registries { + endpoint := "https://" + host + "/v1/repositories/users/" + repository + "/tags" + req, err := http.NewRequest("GET", endpoint, nil) + if err != nil { + return nil, err + } + req.Header["X-Docker-Token"] = token + res, err := client.Do(req) + defer res.Body.Close() + if err != nil || (res.StatusCode != 200 && res.StatusCode != 404) { + Debugf("Registry isn't responding: trying another registry endpoint") + continue + } else if res.StatusCode == 404 { + return nil, fmt.Errorf("Repository not found") + } + + var result *map[string]string + + rawJson, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + if err = json.Unmarshal(rawJson, result); err != nil { + return nil, err + } + + return *result, nil + + } + return nil, fmt.Errorf("Could not reach any registry endpoint") +} + +func (graph *Graph) getImageForTag(stdout io.Writer, tag, remote, registry string, token []string) (string, error) { + client := &http.Client{} + registryEndpoint := "https://" + registry + "/v1" + var repositoryTarget string + if strings.Index(remote, "/") == -1 { + repositoryTarget = registryEndpoint + "/repositories/library/" + + remote + "/tags/" + tag + } else { + repositoryTarget = registryEndpoint + "/repositories/users/" + + remote + "/tags/" + tag + } + + req, err := http.NewRequest("GET", repositoryTarget, nil) + if err != nil { + return "", err + } + req.Header["X-Docker-Token"] = token + res, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("Error while retrieving repository info: %v", err) + } + defer res.Body.Close() + if res.StatusCode == 403 { + return "", fmt.Errorf("You aren't authorized to access this resource") + } else if res.StatusCode != 200 { + return "", fmt.Errorf("HTTP code: %d", res.StatusCode) + } + + var imgId string + rawJson, err := ioutil.ReadAll(res.Body) + if err != nil { + return "", err + } + if err = json.Unmarshal(rawJson, &imgId); err != nil { + return "", err + } + return imgId, nil +} + func (graph *Graph) PullImage(stdout io.Writer, imgId, registry string, token []string) error { history, err := graph.getRemoteHistory(imgId, registry, token) if err != nil { @@ -161,56 +234,6 @@ func (graph *Graph) PullImage(stdout io.Writer, imgId, registry string, token [] return nil } -// // FIXME: Handle the askedTag parameter -// func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag, registry string, repositories *TagStore, authConfig *auth.AuthConfig) error { -// client := &http.Client{} - -// fmt.Fprintf(stdout, "Pulling repository %s\r\n", remote) - -// var repositoryTarget string -// // If we are asking for 'root' repository, lookup on the Library's registry -// if strings.Index(remote, "/") == -1 { -// repositoryTarget = registry + "/library/" + remote -// } else { -// repositoryTarget = registry + "/users/" + remote -// } - -// req, err := http.NewRequest("GET", repositoryTarget, nil) -// if err != nil { -// return err -// } -// req.SetBasicAuth(authConfig.Username, authConfig.Password) -// res, err := client.Do(req) -// if err != nil { -// return err -// } -// defer res.Body.Close() -// if res.StatusCode != 200 { -// return fmt.Errorf("HTTP code: %d", res.StatusCode) -// } -// rawJson, err := ioutil.ReadAll(res.Body) -// if err != nil { -// return err -// } -// t := map[string]string{} -// if err = json.Unmarshal(rawJson, &t); err != nil { -// return err -// } -// for tag, rev := range t { -// fmt.Fprintf(stdout, "Pulling tag %s:%s\r\n", remote, tag) -// if err = graph.PullImage(stdout, rev, registry, authConfig); err != nil { -// return err -// } -// if err = repositories.Set(remote, tag, rev, true); err != nil { -// return err -// } -// } -// if err = repositories.Save(); err != nil { -// return err -// } -// return nil -// } - func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, repositories *TagStore, authConfig *auth.AuthConfig) error { client := &http.Client{} @@ -253,66 +276,48 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re } // FIXME: If askedTag is empty, fetch all tags. + var tagsList map[string]string if askedTag == "" { - askedTag = "latest" - } - - for _, registry := range endpoints { - registryEndpoint := "https://" + registry + "/v1" - if strings.Index(remote, "/") == -1 { - repositoryTarget = registryEndpoint + "/repositories/library/" + - remote + "/tags/" + askedTag - } else { - repositoryTarget = registryEndpoint + "/repositories/users/" + - remote + "/tags/" + askedTag - } - - req, err = http.NewRequest("GET", repositoryTarget, nil) + tagsList, err = graph.getRemoteTags(stdout, endpoints, remote, token) if err != nil { return err } - req.Header["X-Docker-Token"] = token - res, err := client.Do(req) - if err != nil { - fmt.Fprintf(stdout, "Error while retrieving repository info: %v ; " + - "checking next endpoint") - continue - } - defer res.Body.Close() - if res.StatusCode == 403 { - if authConfig == nil { - return fmt.Errorf("You need to be authenticated to access this resource") - } else { - return fmt.Errorf("You aren't authorized to access this resource") + } else { + tagsList = map[string]string{ askedTag : "" } + } + + for askedTag, imgId := range tagsList { + success := false + for _, registry := range endpoints { + if imgId == "" { + imgId, err = graph.getImageForTag(stdout, askedTag, remote, registry, token) + if err != nil { + fmt.Fprintf(stdout, "Error while retrieving image for tag: %v (%v) ; " + + "checking next endpoint", askedTag, err) + continue + } } - } else if res.StatusCode != 200 { - return fmt.Errorf("HTTP code: %d", res.StatusCode) + + if err := graph.PullImage(stdout, imgId, "https://" + registry + "/v1", token); err != nil { + return err + } + + if err = repositories.Set(remote, askedTag, imgId, true); err != nil { + return err + } + success = true } - var imgId string - rawJson, err := ioutil.ReadAll(res.Body) - if err != nil { - return err + if !success { + return fmt.Errorf("Could not find repository on any of the indexed registries.") } - if err = json.Unmarshal(rawJson, &imgId); err != nil { - return err - } - - if err := graph.PullImage(stdout, imgId, registryEndpoint, token); err != nil { - return err - } - - if err = repositories.Set(remote, askedTag, imgId, true); err != nil { - return err - } - - if err = repositories.Save(); err != nil { - return err - } - - return nil } - return fmt.Errorf("Could not find repository on any of the indexed registries.") + + if err = repositories.Save(); err != nil { + return err + } + + return nil } // Push a local image to the registry with its history if needed From 1cf8a2c26c8ae663dea9a613dfb236e6c2257550 Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 18 Apr 2013 08:16:58 -0700 Subject: [PATCH 11/97] Changed some of the routes to reflect changes made to the API ; added HTTPClient singleton to the graph object --- graph.go | 3 +- registry.go | 97 ++++++++++++++++------------------------------------- 2 files changed, 30 insertions(+), 70 deletions(-) diff --git a/graph.go b/graph.go index 3823868c83..1b2c441974 100644 --- a/graph.go +++ b/graph.go @@ -2,7 +2,7 @@ package docker import ( "fmt" - "io" + "net/http" "io/ioutil" "os" "path" @@ -15,6 +15,7 @@ import ( type Graph struct { Root string idIndex *TruncIndex + httpClient *http.Client } // NewGraph instantiates a new graph at the given root path in the filesystem. diff --git a/registry.go b/registry.go index edaaeed502..aacdf8357a 100644 --- a/registry.go +++ b/registry.go @@ -49,7 +49,7 @@ func NewMultipleImgJson(src []byte) ([]*Image, error) { // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([]*Image, error) { - client := &http.Client{} + client := graph.getHttpClient() req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/ancestry", nil) if err != nil { @@ -78,6 +78,13 @@ func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([] return history, nil } +func (graph *Graph) getHttpClient() *http.Client { + if graph.httpClient == nil { + graph.httpClient = new(http.Client) + } + return graph.httpClient +} + // Check if an image exists in the Registry func (graph *Graph) LookupRemoteImage(imgId, registry string, authConfig *auth.AuthConfig) bool { rt := &http.Transport{Proxy: http.ProxyFromEnvironment} @@ -94,7 +101,7 @@ func (graph *Graph) LookupRemoteImage(imgId, registry string, authConfig *auth.A // Retrieve an image from the Registry. // Returns the Image object as well as the layer as an Archive (io.Reader) func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, token []string) (*Image, Archive, error) { - client := &http.Client{} + client := graph.getHttpClient() fmt.Fprintf(stdout, "Pulling %s metadata\r\n", imgId) // Get the Json @@ -140,9 +147,9 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, tok } func (graph *Graph) getRemoteTags(stdout io.Writer, registries []string, repository string, token []string) (map[string]string, error) { - client := &http.Client{} + client := graph.getHttpClient() for _, host := range registries { - endpoint := "https://" + host + "/v1/repositories/users/" + repository + "/tags" + endpoint := "https://" + host + "/v1/repositories/" + repository + "/tags" req, err := http.NewRequest("GET", endpoint, nil) if err != nil { return nil, err @@ -174,16 +181,9 @@ func (graph *Graph) getRemoteTags(stdout io.Writer, registries []string, reposit } func (graph *Graph) getImageForTag(stdout io.Writer, tag, remote, registry string, token []string) (string, error) { - client := &http.Client{} + client := graph.getHttpClient() registryEndpoint := "https://" + registry + "/v1" - var repositoryTarget string - if strings.Index(remote, "/") == -1 { - repositoryTarget = registryEndpoint + "/repositories/library/" + - remote + "/tags/" + tag - } else { - repositoryTarget = registryEndpoint + "/repositories/users/" + - remote + "/tags/" + tag - } + repositoryTarget := registryEndpoint + "/repositories/" + remote + "/tags/" + tag req, err := http.NewRequest("GET", repositoryTarget, nil) if err != nil { @@ -235,16 +235,10 @@ func (graph *Graph) PullImage(stdout io.Writer, imgId, registry string, token [] } func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, repositories *TagStore, authConfig *auth.AuthConfig) error { - client := &http.Client{} + client := graph.getHttpClient() fmt.Fprintf(stdout, "Pulling repository %s\r\n", remote) - var repositoryTarget string - // If we are asking for 'root' repository, lookup on the Library's registry - if strings.Index(remote, "/") == -1 { - repositoryTarget = INDEX_ENDPOINT + "/repositories/library/" + remote + "/checksums" - } else { - repositoryTarget = INDEX_ENDPOINT + "/repositories/" + remote + "/checksums" - } + repositoryTarget := INDEX_ENDPOINT + "/repositories/" + remote + "/checksums" req, err := http.NewRequest("GET", repositoryTarget, nil) if err != nil { @@ -322,7 +316,7 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re // Push a local image to the registry with its history if needed func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, authConfig *auth.AuthConfig) error { - client := &http.Client{} + client := graph.getHttpClient() // FIXME: Factorize the code // FIXME: Do the puts in goroutines @@ -424,7 +418,7 @@ func (graph *Graph) pushTag(remote, revision, tag, registry string, authConfig * Debugf("Pushing tags for rev [%s] on {%s}\n", revision, registry+"/users/"+remote+"/"+tag) - client := &http.Client{} + client := graph.getHttpClient() req, err := http.NewRequest("PUT", registry+"/users/"+remote+"/"+tag, strings.NewReader(revision)) req.Header.Add("Content-type", "application/json") req.SetBasicAuth(authConfig.Username, authConfig.Password) @@ -446,41 +440,6 @@ func (graph *Graph) pushTag(remote, revision, tag, registry string, authConfig * return nil } -func (graph *Graph) LookupRemoteRepository(remote, registry string, authConfig *auth.AuthConfig) bool { - rt := &http.Transport{Proxy: http.ProxyFromEnvironment} - - var repositoryTarget string - // If we are asking for 'root' repository, lookup on the Library's registry - if strings.Index(remote, "/") == -1 { - repositoryTarget = registry + "/library/" + remote + "/lookup" - } else { - parts := strings.Split(remote, "/") - if len(parts) != 2 { - Debugf("Repository must abide to following format: user/repo_name") - return false - } - repositoryTarget = registry + "/users/" + parts[0] + "/repositories/" + parts[1] - } - Debugf("Checking for permissions on: %s", repositoryTarget) - req, err := http.NewRequest("PUT", repositoryTarget, strings.NewReader("\"\"")) - if err != nil { - Debugf("%s\n", err) - return false - } - req.SetBasicAuth(authConfig.Username, authConfig.Password) - req.Header.Add("Content-type", "application/json") - res, err := rt.RoundTrip(req) - if err != nil || res.StatusCode != 404 { - errBody, err := ioutil.ReadAll(res.Body) - if err != nil { - errBody = []byte(err.Error()) - } - Debugf("Lookup status code: %d (body: %s)", res.StatusCode, errBody) - return false - } - return true -} - // FIXME: this should really be PushTag func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId, registry string, authConfig *auth.AuthConfig) error { // Check if the local impage exists @@ -506,17 +465,17 @@ func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId, registry // Remote has the format '/ func (graph *Graph) PushRepository(stdout io.Writer, remote, registry string, localRepo Repository, authConfig *auth.AuthConfig) error { // Check if the remote repository exists/if we have the permission - if !graph.LookupRemoteRepository(remote, registry, authConfig) { - return fmt.Errorf("Permission denied on repository %s\n", remote) - } + // if !graph.LookupRemoteRepository(remote, registry, authConfig) { + // return fmt.Errorf("Permission denied on repository %s\n", remote) + // } - fmt.Fprintf(stdout, "Pushing repository %s (%d tags)\r\n", remote, len(localRepo)) - // For each image within the repo, push them - for tag, imgId := range localRepo { - if err := graph.pushPrimitive(stdout, remote, tag, imgId, registry, authConfig); err != nil { - // FIXME: Continue on error? - return err - } - } + // fmt.Fprintf(stdout, "Pushing repository %s (%d tags)\r\n", remote, len(localRepo)) + // // For each image within the repo, push them + // for tag, imgId := range localRepo { + // if err := graph.pushPrimitive(stdout, remote, tag, imgId, registry, authConfig); err != nil { + // // FIXME: Continue on error? + // return err + // } + // } return nil } From 7c1a27e2add585f22a833020dd0551a892b3af69 Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 18 Apr 2013 08:34:43 -0700 Subject: [PATCH 12/97] gofmt pass --- graph.go | 6 +++--- registry.go | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/graph.go b/graph.go index 1b2c441974..ea0c42b3d2 100644 --- a/graph.go +++ b/graph.go @@ -2,8 +2,8 @@ package docker import ( "fmt" - "net/http" "io/ioutil" + "net/http" "os" "path" "path/filepath" @@ -13,8 +13,8 @@ import ( // A Graph is a store for versioned filesystem images and the relationship between them. type Graph struct { - Root string - idIndex *TruncIndex + Root string + idIndex *TruncIndex httpClient *http.Client } diff --git a/registry.go b/registry.go index aacdf8357a..6ae4a425d7 100644 --- a/registry.go +++ b/registry.go @@ -277,7 +277,7 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re return err } } else { - tagsList = map[string]string{ askedTag : "" } + tagsList = map[string]string{askedTag: ""} } for askedTag, imgId := range tagsList { @@ -286,13 +286,13 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re if imgId == "" { imgId, err = graph.getImageForTag(stdout, askedTag, remote, registry, token) if err != nil { - fmt.Fprintf(stdout, "Error while retrieving image for tag: %v (%v) ; " + - "checking next endpoint", askedTag, err) + fmt.Fprintf(stdout, "Error while retrieving image for tag: %v (%v) ; "+ + "checking next endpoint", askedTag, err) continue } } - if err := graph.PullImage(stdout, imgId, "https://" + registry + "/v1", token); err != nil { + if err := graph.PullImage(stdout, imgId, "https://"+registry+"/v1", token); err != nil { return err } From 048fd671ef107aa14f02e6fe2db4a4100c23d37f Mon Sep 17 00:00:00 2001 From: shin- Date: Fri, 19 Apr 2013 08:25:55 -0700 Subject: [PATCH 13/97] Implemented checksum computation on image creation (necessary for new push primitive) --- container.go | 13 +++++++++++++ graph.go | 10 ++++++---- image.go | 1 + 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/container.go b/container.go index bac0951da4..891ae750ae 100644 --- a/container.go +++ b/container.go @@ -1,6 +1,7 @@ package docker import ( + "crypto/sha256" "encoding/json" "fmt" "github.com/dotcloud/docker/rcli" @@ -695,6 +696,18 @@ func (container *Container) ExportRw() (Archive, error) { return Tar(container.rwPath(), Uncompressed) } +func (container *Container) RwChecksum() (string, error) { + h := sha256.New() + rwData, err := container.ExportRw() + if err != nil { + return "", err + } + if _, err := io.Copy(h, rwData); err != nil { + return "", err + } + return string(h.Sum(nil)), nil +} + func (container *Container) Export() (Archive, error) { if err := container.EnsureMounted(); err != nil { return nil, err diff --git a/graph.go b/graph.go index ea0c42b3d2..59a8cba8db 100644 --- a/graph.go +++ b/graph.go @@ -98,11 +98,13 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut img.Parent = container.Image img.Container = container.Id img.ContainerConfig = *container.Config - if config == nil { - if parentImage, err := graph.Get(container.Image); err == nil && parentImage != nil { - img.Config = parentImage.Config - } + // FIXME: If an image is pulled from a raw URL (not created from a container), + // its checksum will not be computed, which will cause a push to fail + checksum, err := container.RwChecksum() + if err != nil { + return nil, err } + img.Checksum = checksum } if err := graph.Register(layerData, img); err != nil { return nil, err diff --git a/image.go b/image.go index 09c0f8dcf6..8090e583e6 100644 --- a/image.go +++ b/image.go @@ -18,6 +18,7 @@ import ( type Image struct { Id string `json:"id"` Parent string `json:"parent,omitempty"` + Checksum string `json:"checksum,omitempty"` Comment string `json:"comment,omitempty"` Created time.Time `json:"created"` Container string `json:"container,omitempty"` From e179c66400eeafd732218e981c5aaccd5a0b72c4 Mon Sep 17 00:00:00 2001 From: shin- Date: Tue, 23 Apr 2013 09:52:46 -0700 Subject: [PATCH 14/97] Reimplemented docker pull for new registry API (command is still deactivated) --- graph.go | 27 ++++++++++ registry.go | 139 ++++++++++++++++++++++++++++++++-------------------- 2 files changed, 114 insertions(+), 52 deletions(-) diff --git a/graph.go b/graph.go index 59a8cba8db..361a70912f 100644 --- a/graph.go +++ b/graph.go @@ -292,3 +292,30 @@ func (graph *Graph) Heads() (map[string]*Image, error) { func (graph *Graph) imageRoot(id string) string { return path.Join(graph.Root, id) } + +func (graph *Graph) Checksums(repo Repository) ([]map[string]string, error) { + var checksums map[string]string + var result []map[string]string + for _, id := range repo { + img, err := graph.Get(id) + if err != nil { + return nil, err + } + err = img.WalkHistory(func(image *Image) error { + checksums[image.Id] = image.Checksum + return nil + }) + if err != nil { + return nil, err + } + } + i := 0 + for id, sum := range checksums { + result[i] = map[string]string{ + "id": id, + "checksum": sum, + } + i++ + } + return result, nil +} diff --git a/registry.go b/registry.go index 6ae4a425d7..c6179efd6d 100644 --- a/registry.go +++ b/registry.go @@ -1,6 +1,7 @@ package docker import ( + "bytes" "encoding/json" "fmt" "github.com/dotcloud/docker/auth" @@ -315,8 +316,9 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re } // Push a local image to the registry with its history if needed -func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, authConfig *auth.AuthConfig) error { +func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, token []string) error { client := graph.getHttpClient() + registry = "https://" + registry + "/v1" // FIXME: Factorize the code // FIXME: Do the puts in goroutines @@ -336,7 +338,7 @@ func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, return err } req.Header.Add("Content-type", "application/json") - req.SetBasicAuth(authConfig.Username, authConfig.Password) + req.Header["X-Docker-Token"] = token res, err := client.Do(req) if err != nil { return fmt.Errorf("Failed to upload metadata: %s", err) @@ -346,7 +348,7 @@ func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, switch res.StatusCode { case 204: // Case where the image is already on the Registry - // FIXME: Do not be silent? + fmt.Fprintf(stdout, "Image %s already uploaded ; skipping.", img.Id) return nil default: errBody, err := ioutil.ReadAll(res.Body) @@ -358,37 +360,26 @@ func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, } fmt.Fprintf(stdout, "Pushing %s fs layer\r\n", img.Id) - req2, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/layer", nil) - req2.SetBasicAuth(authConfig.Username, authConfig.Password) - res2, err := client.Do(req2) - if err != nil { - return fmt.Errorf("Registry returned error: %s", err) - } - res2.Body.Close() - if res2.StatusCode != 307 { - return fmt.Errorf("Registry returned unexpected HTTP status code %d, expected 307", res2.StatusCode) - } - url, err := res2.Location() - if err != nil || url == nil { - return fmt.Errorf("Failed to retrieve layer upload location: %s", err) - } - - // FIXME: stream the archive directly to the registry instead of buffering it on disk. This requires either: - // a) Implementing S3's proprietary streaming logic, or - // b) Stream directly to the registry instead of S3. - // I prefer option b. because it doesn't lock us into a proprietary cloud service. - tmpLayer, err := graph.TempLayerArchive(img.Id, Xz, stdout) + layerData2, err := Tar(path.Join(graph.Root, img.Id, "layer"), Xz) + tmp, err := ioutil.ReadAll(layerData2) if err != nil { return err } - defer os.Remove(tmpLayer.Name()) - req3, err := http.NewRequest("PUT", url.String(), ProgressReader(tmpLayer, int(tmpLayer.Size), stdout, "Uploading %v/%v (%v)")) + layerLength := len(tmp) + + layerData, err := Tar(path.Join(graph.Root, img.Id, "layer"), Xz) + if err != nil { + return fmt.Errorf("Failed to generate layer archive: %s", err) + } + req3, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/layer", + ProgressReader(layerData.(io.ReadCloser), layerLength, stdout)) if err != nil { return err } req3.ContentLength = int64(tmpLayer.Size) req3.TransferEncoding = []string{"none"} + req3.Header["X-Docker-Token"] = token res3, err := client.Do(req3) if err != nil { return fmt.Errorf("Failed to upload layer: %s", err) @@ -406,7 +397,7 @@ func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, // push a tag on the registry. // Remote has the format '/ -func (graph *Graph) pushTag(remote, revision, tag, registry string, authConfig *auth.AuthConfig) error { +func (graph *Graph) pushTag(remote, revision, tag, registry string, token []string) error { // Keep this for backward compatibility if tag == "" { @@ -415,13 +406,14 @@ func (graph *Graph) pushTag(remote, revision, tag, registry string, authConfig * // "jsonify" the string revision = "\"" + revision + "\"" + registry = "https://" + registry + "/v1" Debugf("Pushing tags for rev [%s] on {%s}\n", revision, registry+"/users/"+remote+"/"+tag) client := graph.getHttpClient() - req, err := http.NewRequest("PUT", registry+"/users/"+remote+"/"+tag, strings.NewReader(revision)) + req, err := http.NewRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) req.Header.Add("Content-type", "application/json") - req.SetBasicAuth(authConfig.Username, authConfig.Password) + req.Header["X-Docker-Token"] = token res, err := client.Do(req) if err != nil { return err @@ -430,32 +422,25 @@ func (graph *Graph) pushTag(remote, revision, tag, registry string, authConfig * if res.StatusCode != 200 && res.StatusCode != 201 { return fmt.Errorf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote) } - Debugf("Result of push tag: %d\n", res.StatusCode) - switch res.StatusCode { - default: - return fmt.Errorf("Error %d\n", res.StatusCode) - case 200: - case 201: - } return nil } // FIXME: this should really be PushTag -func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId, registry string, authConfig *auth.AuthConfig) error { +func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId, registry string, token []string) error { // Check if the local impage exists img, err := graph.Get(imgId) if err != nil { fmt.Fprintf(stdout, "Skipping tag %s:%s: %s does not exist\r\n", remote, tag, imgId) return nil } - fmt.Fprintf(stdout, "Pushing tag %s:%s\r\n", remote, tag) + fmt.Fprintf(stdout, "Pushing image %s:%s\r\n", remote, tag) // Push the image - if err = graph.PushImage(stdout, img, registry, authConfig); err != nil { + if err = graph.PushImage(stdout, img, registry, token); err != nil { return err } fmt.Fprintf(stdout, "Registering tag %s:%s\r\n", remote, tag) // And then the tag - if err = graph.pushTag(remote, imgId, registry, tag, authConfig); err != nil { + if err = graph.pushTag(remote, imgId, registry, tag, token); err != nil { return err } return nil @@ -463,19 +448,69 @@ func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId, registry // Push a repository to the registry. // Remote has the format '/ -func (graph *Graph) PushRepository(stdout io.Writer, remote, registry string, localRepo Repository, authConfig *auth.AuthConfig) error { - // Check if the remote repository exists/if we have the permission - // if !graph.LookupRemoteRepository(remote, registry, authConfig) { - // return fmt.Errorf("Permission denied on repository %s\n", remote) - // } +func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Repository, authConfig *auth.AuthConfig) error { + client := graph.getHttpClient() + + checksums, err := graph.Checksums(localRepo) + if err != nil { + return err + } + + req, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote, nil) + if err != nil { + return err + } + req.SetBasicAuth(authConfig.Username, authConfig.Password) + res, err := client.Do(req) + if err != nil { + return err + } + res.Body.Close() + if res.StatusCode != 200 && res.StatusCode != 201 { + return fmt.Errorf("Error: Status %d trying to push repository %s", res.StatusCode, remote) + } + + var token, endpoints []string + if res.Header.Get("X-Docker-Token") != "" { + token = res.Header["X-Docker-Token"] + } else { + return fmt.Errorf("Index response didn't contain an access token") + } + if res.Header.Get("X-Docker-Endpoints") != "" { + endpoints = res.Header["X-Docker-Endpoints"] + } else { + return fmt.Errorf("Index response didn't contain any endpoints") + } + + for _, registry := range endpoints { + fmt.Fprintf(stdout, "Pushing repository %s to %s (%d tags)\r\n", remote, registry, + len(localRepo)) + // For each image within the repo, push them + for tag, imgId := range localRepo { + if err := graph.pushPrimitive(stdout, remote, tag, imgId, registry, token); err != nil { + // FIXME: Continue on error? + return err + } + } + } + checksumsJson, err := json.Marshal(checksums) + if err != nil { + return err + } + req2, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote+"/images", bytes.NewBuffer(checksumsJson)) + if err != nil { + return err + } + req2.SetBasicAuth(authConfig.Username, authConfig.Password) + req2.Header["X-Docker-Endpoints"] = endpoints + res2, err := client.Do(req) + if err != nil { + return err + } + res2.Body.Close() + if res2.StatusCode != 204 { + return fmt.Errorf("Error: Status %d trying to push checksums %s", res.StatusCode, remote) + } - // fmt.Fprintf(stdout, "Pushing repository %s (%d tags)\r\n", remote, len(localRepo)) - // // For each image within the repo, push them - // for tag, imgId := range localRepo { - // if err := graph.pushPrimitive(stdout, remote, tag, imgId, registry, authConfig); err != nil { - // // FIXME: Continue on error? - // return err - // } - // } return nil } From 6644a3c78a91d2212cc6449f25499e59b42a4397 Mon Sep 17 00:00:00 2001 From: shin- Date: Tue, 23 Apr 2013 12:02:16 -0700 Subject: [PATCH 15/97] Reactivated CmdPush in commands.go --- commands.go | 103 ++++++++++++++++++++++++++-------------------------- graph.go | 2 +- 2 files changed, 53 insertions(+), 52 deletions(-) diff --git a/commands.go b/commands.go index 8671243f9f..df4746587d 100644 --- a/commands.go +++ b/commands.go @@ -492,64 +492,65 @@ func (srv *Server) CmdImport(stdin io.ReadCloser, stdout rcli.DockerConn, args . return nil } -// func (srv *Server) CmdPush(stdin io.ReadCloser, stdout rcli.DockerConn, args ...string) error { -// cmd := rcli.Subcmd(stdout, "push", "NAME", "Push an image or a repository to the registry") -// if err := cmd.Parse(args); err != nil { -// return nil -// } -// local := cmd.Arg(0) +func (srv *Server) CmdPush(stdin io.ReadCloser, stdout rcli.DockerConn, args ...string) error { + cmd := rcli.Subcmd(stdout, "push", "NAME", "Push an image or a repository to the registry") + registry := cmd.String("registry", "", "Registry host to push the image to") + if err := cmd.Parse(args); err != nil { + return nil + } + local := cmd.Arg(0) -// if local == "" { -// cmd.Usage() -// return nil -// } + if local == "" { + cmd.Usage() + return nil + } -// // If the login failed, abort -// if srv.runtime.authConfig == nil || srv.runtime.authConfig.Username == "" { -// if err := srv.CmdLogin(stdin, stdout, args...); err != nil { -// return err -// } -// if srv.runtime.authConfig == nil || srv.runtime.authConfig.Username == "" { -// return fmt.Errorf("Please login prior to push. ('docker login')") -// } -// } + // If the login failed AND we're using the index, abort + if *registry == "" && (srv.runtime.authConfig == nil || srv.runtime.authConfig.Username == "") { + if err := srv.CmdLogin(stdin, stdout, args...); err != nil { + return err + } + if srv.runtime.authConfig == nil || srv.runtime.authConfig.Username == "" { + return fmt.Errorf("Please login prior to push. ('docker login')") + } + } -// var remote string + var remote string -// tmp := strings.SplitN(local, "/", 2) -// if len(tmp) == 1 { -// return fmt.Errorf( -// "Impossible to push a \"root\" repository. Please rename your repository in / (ex: %s/%s)", -// srv.runtime.authConfig.Username, local) -// } else { -// remote = local -// } + tmp := strings.SplitN(local, "/", 2) + if len(tmp) == 1 { + return fmt.Errorf( + "Impossible to push a \"root\" repository. Please rename your repository in / (ex: %s/%s)", + srv.runtime.authConfig.Username, local) + } else { + remote = local + } -// Debugf("Pushing [%s] to [%s]\n", local, remote) + Debugf("Pushing [%s] to [%s]\n", local, remote) -// // Try to get the image -// // FIXME: Handle lookup -// // FIXME: Also push the tags in case of ./docker push myrepo:mytag -// // img, err := srv.runtime.LookupImage(cmd.Arg(0)) -// img, err := srv.runtime.graph.Get(local) -// if err != nil { -// Debugf("The push refers to a repository [%s] (len: %d)\n", local, len(srv.runtime.repositories.Repositories[local])) -// // If it fails, try to get the repository -// if localRepo, exists := srv.runtime.repositories.Repositories[local]; exists { -// if err := srv.runtime.graph.PushRepository(stdout, remote, localRepo, srv.runtime.authConfig); err != nil { -// return err -// } -// return nil -// } + // Try to get the image + // FIXME: Handle lookup + // FIXME: Also push the tags in case of ./docker push myrepo:mytag + // img, err := srv.runtime.LookupImage(cmd.Arg(0)) + img, err := srv.runtime.graph.Get(local) + if err != nil { + Debugf("The push refers to a repository [%s] (len: %d)\n", local, len(srv.runtime.repositories.Repositories[local])) + // If it fails, try to get the repository + if localRepo, exists := srv.runtime.repositories.Repositories[local]; exists { + if err := srv.runtime.graph.PushRepository(stdout, remote, localRepo, srv.runtime.authConfig); err != nil { + return err + } + return nil + } -// return err -// } -// err = srv.runtime.graph.PushImage(stdout, img, srv.runtime.authConfig) -// if err != nil { -// return err -// } -// return nil -// } + return err + } + err = srv.runtime.graph.PushImage(stdout, img, *registry, nil) + if err != nil { + return err + } + return nil +} func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "pull", "NAME", "Pull an image or a repository from the registry") diff --git a/graph.go b/graph.go index 361a70912f..f6965aa1cf 100644 --- a/graph.go +++ b/graph.go @@ -312,7 +312,7 @@ func (graph *Graph) Checksums(repo Repository) ([]map[string]string, error) { i := 0 for id, sum := range checksums { result[i] = map[string]string{ - "id": id, + "id": id, "checksum": sum, } i++ From 23953e7d67c83bad8514a262acec5ed20bbbfa94 Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 24 Apr 2013 09:11:29 -0700 Subject: [PATCH 16/97] Style changes in auth.Login --- auth/auth.go | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/auth/auth.go b/auth/auth.go index 904198da5f..4696231a93 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -3,7 +3,6 @@ package auth import ( "encoding/base64" "encoding/json" - "errors" "fmt" "io/ioutil" "net/http" @@ -16,7 +15,7 @@ import ( const CONFIGFILE = ".dockercfg" // the registry server we want to login against -const INDEX_SERVER = "https://index.docker.io" +const INDEX_SERVER = "https://indexstaging-docker.dotcloud.com" type AuthConfig struct { Username string `json:"username"` @@ -103,28 +102,24 @@ func Login(authConfig *AuthConfig) (string, error) { storeConfig := false reqStatusCode := 0 var status string - var errMsg string var reqBody []byte jsonBody, err := json.Marshal(authConfig) if err != nil { - errMsg = fmt.Sprintf("Config Error: %s", err) - return "", errors.New(errMsg) + return "", fmt.Errorf("Config Error: %s", err) } // using `bytes.NewReader(jsonBody)` here causes the server to respond with a 411 status. b := strings.NewReader(string(jsonBody)) req1, err := http.Post(INDEX_SERVER+"/v1/users", "application/json; charset=utf-8", b) if err != nil { - errMsg = fmt.Sprintf("Server Error: %s", err) - return "", errors.New(errMsg) + return "", fmt.Errorf("Server Error: %s", err) } reqStatusCode = req1.StatusCode defer req1.Body.Close() reqBody, err = ioutil.ReadAll(req1.Body) if err != nil { - errMsg = fmt.Sprintf("Server Error: [%#v] %s", reqStatusCode, err) - return "", errors.New(errMsg) + return "", fmt.Errorf("Server Error: [%#v] %s", reqStatusCode, err) } if reqStatusCode == 201 { @@ -149,16 +144,13 @@ func Login(authConfig *AuthConfig) (string, error) { status = "Login Succeeded\n" storeConfig = true } else { - status = fmt.Sprintf("Login: %s", body) - return "", errors.New(status) + return "", fmt.Errorf("Login: %s", body) } } else { - status = fmt.Sprintf("Registration: %s", reqBody) - return "", errors.New(status) + return "", fmt.Errorf("Registration: %s", reqBody) } } else { - status = fmt.Sprintf("[%s] : %s", reqStatusCode, reqBody) - return "", errors.New(status) + return "", fmt.Errorf("Unexpected status code [%d] : %s", reqStatusCode, reqBody) } if storeConfig { authStr := EncodeAuth(authConfig) From 84be35dce10682266b9a35d4156997c84485c769 Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 24 Apr 2013 12:15:34 -0700 Subject: [PATCH 17/97] Fixed docker login --- auth/auth.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/auth/auth.go b/auth/auth.go index 4696231a93..2d42667017 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -15,7 +15,7 @@ import ( const CONFIGFILE = ".dockercfg" // the registry server we want to login against -const INDEX_SERVER = "https://indexstaging-docker.dotcloud.com" +const INDEX_SERVER = "http://indexstaging-docker.dotcloud.com" type AuthConfig struct { Username string `json:"username"` @@ -100,6 +100,7 @@ func saveConfig(rootPath, authStr string, email string) error { // try to register/login to the registry server func Login(authConfig *AuthConfig) (string, error) { storeConfig := false + client := &http.Client{} reqStatusCode := 0 var status string var reqBody []byte @@ -110,11 +111,10 @@ func Login(authConfig *AuthConfig) (string, error) { // using `bytes.NewReader(jsonBody)` here causes the server to respond with a 411 status. b := strings.NewReader(string(jsonBody)) - req1, err := http.Post(INDEX_SERVER+"/v1/users", "application/json; charset=utf-8", b) + req1, err := http.Post(INDEX_SERVER+"/v1/users/", "application/json; charset=utf-8", b) if err != nil { return "", fmt.Errorf("Server Error: %s", err) } - reqStatusCode = req1.StatusCode defer req1.Body.Close() reqBody, err = ioutil.ReadAll(req1.Body) @@ -127,9 +127,8 @@ func Login(authConfig *AuthConfig) (string, error) { storeConfig = true } else if reqStatusCode == 400 { // FIXME: This should be 'exists', not 'exist'. Need to change on the server first. - if string(reqBody) == "Username or email already exist" { - client := &http.Client{} - req, err := http.NewRequest("GET", INDEX_SERVER+"/v1/users", nil) + if string(reqBody) == "\"Username or email already exist\"" { + req, err := http.NewRequest("GET", INDEX_SERVER+"/v1/users/", nil) req.SetBasicAuth(authConfig.Username, authConfig.Password) resp, err := client.Do(req) if err != nil { From 630d358384194489e834d86cb4ae2f6dcc6a0ee5 Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 25 Apr 2013 09:15:36 -0700 Subject: [PATCH 18/97] Fixed checksum representation --- container.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/container.go b/container.go index 891ae750ae..71686d840d 100644 --- a/container.go +++ b/container.go @@ -2,6 +2,7 @@ package docker import ( "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "github.com/dotcloud/docker/rcli" @@ -705,7 +706,7 @@ func (container *Container) RwChecksum() (string, error) { if _, err := io.Copy(h, rwData); err != nil { return "", err } - return string(h.Sum(nil)), nil + return hex.EncodeToString(h.Sum(nil)), nil } func (container *Container) Export() (Archive, error) { From 4cd9e4722c92f05438779d10bf18206f65417644 Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 25 Apr 2013 09:24:29 -0700 Subject: [PATCH 19/97] Fixed graph.Checksums() --- graph.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/graph.go b/graph.go index f6965aa1cf..a2a6f47af7 100644 --- a/graph.go +++ b/graph.go @@ -294,8 +294,8 @@ func (graph *Graph) imageRoot(id string) string { } func (graph *Graph) Checksums(repo Repository) ([]map[string]string, error) { - var checksums map[string]string var result []map[string]string + checksums := map[string]string{} for _, id := range repo { img, err := graph.Get(id) if err != nil { @@ -310,6 +310,7 @@ func (graph *Graph) Checksums(repo Repository) ([]map[string]string, error) { } } i := 0 + result = make([]map[string]string, len(checksums)) for id, sum := range checksums { result[i] = map[string]string{ "id": id, From 6e936c8fd32357e839cb9337e621b1e152b9fd8b Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 25 Apr 2013 09:36:05 -0700 Subject: [PATCH 20/97] Follow redirections when sending PUT request in PushRepository --- registry.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/registry.go b/registry.go index c6179efd6d..45a9d3bbfc 100644 --- a/registry.go +++ b/registry.go @@ -466,6 +466,20 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re return err } res.Body.Close() + for res.StatusCode >= 300 && res.StatusCode < 400 { + Debugf("Redirected to %s\n", res.Header.Get("Location")) + req, err = http.NewRequest("PUT", res.Header.Get("Location"), nil) + if err != nil { + return err + } + req.SetBasicAuth(authConfig.Username, authConfig.Password) + res, err = client.Do(req) + if err != nil { + return err + } + res.Body.Close() + } + if res.StatusCode != 200 && res.StatusCode != 201 { return fmt.Errorf("Error: Status %d trying to push repository %s", res.StatusCode, remote) } From ea3374bcb0b7a78ccbaeb9cd927fbaf259a5cf29 Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 25 Apr 2013 10:10:47 -0700 Subject: [PATCH 21/97] Prepend hash method to the image checksum --- container.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container.go b/container.go index 71686d840d..519137ff85 100644 --- a/container.go +++ b/container.go @@ -706,7 +706,7 @@ func (container *Container) RwChecksum() (string, error) { if _, err := io.Copy(h, rwData); err != nil { return "", err } - return hex.EncodeToString(h.Sum(nil)), nil + return "sha256:"+hex.EncodeToString(h.Sum(nil)), nil } func (container *Container) Export() (Archive, error) { From 3c85e9390e2d5fac418e03bac376eb1618487f3c Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 25 Apr 2013 10:12:41 -0700 Subject: [PATCH 22/97] Added X-Docker-Token header to initial index requests --- registry.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/registry.go b/registry.go index 45a9d3bbfc..af69c58adb 100644 --- a/registry.go +++ b/registry.go @@ -239,7 +239,7 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re client := graph.getHttpClient() fmt.Fprintf(stdout, "Pulling repository %s\r\n", remote) - repositoryTarget := INDEX_ENDPOINT + "/repositories/" + remote + "/checksums" + repositoryTarget := INDEX_ENDPOINT + "/repositories/" + remote + "/checksums/" req, err := http.NewRequest("GET", repositoryTarget, nil) if err != nil { @@ -248,6 +248,7 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re if authConfig != nil { req.SetBasicAuth(authConfig.Username, authConfig.Password) } + req.Header.Set("X-Docker-Token", "true") res, err := client.Do(req) if err != nil { @@ -461,6 +462,7 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re return err } req.SetBasicAuth(authConfig.Username, authConfig.Password) + req.Header.Set("X-Docker-Token", "true") res, err := client.Do(req) if err != nil { return err @@ -473,6 +475,7 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re return err } req.SetBasicAuth(authConfig.Username, authConfig.Password) + req.Header.Set("X-Docker-Token", "true") res, err = client.Do(req) if err != nil { return err @@ -488,6 +491,7 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re if res.Header.Get("X-Docker-Token") != "" { token = res.Header["X-Docker-Token"] } else { + Debugf("Response headers:\n %s\n", res.Header) return fmt.Errorf("Index response didn't contain an access token") } if res.Header.Get("X-Docker-Endpoints") != "" { From be7560890644759c0ddd229afed208c537f075d7 Mon Sep 17 00:00:00 2001 From: shin- Date: Fri, 26 Apr 2013 14:00:09 -0700 Subject: [PATCH 23/97] Fixed checksum computing. Ensure checksum is computed when image metadata is loaded from disk. Fixed docker push workflow. Moved hash computing to utils --- container.go | 10 +-- graph.go | 9 ++- image.go | 29 ++++++++ registry.go | 190 ++++++++++++++++++++++++++++----------------------- utils.go | 10 +++ 5 files changed, 155 insertions(+), 93 deletions(-) diff --git a/container.go b/container.go index 519137ff85..5040203227 100644 --- a/container.go +++ b/container.go @@ -1,8 +1,6 @@ package docker import ( - "crypto/sha256" - "encoding/hex" "encoding/json" "fmt" "github.com/dotcloud/docker/rcli" @@ -698,15 +696,11 @@ func (container *Container) ExportRw() (Archive, error) { } func (container *Container) RwChecksum() (string, error) { - h := sha256.New() - rwData, err := container.ExportRw() + rwData, err := Tar(container.rwPath(), Xz) if err != nil { return "", err } - if _, err := io.Copy(h, rwData); err != nil { - return "", err - } - return "sha256:"+hex.EncodeToString(h.Sum(nil)), nil + return HashData(rwData) } func (container *Container) Export() (Archive, error) { diff --git a/graph.go b/graph.go index a2a6f47af7..d7a53be7c3 100644 --- a/graph.go +++ b/graph.go @@ -81,6 +81,13 @@ func (graph *Graph) Get(name string) (*Image, error) { return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.Id) } img.graph = graph + + if img.Checksum == "" { + err := img.FixChecksum() + if err != nil { + return nil, err + } + } return img, nil } @@ -98,7 +105,7 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut img.Parent = container.Image img.Container = container.Id img.ContainerConfig = *container.Config - // FIXME: If an image is pulled from a raw URL (not created from a container), + // FIXME: If an image is imported from a raw URL (not created from a container), // its checksum will not be computed, which will cause a push to fail checksum, err := container.RwChecksum() if err != nil { diff --git a/image.go b/image.go index 8090e583e6..4d625d6c0f 100644 --- a/image.go +++ b/image.go @@ -52,6 +52,7 @@ func LoadImage(root string) (*Image, error) { } else if !stat.IsDir() { return nil, fmt.Errorf("Couldn't load image %s: %s is not a directory", img.Id, layerPath(root)) } + return &img, nil } @@ -258,3 +259,31 @@ func (img *Image) layer() (string, error) { } return layerPath(root), nil } + +func (img *Image) FixChecksum() error { + layer, err := img.layer() + if err != nil { + return err + } + layerData, err := Tar(layer, Xz) + if err != nil { + return err + } + sum, err := HashData(layerData) + if err != nil { + return err + } + img.Checksum = sum + jsonData, err := json.Marshal(img) + if err != nil { + return err + } + root, err := img.root() + if err != nil { + return err + } + if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil { + return err + } + return nil +} diff --git a/registry.go b/registry.go index af69c58adb..d9850843d1 100644 --- a/registry.go +++ b/registry.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "github.com/dotcloud/docker/auth" + "github.com/shin-/cookiejar" "io" "io/ioutil" "net/http" @@ -14,7 +15,6 @@ import ( ) //FIXME: Set the endpoint in a conf file or via commandline -//const INDEX_ENDPOINT = "http://registry-creack.dotcloud.com/v1" const INDEX_ENDPOINT = auth.INDEX_SERVER + "/v1" // Build an Image object from raw json data @@ -47,6 +47,13 @@ func NewMultipleImgJson(src []byte) ([]*Image, error) { return ret, nil } +func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { + for _, cookie := range c.Jar.Cookies(req.URL) { + req.AddCookie(cookie) + } + return c.Do(req) +} + // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([]*Image, error) { @@ -57,7 +64,6 @@ func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([] return nil, err } req.Header["X-Docker-Token"] = token - // req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := client.Do(req) if err != nil || res.StatusCode != 200 { if res != nil { @@ -82,7 +88,9 @@ func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([] func (graph *Graph) getHttpClient() *http.Client { if graph.httpClient == nil { graph.httpClient = new(http.Client) + graph.httpClient.Jar = cookiejar.NewCookieJar() } + Debugf("cookies: %v",graph.httpClient.Jar) return graph.httpClient } @@ -111,7 +119,6 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, tok return nil, nil, fmt.Errorf("Failed to download json: %s", err) } req.Header["X-Docker-Token"] = token - // req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err := client.Do(req) if err != nil { return nil, nil, fmt.Errorf("Failed to download json: %s", err) @@ -139,7 +146,6 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, tok return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err) } req.Header["X-Docker-Token"] = token - // req.SetBasicAuth(authConfig.Username, authConfig.Password) res, err = client.Do(req) if err != nil { return nil, nil, err @@ -271,7 +277,6 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re return fmt.Errorf("Index response didn't contain any endpoints") } - // FIXME: If askedTag is empty, fetch all tags. var tagsList map[string]string if askedTag == "" { tagsList, err = graph.getRemoteTags(stdout, endpoints, remote, token) @@ -316,86 +321,86 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re return nil } -// Push a local image to the registry with its history if needed -func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, token []string) error { +func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, token []string) error { + if parent, err := img.GetParent(); err != nil { + return err + } else if parent != nil { + if err := pushImageRec(graph, stdout, parent, registry, token); err != nil { + return err + } + } client := graph.getHttpClient() - registry = "https://" + registry + "/v1" + jsonRaw, err := ioutil.ReadFile(path.Join(graph.Root, img.Id, "json")) + if err != nil { + return fmt.Errorf("Error while retreiving the path for {%s}: %s", img.Id, err) + } - // FIXME: Factorize the code - // FIXME: Do the puts in goroutines - if err := imgOrig.WalkHistory(func(img *Image) error { + fmt.Fprintf(stdout, "Pushing %s metadata\r\n", img.Id) - jsonRaw, err := ioutil.ReadFile(path.Join(graph.Root, img.Id, "json")) - if err != nil { - return fmt.Errorf("Error while retreiving the path for {%s}: %s", img.Id, err) - } - - fmt.Fprintf(stdout, "Pushing %s metadata\r\n", img.Id) - - // FIXME: try json with UTF8 - jsonData := strings.NewReader(string(jsonRaw)) - req, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/json", jsonData) - if err != nil { - return err - } - req.Header.Add("Content-type", "application/json") - req.Header["X-Docker-Token"] = token - res, err := client.Do(req) - if err != nil { - return fmt.Errorf("Failed to upload metadata: %s", err) - } - defer res.Body.Close() - if res.StatusCode != 200 { - switch res.StatusCode { - case 204: - // Case where the image is already on the Registry - fmt.Fprintf(stdout, "Image %s already uploaded ; skipping.", img.Id) - return nil - default: - errBody, err := ioutil.ReadAll(res.Body) - if err != nil { - errBody = []byte(err.Error()) - } - return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) - } - } - - fmt.Fprintf(stdout, "Pushing %s fs layer\r\n", img.Id) - layerData2, err := Tar(path.Join(graph.Root, img.Id, "layer"), Xz) - tmp, err := ioutil.ReadAll(layerData2) - if err != nil { - return err - } - layerLength := len(tmp) - - layerData, err := Tar(path.Join(graph.Root, img.Id, "layer"), Xz) - if err != nil { - return fmt.Errorf("Failed to generate layer archive: %s", err) - } - req3, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/layer", - ProgressReader(layerData.(io.ReadCloser), layerLength, stdout)) - if err != nil { - return err - } - req3.ContentLength = int64(tmpLayer.Size) - - req3.TransferEncoding = []string{"none"} - req3.Header["X-Docker-Token"] = token - res3, err := client.Do(req3) - if err != nil { - return fmt.Errorf("Failed to upload layer: %s", err) - } - res3.Body.Close() - if res3.StatusCode != 200 { - return fmt.Errorf("Received HTTP code %d while uploading layer", res3.StatusCode) - } - return nil - }); err != nil { + // FIXME: try json with UTF8 + jsonData := strings.NewReader(string(jsonRaw)) + req, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/json", jsonData) + if err != nil { return err } + req.Header.Add("Content-type", "application/json") + req.Header.Set("Authorization", "Token " + strings.Join(token, ",")) + res, err := doWithCookies(client, req) + if err != nil { + return fmt.Errorf("Failed to upload metadata: %s", err) + } + defer res.Body.Close() + if len(res.Cookies()) > 0 { + client.Jar.SetCookies(req.URL, res.Cookies()) + } + if res.StatusCode != 200 { + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + errBody = []byte(err.Error()) + } + var jsonBody map[string]string + if err := json.Unmarshal(errBody, &jsonBody); err != nil { + errBody = []byte(err.Error()) + } else if jsonBody["error"] == "Image already exists" { + fmt.Fprintf(stdout, "Image %v already uploaded ; skipping\n", img.Id) + return nil + } + return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) + } + + fmt.Fprintf(stdout, "Pushing %s fs layer\r\n", img.Id) + + layerData, err := Tar(path.Join(graph.Root, img.Id, "layer"), Xz) + if err != nil { + return fmt.Errorf("Failed to generate layer archive: %s", err) + } + req3, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/layer", + layerData) + if err != nil { + return err + } + + req3.ContentLength = -1 + req3.TransferEncoding = []string{"chunked"} + req3.Header.Set("Authorization", "Token " + strings.Join(token, ",")) + fmt.Printf("%v", req3.Header) + res3, err := doWithCookies(client, req3) + if err != nil { + return fmt.Errorf("Failed to upload layer: %s", err) + } + res3.Body.Close() + if res3.StatusCode != 200 { + return fmt.Errorf("Received HTTP code %d while uploading layer", res3.StatusCode) + } return nil } +// Push a local image to the registry with its history if needed +func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, token []string) error { + registry = "https://" + registry + "/v1" + return pushImageRec(graph, stdout, imgOrig, registry, token) +} + // push a tag on the registry. // Remote has the format '/ func (graph *Graph) pushTag(remote, revision, tag, registry string, token []string) error { @@ -413,9 +418,13 @@ func (graph *Graph) pushTag(remote, revision, tag, registry string, token []stri client := graph.getHttpClient() req, err := http.NewRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) + if err != nil { + return err + } req.Header.Add("Content-type", "application/json") - req.Header["X-Docker-Token"] = token - res, err := client.Do(req) + req.Header.Set("Authorization", "Token " + strings.Join(token, ",")) + req.ContentLength = int64(len(revision)) + res, err := doWithCookies(client, req) if err != nil { return err } @@ -441,7 +450,7 @@ func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId, registry } fmt.Fprintf(stdout, "Registering tag %s:%s\r\n", remote, tag) // And then the tag - if err = graph.pushTag(remote, imgId, registry, tag, token); err != nil { + if err = graph.pushTag(remote, imgId, tag, registry, token); err != nil { return err } return nil @@ -453,15 +462,25 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re client := graph.getHttpClient() checksums, err := graph.Checksums(localRepo) + imgList := make([]map[string]string, len(checksums)) if err != nil { return err } - req, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote, nil) + for i, obj := range checksums { + imgList[i] = map[string]string{"id": obj["id"]} + } + imgListJson, err := json.Marshal(imgList) + if err != nil { + return err + } + + req, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote, bytes.NewReader(imgListJson)) if err != nil { return err } req.SetBasicAuth(authConfig.Username, authConfig.Password) + req.ContentLength = int64(len(imgListJson)) req.Header.Set("X-Docker-Token", "true") res, err := client.Do(req) if err != nil { @@ -470,11 +489,12 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re res.Body.Close() for res.StatusCode >= 300 && res.StatusCode < 400 { Debugf("Redirected to %s\n", res.Header.Get("Location")) - req, err = http.NewRequest("PUT", res.Header.Get("Location"), nil) + req, err = http.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJson)) if err != nil { return err } req.SetBasicAuth(authConfig.Username, authConfig.Password) + req.ContentLength = int64(len(imgListJson)) req.Header.Set("X-Docker-Token", "true") res, err = client.Do(req) if err != nil { @@ -490,6 +510,7 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re var token, endpoints []string if res.Header.Get("X-Docker-Token") != "" { token = res.Header["X-Docker-Token"] + Debugf("Auth token: %v", token) } else { Debugf("Response headers:\n %s\n", res.Header) return fmt.Errorf("Index response didn't contain an access token") @@ -515,13 +536,14 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re if err != nil { return err } - req2, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote+"/images", bytes.NewBuffer(checksumsJson)) + req2, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote+"/images", bytes.NewReader(checksumsJson)) if err != nil { return err } req2.SetBasicAuth(authConfig.Username, authConfig.Password) req2.Header["X-Docker-Endpoints"] = endpoints - res2, err := client.Do(req) + req2.ContentLength = int64(len(checksumsJson)) + res2, err := client.Do(req2) if err != nil { return err } diff --git a/utils.go b/utils.go index 297b798af8..bcd3e0d1b3 100644 --- a/utils.go +++ b/utils.go @@ -2,6 +2,8 @@ package docker import ( "bytes" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "github.com/dotcloud/docker/rcli" @@ -456,3 +458,11 @@ func FindCgroupMountpoint(cgroupType string) (string, error) { return "", fmt.Errorf("cgroup mountpoint not found for %s", cgroupType) } + +func HashData(src io.Reader) (string, error) { + h := sha256.New() + if _, err := io.Copy(h, src); err != nil { + return "", err + } + return "sha256:"+hex.EncodeToString(h.Sum(nil)), nil +} \ No newline at end of file From 1c76f91fc47637efff99ab7e5c5be8f4f84facb0 Mon Sep 17 00:00:00 2001 From: shin- Date: Mon, 29 Apr 2013 06:56:33 -0700 Subject: [PATCH 24/97] Fixed minor bugs in docker pull --- registry.go | 51 ++++++++++++++++----------------------------------- 1 file changed, 16 insertions(+), 35 deletions(-) diff --git a/registry.go b/registry.go index d9850843d1..5da811b123 100644 --- a/registry.go +++ b/registry.go @@ -29,24 +29,6 @@ func NewImgJson(src []byte) (*Image, error) { return ret, nil } -// Build an Image object list from a raw json data -// FIXME: Do this in "stream" mode -func NewMultipleImgJson(src []byte) ([]*Image, error) { - ret := []*Image{} - - dec := json.NewDecoder(strings.NewReader(string(src))) - for { - m := &Image{} - if err := dec.Decode(m); err == io.EOF { - break - } else if err != nil { - return nil, err - } - ret = append(ret, m) - } - return ret, nil -} - func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { for _, cookie := range c.Jar.Cookies(req.URL) { req.AddCookie(cookie) @@ -56,14 +38,14 @@ func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) -func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([]*Image, error) { +func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([]string, error) { client := graph.getHttpClient() req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/ancestry", nil) if err != nil { return nil, err } - req.Header["X-Docker-Token"] = token + req.Header.Set("Authorization", "Token " + strings.Join(token, ", ")) res, err := client.Do(req) if err != nil || res.StatusCode != 200 { if res != nil { @@ -78,11 +60,12 @@ func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([] return nil, fmt.Errorf("Error while reading the http response: %s\n", err) } - history, err := NewMultipleImgJson(jsonString) - if err != nil { - return nil, fmt.Errorf("Error while parsing the json: %s\n", err) + Debugf("Ancestry: %s", jsonString) + history := new([]string) + if err := json.Unmarshal(jsonString, history); err != nil { + return nil, err } - return history, nil + return *history, nil } func (graph *Graph) getHttpClient() *http.Client { @@ -90,7 +73,6 @@ func (graph *Graph) getHttpClient() *http.Client { graph.httpClient = new(http.Client) graph.httpClient.Jar = cookiejar.NewCookieJar() } - Debugf("cookies: %v",graph.httpClient.Jar) return graph.httpClient } @@ -118,7 +100,7 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, tok if err != nil { return nil, nil, fmt.Errorf("Failed to download json: %s", err) } - req.Header["X-Docker-Token"] = token + req.Header.Set("Authorization", "Token " + strings.Join(token, ", ")) res, err := client.Do(req) if err != nil { return nil, nil, fmt.Errorf("Failed to download json: %s", err) @@ -145,7 +127,7 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, tok if err != nil { return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err) } - req.Header["X-Docker-Token"] = token + req.Header.Set("Authorization", "Token " + strings.Join(token, ", ")) res, err = client.Do(req) if err != nil { return nil, nil, err @@ -161,7 +143,7 @@ func (graph *Graph) getRemoteTags(stdout io.Writer, registries []string, reposit if err != nil { return nil, err } - req.Header["X-Docker-Token"] = token + req.Header.Set("Authorization", "Token " + strings.Join(token, ", ")) res, err := client.Do(req) defer res.Body.Close() if err != nil || (res.StatusCode != 200 && res.StatusCode != 404) { @@ -171,7 +153,7 @@ func (graph *Graph) getRemoteTags(stdout io.Writer, registries []string, reposit return nil, fmt.Errorf("Repository not found") } - var result *map[string]string + result := new(map[string]string) rawJson, err := ioutil.ReadAll(res.Body) if err != nil { @@ -196,7 +178,7 @@ func (graph *Graph) getImageForTag(stdout io.Writer, tag, remote, registry strin if err != nil { return "", err } - req.Header["X-Docker-Token"] = token + req.Header.Set("Authorization", "Token " + strings.Join(token, ", ")) res, err := client.Do(req) if err != nil { return "", fmt.Errorf("Error while retrieving repository info: %v", err) @@ -226,9 +208,9 @@ func (graph *Graph) PullImage(stdout io.Writer, imgId, registry string, token [] } // FIXME: Try to stream the images? // FIXME: Launch the getRemoteImage() in goroutines - for _, j := range history { - if !graph.Exists(j.Id) { - img, layer, err := graph.getRemoteImage(stdout, j.Id, registry, token) + for _, id := range history { + if !graph.Exists(id) { + img, layer, err := graph.getRemoteImage(stdout, id, registry, token) if err != nil { // FIXME: Keep goging in case of error? return err @@ -245,7 +227,7 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re client := graph.getHttpClient() fmt.Fprintf(stdout, "Pulling repository %s\r\n", remote) - repositoryTarget := INDEX_ENDPOINT + "/repositories/" + remote + "/checksums/" + repositoryTarget := INDEX_ENDPOINT + "/repositories/" + remote + "/images" req, err := http.NewRequest("GET", repositoryTarget, nil) if err != nil { @@ -512,7 +494,6 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re token = res.Header["X-Docker-Token"] Debugf("Auth token: %v", token) } else { - Debugf("Response headers:\n %s\n", res.Header) return fmt.Errorf("Index response didn't contain an access token") } if res.Header.Get("X-Docker-Endpoints") != "" { From e81a53eea9d5a44c3fde8d3060845ca18e95b4c4 Mon Sep 17 00:00:00 2001 From: shin- Date: Tue, 30 Apr 2013 06:55:24 -0700 Subject: [PATCH 25/97] Added support for REPO:TAG format in docker pull (overrides -t option) --- commands.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/commands.go b/commands.go index df4746587d..241923cef7 100644 --- a/commands.go +++ b/commands.go @@ -565,6 +565,12 @@ func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string return nil } + if strings.Contains(remote, ":") { + remoteParts := strings.Split(remote, ":") + tag = &remoteParts[1] + remote = remoteParts[0] + } + // FIXME: CmdPull should be a wrapper around Runtime.Pull() if *registry != "" { if err := srv.runtime.graph.PullImage(stdout, remote, *registry, nil); err != nil { @@ -572,7 +578,6 @@ func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string } return nil } - // FIXME: Allow pull repo:tag if err := srv.runtime.graph.PullRepository(stdout, remote, *tag, srv.runtime.repositories, srv.runtime.authConfig); err != nil { return err } From 6e2ddf6f608aa3d73b26b22dc3ff27f11534cdee Mon Sep 17 00:00:00 2001 From: shin- Date: Tue, 30 Apr 2013 10:01:19 -0700 Subject: [PATCH 26/97] Checksum system overhaul --- commands.go | 3 --- graph.go | 20 +++++------------- image.go | 60 +++++++++++++++++++++++++++++++++++++++-------------- registry.go | 14 ++++++++++--- 4 files changed, 60 insertions(+), 37 deletions(-) diff --git a/commands.go b/commands.go index 241923cef7..06443b559e 100644 --- a/commands.go +++ b/commands.go @@ -529,9 +529,6 @@ func (srv *Server) CmdPush(stdin io.ReadCloser, stdout rcli.DockerConn, args ... Debugf("Pushing [%s] to [%s]\n", local, remote) // Try to get the image - // FIXME: Handle lookup - // FIXME: Also push the tags in case of ./docker push myrepo:mytag - // img, err := srv.runtime.LookupImage(cmd.Arg(0)) img, err := srv.runtime.graph.Get(local) if err != nil { Debugf("The push refers to a repository [%s] (len: %d)\n", local, len(srv.runtime.repositories.Repositories[local])) diff --git a/graph.go b/graph.go index d7a53be7c3..6e443f206b 100644 --- a/graph.go +++ b/graph.go @@ -81,13 +81,6 @@ func (graph *Graph) Get(name string) (*Image, error) { return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.Id) } img.graph = graph - - if img.Checksum == "" { - err := img.FixChecksum() - if err != nil { - return nil, err - } - } return img, nil } @@ -105,17 +98,11 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut img.Parent = container.Image img.Container = container.Id img.ContainerConfig = *container.Config - // FIXME: If an image is imported from a raw URL (not created from a container), - // its checksum will not be computed, which will cause a push to fail - checksum, err := container.RwChecksum() - if err != nil { - return nil, err - } - img.Checksum = checksum } if err := graph.Register(layerData, img); err != nil { return nil, err } + img.Checksum() return img, nil } @@ -309,7 +296,10 @@ func (graph *Graph) Checksums(repo Repository) ([]map[string]string, error) { return nil, err } err = img.WalkHistory(func(image *Image) error { - checksums[image.Id] = image.Checksum + checksums[image.Id], err = image.Checksum() + if err != nil { + return err + } return nil }) if err != nil { diff --git a/image.go b/image.go index 4d625d6c0f..dcad130d3b 100644 --- a/image.go +++ b/image.go @@ -1,7 +1,9 @@ package docker import ( + "bytes" "crypto/rand" + "crypto/sha256" "encoding/hex" "encoding/json" "fmt" @@ -18,7 +20,6 @@ import ( type Image struct { Id string `json:"id"` Parent string `json:"parent,omitempty"` - Checksum string `json:"checksum,omitempty"` Comment string `json:"comment,omitempty"` Created time.Time `json:"created"` Container string `json:"container,omitempty"` @@ -260,30 +261,57 @@ func (img *Image) layer() (string, error) { return layerPath(root), nil } -func (img *Image) FixChecksum() error { +func (img *Image) Checksum() (string, error) { + root, err := img.root() + if err != nil { + return "", err + } + + checksumDictPth := path.Join(root, "..", "..", "checksums") + checksums := new(map[string]string) + + if checksumDict, err := ioutil.ReadFile(checksumDictPth); err == nil { + if err := json.Unmarshal(checksumDict, checksums); err != nil { + return "", err + } + if checksum, ok := (*checksums)[img.Id]; ok { + return checksum, nil + } + } + layer, err := img.layer() if err != nil { - return err + return "", err } layerData, err := Tar(layer, Xz) if err != nil { - return err + return "", err } - sum, err := HashData(layerData) + h := sha256.New() + if _, err := io.Copy(h, layerData); err != nil { + return "", err + } + + jsonData, err := ioutil.ReadFile(jsonPath(root)) if err != nil { - return err + return "", err } - img.Checksum = sum - jsonData, err := json.Marshal(img) + if _, err := io.Copy(h, bytes.NewBuffer(jsonData)); err != nil { + return "", err + } + + hash := "sha256:"+hex.EncodeToString(h.Sum(nil)) + if *checksums == nil { + *checksums = map[string]string{} + } + (*checksums)[img.Id] = hash + checksumJson, err := json.Marshal(checksums) if err != nil { - return err + return hash, err } - root, err := img.root() - if err != nil { - return err + + if err := ioutil.WriteFile(checksumDictPth, checksumJson, 0600); err != nil { + return hash, err } - if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil { - return err - } - return nil + return hash, nil } diff --git a/registry.go b/registry.go index 5da811b123..be9d37edc1 100644 --- a/registry.go +++ b/registry.go @@ -327,6 +327,12 @@ func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, t } req.Header.Add("Content-type", "application/json") req.Header.Set("Authorization", "Token " + strings.Join(token, ",")) + + checksum, err := img.Checksum() + if err != nil { + return fmt.Errorf("Error while retrieving checksum for %s: %v", img.Id, err) + } + req.Header.Set("X-Docker-Checksum", checksum) res, err := doWithCookies(client, req) if err != nil { return fmt.Errorf("Failed to upload metadata: %s", err) @@ -457,7 +463,7 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re return err } - req, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote, bytes.NewReader(imgListJson)) + req, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote+"/", bytes.NewReader(imgListJson)) if err != nil { return err } @@ -468,7 +474,7 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re if err != nil { return err } - res.Body.Close() + defer res.Body.Close() for res.StatusCode >= 300 && res.StatusCode < 400 { Debugf("Redirected to %s\n", res.Header.Get("Location")) req, err = http.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJson)) @@ -482,10 +488,12 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re if err != nil { return err } - res.Body.Close() + defer res.Body.Close() } if res.StatusCode != 200 && res.StatusCode != 201 { + info, err := ioutil.ReadAll(res.Body) + Debugf("%v %v", err, string(info)) return fmt.Errorf("Error: Status %d trying to push repository %s", res.StatusCode, remote) } From f10b0f75e02041db64176760a3be9639408ad923 Mon Sep 17 00:00:00 2001 From: shin- Date: Tue, 30 Apr 2013 14:05:33 -0700 Subject: [PATCH 27/97] Fix checksum computing --- image.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/image.go b/image.go index dcad130d3b..2142716d4f 100644 --- a/image.go +++ b/image.go @@ -283,22 +283,26 @@ func (img *Image) Checksum() (string, error) { if err != nil { return "", err } - layerData, err := Tar(layer, Xz) - if err != nil { - return "", err - } - h := sha256.New() - if _, err := io.Copy(h, layerData); err != nil { - return "", err - } - jsonData, err := ioutil.ReadFile(jsonPath(root)) if err != nil { return "", err } + + layerData, err := Tar(layer, Xz) + if err != nil { + return "", err + } + + h := sha256.New() if _, err := io.Copy(h, bytes.NewBuffer(jsonData)); err != nil { return "", err } + if _, err := io.Copy(h, strings.NewReader("\n")); err != nil { + return "", err + } + if _, err := io.Copy(h, layerData); err != nil { + return "", err + } hash := "sha256:"+hex.EncodeToString(h.Sum(nil)) if *checksums == nil { From b5873806d0f155b0fc16d224427f90548bfa8afe Mon Sep 17 00:00:00 2001 From: shin- Date: Tue, 30 Apr 2013 14:11:16 -0700 Subject: [PATCH 28/97] Only send checksums for images not uploaded yet --- registry.go | 62 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/registry.go b/registry.go index be9d37edc1..d44a6932b1 100644 --- a/registry.go +++ b/registry.go @@ -89,6 +89,37 @@ func (graph *Graph) LookupRemoteImage(imgId, registry string, authConfig *auth.A return err == nil && res.StatusCode == 307 } +func (graph *Graph) getImagesInRepository(repository string, authConfig *auth.AuthConfig) ([]map[string]string, error) { + u := INDEX_ENDPOINT+"/repositories/"+repository+"/images" + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return nil, err + } + req.SetBasicAuth(authConfig.Username, authConfig.Password) + res, err := graph.getHttpClient().Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + + // Repository doesn't exist yet + if res.StatusCode == 404 { + return nil, nil + } + + jsonData, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + imageList := []map[string]string{} + err = json.Unmarshal(jsonData, &imageList) + if err != nil { + return nil, err + } + + return imageList, nil +} + // Retrieve an image from the Registry. // Returns the Image object as well as the layer as an Archive (io.Reader) func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, token []string) (*Image, Archive, error) { @@ -362,6 +393,7 @@ func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, t if err != nil { return fmt.Errorf("Failed to generate layer archive: %s", err) } + req3, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/layer", layerData) if err != nil { @@ -451,13 +483,36 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re checksums, err := graph.Checksums(localRepo) imgList := make([]map[string]string, len(checksums)) + checksums2 := make([]map[string]string, len(checksums)) if err != nil { return err } - for i, obj := range checksums { - imgList[i] = map[string]string{"id": obj["id"]} + uploadedImages, err := graph.getImagesInRepository(remote, authConfig) + if err != nil { + return fmt.Errorf("Error occured while fetching the list") } + + + // Filter list to only send images/checksums not already uploaded + i := 0 + for _, obj := range checksums { + found := false + for _, uploadedImg := range uploadedImages { + if obj["id"] == uploadedImg["id"] && uploadedImg["checksum"] != "" { + found = true + break + } + } + if !found { + imgList[i] = map[string]string{"id": obj["id"]} + checksums2[i] = obj + i += 1 + } + } + checksums = checksums2[:i] + imgList = imgList[:i] + imgListJson, err := json.Marshal(imgList) if err != nil { return err @@ -492,8 +547,6 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re } if res.StatusCode != 200 && res.StatusCode != 201 { - info, err := ioutil.ReadAll(res.Body) - Debugf("%v %v", err, string(info)) return fmt.Errorf("Error: Status %d trying to push repository %s", res.StatusCode, remote) } @@ -525,6 +578,7 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re if err != nil { return err } + req2, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote+"/images", bytes.NewReader(checksumsJson)) if err != nil { return err From 19045b530eeacc97941cc1b63c2626f92df10f67 Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 1 May 2013 05:08:27 -0700 Subject: [PATCH 29/97] simplify graph.Checksums --- graph.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/graph.go b/graph.go index 6e443f206b..f51597df9a 100644 --- a/graph.go +++ b/graph.go @@ -297,10 +297,7 @@ func (graph *Graph) Checksums(repo Repository) ([]map[string]string, error) { } err = img.WalkHistory(func(image *Image) error { checksums[image.Id], err = image.Checksum() - if err != nil { - return err - } - return nil + return err }) if err != nil { return nil, err From be791a223b65d5641e032ea78ad7289b6eb58979 Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 1 May 2013 05:11:06 -0700 Subject: [PATCH 30/97] simplify image.Checksum --- image.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/image.go b/image.go index 2142716d4f..583aad0ba8 100644 --- a/image.go +++ b/image.go @@ -1,7 +1,6 @@ package docker import ( - "bytes" "crypto/rand" "crypto/sha256" "encoding/hex" @@ -294,10 +293,10 @@ func (img *Image) Checksum() (string, error) { } h := sha256.New() - if _, err := io.Copy(h, bytes.NewBuffer(jsonData)); err != nil { + if _, err := h.Write(jsonData); err != nil { return "", err } - if _, err := io.Copy(h, strings.NewReader("\n")); err != nil { + if _, err := h.Write([]byte("\n")); err != nil { return "", err } if _, err := io.Copy(h, layerData); err != nil { From 5690562fc855a7e813a7e3cfb3340f60bd626530 Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 1 May 2013 05:26:52 -0700 Subject: [PATCH 31/97] Fix error in PushImage --- registry.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/registry.go b/registry.go index d44a6932b1..e8fcd1fd11 100644 --- a/registry.go +++ b/registry.go @@ -375,7 +375,8 @@ func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, t if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - errBody = []byte(err.Error()) + return fmt.Errorf("HTTP code %d while uploading metadata and error when"+ + " trying to parse response body: %v", res.StatusCode, err) } var jsonBody map[string]string if err := json.Unmarshal(errBody, &jsonBody); err != nil { From 594827d41640eb6a6a878c29aca87f7a4ce6249c Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 1 May 2013 09:06:17 -0700 Subject: [PATCH 32/97] Fixed typo in 'username or email already exists' --- auth/auth.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/auth/auth.go b/auth/auth.go index 2d42667017..a653f5563e 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -126,8 +126,7 @@ func Login(authConfig *AuthConfig) (string, error) { status = "Account Created\n" storeConfig = true } else if reqStatusCode == 400 { - // FIXME: This should be 'exists', not 'exist'. Need to change on the server first. - if string(reqBody) == "\"Username or email already exist\"" { + if string(reqBody) == "\"Username or email already exists\"" { req, err := http.NewRequest("GET", INDEX_SERVER+"/v1/users/", nil) req.SetBasicAuth(authConfig.Username, authConfig.Password) resp, err := client.Do(req) From 18796d55a66f2f072349c54085516fe038bd8283 Mon Sep 17 00:00:00 2001 From: shin- Date: Wed, 1 May 2013 13:41:58 -0700 Subject: [PATCH 33/97] Fixed some login quirks --- auth/auth.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/auth/auth.go b/auth/auth.go index a653f5563e..f8e94884dd 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -15,7 +15,7 @@ import ( const CONFIGFILE = ".dockercfg" // the registry server we want to login against -const INDEX_SERVER = "http://indexstaging-docker.dotcloud.com" +const INDEX_SERVER = "https://indexstaging-docker.dotcloud.com" type AuthConfig struct { Username string `json:"username"` @@ -123,8 +123,12 @@ func Login(authConfig *AuthConfig) (string, error) { } if reqStatusCode == 201 { - status = "Account Created\n" + status = "Account created. Please use the confirmation link we sent"+ + " to your e-mail to activate it.\n" storeConfig = true + } else if reqStatusCode == 403 { + return "", fmt.Errorf("Login: Your account hasn't been activated. "+ + "Please check your e-mail for a confirmation link.") } else if reqStatusCode == 400 { if string(reqBody) == "\"Username or email already exists\"" { req, err := http.NewRequest("GET", INDEX_SERVER+"/v1/users/", nil) From 0f68042053ac652de13cd68086d0188ac0ecf9b6 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Wed, 1 May 2013 16:36:01 -0700 Subject: [PATCH 34/97] Handled wrong user credentials by re-init the auth file (it was impossible to login after having wrong crendentials) --- auth/auth.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/auth/auth.go b/auth/auth.go index f8e94884dd..b06086ff6a 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -75,6 +75,9 @@ func LoadConfig(rootPath string) (*AuthConfig, error) { return nil, err } arr := strings.Split(string(b), "\n") + if len(arr) < 2 { + return nil, fmt.Errorf("The Auth config file is empty") + } origAuth := strings.Split(arr[0], " = ") origEmail := strings.Split(arr[1], " = ") authConfig, err := DecodeAuth(origAuth[1]) @@ -88,9 +91,14 @@ func LoadConfig(rootPath string) (*AuthConfig, error) { // save the auth config func saveConfig(rootPath, authStr string, email string) error { + confFile := path.Join(rootPath, CONFIGFILE) + if len(email) == 0 { + os.Remove(confFile) + return nil + } lines := "auth = " + authStr + "\n" + "email = " + email + "\n" b := []byte(lines) - err := ioutil.WriteFile(path.Join(rootPath, CONFIGFILE), b, 0600) + err := ioutil.WriteFile(confFile, b, 0600) if err != nil { return err } @@ -145,8 +153,12 @@ func Login(authConfig *AuthConfig) (string, error) { if resp.StatusCode == 200 { status = "Login Succeeded\n" storeConfig = true + } else if resp.StatusCode == 401 { + saveConfig(authConfig.rootPath, "", "") + return "", fmt.Errorf("Wrong login/password, please try again") } else { - return "", fmt.Errorf("Login: %s", body) + return "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body, + resp.StatusCode, resp.Header) } } else { return "", fmt.Errorf("Registration: %s", reqBody) From 0c5e76958bbfd5c4d315d75262381cbae5c5f048 Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 2 May 2013 08:06:11 -0700 Subject: [PATCH 35/97] Use progress reader when uploading/downloading layers --- registry.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/registry.go b/registry.go index e8fcd1fd11..b5fd734bcd 100644 --- a/registry.go +++ b/registry.go @@ -390,13 +390,13 @@ func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, t fmt.Fprintf(stdout, "Pushing %s fs layer\r\n", img.Id) - layerData, err := Tar(path.Join(graph.Root, img.Id, "layer"), Xz) + layerData, err := graph.TempLayerArchive(img.Id, Xz, stdout) if err != nil { return fmt.Errorf("Failed to generate layer archive: %s", err) } req3, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/layer", - layerData) + ProgressReader(layerData, -1, stdout, "")) if err != nil { return err } @@ -404,7 +404,6 @@ func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, t req3.ContentLength = -1 req3.TransferEncoding = []string{"chunked"} req3.Header.Set("Authorization", "Token " + strings.Join(token, ",")) - fmt.Printf("%v", req3.Header) res3, err := doWithCookies(client, req3) if err != nil { return fmt.Errorf("Failed to upload layer: %s", err) @@ -491,7 +490,7 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re uploadedImages, err := graph.getImagesInRepository(remote, authConfig) if err != nil { - return fmt.Errorf("Error occured while fetching the list") + return fmt.Errorf("Error occured while fetching the list: %v", err) } From d985050aeb3f1411b94b21f6db35a24f9a1f93f2 Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 2 May 2013 08:18:33 -0700 Subject: [PATCH 36/97] gofmt pass --- image.go | 2 +- registry.go | 19 +++++++++---------- utils.go | 17 +++++++++-------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/image.go b/image.go index 583aad0ba8..bf86e2e7f7 100644 --- a/image.go +++ b/image.go @@ -303,7 +303,7 @@ func (img *Image) Checksum() (string, error) { return "", err } - hash := "sha256:"+hex.EncodeToString(h.Sum(nil)) + hash := "sha256:" + hex.EncodeToString(h.Sum(nil)) if *checksums == nil { *checksums = map[string]string{} } diff --git a/registry.go b/registry.go index b5fd734bcd..0aab3f003e 100644 --- a/registry.go +++ b/registry.go @@ -45,7 +45,7 @@ func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([] if err != nil { return nil, err } - req.Header.Set("Authorization", "Token " + strings.Join(token, ", ")) + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) res, err := client.Do(req) if err != nil || res.StatusCode != 200 { if res != nil { @@ -90,7 +90,7 @@ func (graph *Graph) LookupRemoteImage(imgId, registry string, authConfig *auth.A } func (graph *Graph) getImagesInRepository(repository string, authConfig *auth.AuthConfig) ([]map[string]string, error) { - u := INDEX_ENDPOINT+"/repositories/"+repository+"/images" + u := INDEX_ENDPOINT + "/repositories/" + repository + "/images" req, err := http.NewRequest("GET", u, nil) if err != nil { return nil, err @@ -131,7 +131,7 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, tok if err != nil { return nil, nil, fmt.Errorf("Failed to download json: %s", err) } - req.Header.Set("Authorization", "Token " + strings.Join(token, ", ")) + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) res, err := client.Do(req) if err != nil { return nil, nil, fmt.Errorf("Failed to download json: %s", err) @@ -158,7 +158,7 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, tok if err != nil { return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err) } - req.Header.Set("Authorization", "Token " + strings.Join(token, ", ")) + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) res, err = client.Do(req) if err != nil { return nil, nil, err @@ -174,7 +174,7 @@ func (graph *Graph) getRemoteTags(stdout io.Writer, registries []string, reposit if err != nil { return nil, err } - req.Header.Set("Authorization", "Token " + strings.Join(token, ", ")) + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) res, err := client.Do(req) defer res.Body.Close() if err != nil || (res.StatusCode != 200 && res.StatusCode != 404) { @@ -209,7 +209,7 @@ func (graph *Graph) getImageForTag(stdout io.Writer, tag, remote, registry strin if err != nil { return "", err } - req.Header.Set("Authorization", "Token " + strings.Join(token, ", ")) + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) res, err := client.Do(req) if err != nil { return "", fmt.Errorf("Error while retrieving repository info: %v", err) @@ -357,7 +357,7 @@ func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, t return err } req.Header.Add("Content-type", "application/json") - req.Header.Set("Authorization", "Token " + strings.Join(token, ",")) + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) checksum, err := img.Checksum() if err != nil { @@ -403,7 +403,7 @@ func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, t req3.ContentLength = -1 req3.TransferEncoding = []string{"chunked"} - req3.Header.Set("Authorization", "Token " + strings.Join(token, ",")) + req3.Header.Set("Authorization", "Token "+strings.Join(token, ",")) res3, err := doWithCookies(client, req3) if err != nil { return fmt.Errorf("Failed to upload layer: %s", err) @@ -442,7 +442,7 @@ func (graph *Graph) pushTag(remote, revision, tag, registry string, token []stri return err } req.Header.Add("Content-type", "application/json") - req.Header.Set("Authorization", "Token " + strings.Join(token, ",")) + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) req.ContentLength = int64(len(revision)) res, err := doWithCookies(client, req) if err != nil { @@ -493,7 +493,6 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re return fmt.Errorf("Error occured while fetching the list: %v", err) } - // Filter list to only send images/checksums not already uploaded i := 0 for _, obj := range checksums { diff --git a/utils.go b/utils.go index bcd3e0d1b3..78e13ed881 100644 --- a/utils.go +++ b/utils.go @@ -397,6 +397,15 @@ func CopyEscapable(dst io.Writer, src io.ReadCloser) (written int64, err error) return written, err } + +func HashData(src io.Reader) (string, error) { + h := sha256.New() + if _, err := io.Copy(h, src); err != nil { + return "", err + } + return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil +} + type KernelVersionInfo struct { Kernel int Major int @@ -457,12 +466,4 @@ func FindCgroupMountpoint(cgroupType string) (string, error) { } return "", fmt.Errorf("cgroup mountpoint not found for %s", cgroupType) -} - -func HashData(src io.Reader) (string, error) { - h := sha256.New() - if _, err := io.Copy(h, src); err != nil { - return "", err - } - return "sha256:"+hex.EncodeToString(h.Sum(nil)), nil } \ No newline at end of file From a372f982c1383a3b4d742ef4f4de1a645cd7434d Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Thu, 2 May 2013 15:39:44 -0700 Subject: [PATCH 37/97] Switching to prod index server --- auth/auth.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/auth/auth.go b/auth/auth.go index b06086ff6a..5a5987ace8 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -15,7 +15,7 @@ import ( const CONFIGFILE = ".dockercfg" // the registry server we want to login against -const INDEX_SERVER = "https://indexstaging-docker.dotcloud.com" +const INDEX_SERVER = "https://index.docker.io" type AuthConfig struct { Username string `json:"username"` @@ -131,11 +131,11 @@ func Login(authConfig *AuthConfig) (string, error) { } if reqStatusCode == 201 { - status = "Account created. Please use the confirmation link we sent"+ + status = "Account created. Please use the confirmation link we sent" + " to your e-mail to activate it.\n" storeConfig = true } else if reqStatusCode == 403 { - return "", fmt.Errorf("Login: Your account hasn't been activated. "+ + return "", fmt.Errorf("Login: Your account hasn't been activated. " + "Please check your e-mail for a confirmation link.") } else if reqStatusCode == 400 { if string(reqBody) == "\"Username or email already exists\"" { From bcdf03037b5f11d3760b45c96b9180f7b8da4b40 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Thu, 2 May 2013 18:52:23 -0700 Subject: [PATCH 38/97] Fixed pulling repositories from library --- registry.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/registry.go b/registry.go index 0aab3f003e..592b13da6e 100644 --- a/registry.go +++ b/registry.go @@ -168,6 +168,11 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, tok func (graph *Graph) getRemoteTags(stdout io.Writer, registries []string, repository string, token []string) (map[string]string, error) { client := graph.getHttpClient() + if strings.Count(repository, "/") == 0 { + // This will be removed once the Registry supports auto-resolution on + // the "library" namespace + repository = "library/" + repository + } for _, host := range registries { endpoint := "https://" + host + "/v1/repositories/" + repository + "/tags" req, err := http.NewRequest("GET", endpoint, nil) @@ -257,7 +262,7 @@ func (graph *Graph) PullImage(stdout io.Writer, imgId, registry string, token [] func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, repositories *TagStore, authConfig *auth.AuthConfig) error { client := graph.getHttpClient() - fmt.Fprintf(stdout, "Pulling repository %s\r\n", remote) + fmt.Fprintf(stdout, "Pulling repository %s from %s\r\n", remote, INDEX_ENDPOINT) repositoryTarget := INDEX_ENDPOINT + "/repositories/" + remote + "/images" req, err := http.NewRequest("GET", repositoryTarget, nil) From 3febeb93f553be0dceec81591a6e0ba578b42a12 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Thu, 2 May 2013 18:57:15 -0700 Subject: [PATCH 39/97] Added help message to invite to login when getting a 401 --- registry.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/registry.go b/registry.go index 592b13da6e..a471687609 100644 --- a/registry.go +++ b/registry.go @@ -279,6 +279,9 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re return err } defer res.Body.Close() + if res.StatusCode == 401 { + return fmt.Errorf("Please login first (HTTP code %d)", res.StatusCode) + } // TODO: Right now we're ignoring checksums in the response body. // In the future, we need to use them to check image validity. if res.StatusCode != 200 { From 00266df8ac64bb3b7e19818ad2dfb852dc0d2e7f Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Fri, 3 May 2013 15:37:28 -0700 Subject: [PATCH 40/97] Fixed public pull + Added some verbosity about what is happening --- registry.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/registry.go b/registry.go index a471687609..43738646db 100644 --- a/registry.go +++ b/registry.go @@ -269,7 +269,7 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re if err != nil { return err } - if authConfig != nil { + if authConfig != nil && len(authConfig.Username) > 0 { req.SetBasicAuth(authConfig.Username, authConfig.Password) } req.Header.Set("X-Docker-Token", "true") @@ -309,6 +309,7 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re } for askedTag, imgId := range tagsList { + fmt.Fprintf(stdout, "Resolving tag \"%s:%s\" from %s\n", remote, askedTag, endpoints) success := false for _, registry := range endpoints { if imgId == "" { From b0e076f374c8919ba29ec24a8cd27785a8e0827c Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 2 May 2013 15:30:03 -0700 Subject: [PATCH 41/97] Add output to checksums, code cleaning --- graph.go | 28 -------------------------- registry.go | 58 ++++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 46 insertions(+), 40 deletions(-) diff --git a/graph.go b/graph.go index f51597df9a..853f8c736b 100644 --- a/graph.go +++ b/graph.go @@ -286,31 +286,3 @@ func (graph *Graph) Heads() (map[string]*Image, error) { func (graph *Graph) imageRoot(id string) string { return path.Join(graph.Root, id) } - -func (graph *Graph) Checksums(repo Repository) ([]map[string]string, error) { - var result []map[string]string - checksums := map[string]string{} - for _, id := range repo { - img, err := graph.Get(id) - if err != nil { - return nil, err - } - err = img.WalkHistory(func(image *Image) error { - checksums[image.Id], err = image.Checksum() - return err - }) - if err != nil { - return nil, err - } - } - i := 0 - result = make([]map[string]string, len(checksums)) - for id, sum := range checksums { - result[i] = map[string]string{ - "id": id, - "checksum": sum, - } - i++ - } - return result, nil -} diff --git a/registry.go b/registry.go index 43738646db..b0faee3e71 100644 --- a/registry.go +++ b/registry.go @@ -95,7 +95,9 @@ func (graph *Graph) getImagesInRepository(repository string, authConfig *auth.Au if err != nil { return nil, err } - req.SetBasicAuth(authConfig.Username, authConfig.Password) + if authConfig != nil && len(authConfig.Username) > 0 { + req.SetBasicAuth(authConfig.Username, authConfig.Password) + } res, err := graph.getHttpClient().Do(req) if err != nil { return nil, err @@ -111,9 +113,12 @@ func (graph *Graph) getImagesInRepository(repository string, authConfig *auth.Au if err != nil { return nil, err } + imageList := []map[string]string{} + err = json.Unmarshal(jsonData, &imageList) if err != nil { + Debugf("Body: %s (%s)\n", res.Body, u) return nil, err } @@ -174,7 +179,7 @@ func (graph *Graph) getRemoteTags(stdout io.Writer, registries []string, reposit repository = "library/" + repository } for _, host := range registries { - endpoint := "https://" + host + "/v1/repositories/" + repository + "/tags" + endpoint := fmt.Sprintf("https://%s/v1/repositories/%s/tags", host, repository) req, err := http.NewRequest("GET", endpoint, nil) if err != nil { return nil, err @@ -433,12 +438,6 @@ func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, // push a tag on the registry. // Remote has the format '/ func (graph *Graph) pushTag(remote, revision, tag, registry string, token []string) error { - - // Keep this for backward compatibility - if tag == "" { - tag = "lastest" - } - // "jsonify" the string revision = "\"" + revision + "\"" registry = "https://" + registry + "/v1" @@ -490,16 +489,17 @@ func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId, registry func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Repository, authConfig *auth.AuthConfig) error { client := graph.getHttpClient() - checksums, err := graph.Checksums(localRepo) - imgList := make([]map[string]string, len(checksums)) - checksums2 := make([]map[string]string, len(checksums)) + checksums, err := graph.Checksums(stdout, localRepo) if err != nil { return err } + imgList := make([]map[string]string, len(checksums)) + checksums2 := make([]map[string]string, len(checksums)) + uploadedImages, err := graph.getImagesInRepository(remote, authConfig) if err != nil { - return fmt.Errorf("Error occured while fetching the list: %v", err) + return fmt.Errorf("Error occured while fetching the list: %s", err) } // Filter list to only send images/checksums not already uploaded @@ -605,3 +605,37 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re return nil } + +func (graph *Graph) Checksums(output io.Writer, repo Repository) ([]map[string]string, error) { + var result []map[string]string + checksums := map[string]string{} + for _, id := range repo { + img, err := graph.Get(id) + if err != nil { + return nil, err + } + err = img.WalkHistory(func(image *Image) error { + fmt.Fprintf(output, "Computing checksum for image %s\n", image.Id) + if _, exists := checksums[image.Id]; !exists { + checksums[image.Id], err = image.Checksum() + if err != nil { + return err + } + } + return nil + }) + if err != nil { + return nil, err + } + } + i := 0 + result = make([]map[string]string, len(checksums)) + for id, sum := range checksums { + result[i] = map[string]string{ + "id": id, + "checksum": sum, + } + i++ + } + return result, nil +} From c9994ed0fb01c65304fa8bd2eaf80f8a5edd6e41 Mon Sep 17 00:00:00 2001 From: Sam Alba Date: Sat, 4 May 2013 09:22:45 -0700 Subject: [PATCH 42/97] Moved the Debugf message in a registry to a more useful place --- registry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/registry.go b/registry.go index b0faee3e71..72379dd3dd 100644 --- a/registry.go +++ b/registry.go @@ -187,8 +187,8 @@ func (graph *Graph) getRemoteTags(stdout io.Writer, registries []string, reposit req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) res, err := client.Do(req) defer res.Body.Close() + Debugf("Got status code %d from %s", res.StatusCode, endpoint) if err != nil || (res.StatusCode != 200 && res.StatusCode != 404) { - Debugf("Registry isn't responding: trying another registry endpoint") continue } else if res.StatusCode == 404 { return nil, fmt.Errorf("Repository not found") From 09f1cbabb9ebc46776a832e1e695af7ef32a3de3 Mon Sep 17 00:00:00 2001 From: shin- Date: Mon, 6 May 2013 11:06:44 -0700 Subject: [PATCH 43/97] Fixed imports --- graph.go | 1 + registry.go | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/graph.go b/graph.go index 853f8c736b..21d9d9407c 100644 --- a/graph.go +++ b/graph.go @@ -2,6 +2,7 @@ package docker import ( "fmt" + "io" "io/ioutil" "net/http" "os" diff --git a/registry.go b/registry.go index 72379dd3dd..1028d72725 100644 --- a/registry.go +++ b/registry.go @@ -9,7 +9,6 @@ import ( "io" "io/ioutil" "net/http" - "os" "path" "strings" ) From 4f202cd07f3d9281e186024bbf6770bf93537235 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Sun, 5 May 2013 07:42:23 -0700 Subject: [PATCH 44/97] Bumped version to 0.3.0 --- CHANGELOG.md | 16 ++++++++++++---- commands.go | 2 +- packaging/ubuntu/changelog | 14 +++++++++++++- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93c3997608..5036d87c15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 0.3.0 (2013-05-06) + + Registry: Implement the new registry + + Documentation: new example: sharing data between 2 couchdb databases + - Runtime: Fix the command existance check + - Runtime: strings.Split may return an empty string on no match + - Runtime: Fix an index out of range crash if cgroup memory is not + * Documentation: Various improvments + * Vagrant: Use only one deb line in /etc/apt + ## 0.2.2 (2013-05-03) + Support for data volumes ('docker run -v=PATH') + Share data volumes between containers ('docker run -volumes-from') @@ -8,7 +17,7 @@ * Various upgrades to the dev environment for contributors ## 0.2.1 (2013-05-01) - + 'docker commit -run' bundles a layer with default runtime options: command, ports etc. + + 'docker commit -run' bundles a layer with default runtime options: command, ports etc. * Improve install process on Vagrant + New Dockerfile operation: "maintainer" + New Dockerfile operation: "expose" @@ -25,13 +34,12 @@ + Add a changelog - Various bugfixes - ## 0.1.8 (2013-04-22) - Dynamically detect cgroup capabilities - Issue stability warning on kernels <3.8 - 'docker push' buffers on disk instead of memory - Fix 'docker diff' for removed files - - Fix 'docker stop' for ghost containers + - Fix 'docker stop' for ghost containers - Fix handling of pidfile - Various bugfixes and stability improvements @@ -52,7 +60,7 @@ - Improve diagnosis of missing system capabilities - Allow disabling memory limits at compile time - Add debian packaging - - Documentation: installing on Arch Linux + - Documentation: installing on Arch Linux - Documentation: running Redis on docker - Fixed lxc 0.9 compatibility - Automatically load aufs module diff --git a/commands.go b/commands.go index 48125992f6..cdc948e785 100644 --- a/commands.go +++ b/commands.go @@ -19,7 +19,7 @@ import ( "unicode" ) -const VERSION = "0.2.2" +const VERSION = "0.3.0" var ( GIT_COMMIT string diff --git a/packaging/ubuntu/changelog b/packaging/ubuntu/changelog index 49eabfbb32..d8cd9f0e5a 100644 --- a/packaging/ubuntu/changelog +++ b/packaging/ubuntu/changelog @@ -1,3 +1,15 @@ +lxc-docker (0.3.0-1) precise; urgency=low + - Registry: Implement the new registry + - Documentation: new example: sharing data between 2 couchdb databases + - Runtime: Fix the command existance check + - Runtime: strings.Split may return an empty string on no match + - Runtime: Fix an index out of range crash if cgroup memory is not + - Documentation: Various improvments + - Vagrant: Use only one deb line in /etc/apt + + -- dotCloud Fri, 5 May 2013 00:00:00 -0700 + + lxc-docker (0.2.2-1) precise; urgency=low - Support for data volumes ('docker run -v=PATH') - Share data volumes between containers ('docker run -volumes-from') @@ -10,7 +22,7 @@ lxc-docker (0.2.2-1) precise; urgency=low lxc-docker (0.2.1-1) precise; urgency=low - - 'docker commit -run' bundles a layer with default runtime options: command, ports etc. + - 'docker commit -run' bundles a layer with default runtime options: command, ports etc. - Improve install process on Vagrant - New Dockerfile operation: "maintainer" - New Dockerfile operation: "expose" From bbad653b1aedb174500cf7b643ad945bea0d634a Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 6 May 2013 12:00:39 -0700 Subject: [PATCH 45/97] Update ubuntulinux.rst --- docs/sources/installation/ubuntulinux.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/sources/installation/ubuntulinux.rst b/docs/sources/installation/ubuntulinux.rst index ebf9876e22..3082e98da5 100644 --- a/docs/sources/installation/ubuntulinux.rst +++ b/docs/sources/installation/ubuntulinux.rst @@ -18,7 +18,7 @@ The linux-image-extra package is only needed on standard Ubuntu EC2 AMIs in orde .. code-block:: bash - sudo apt-get install linux-image-extra-`uname -r` + sudo apt-get install linux-image-extra-`uname -r` lxc bsdtar Installation @@ -48,7 +48,8 @@ Now install it, you will see another warning that the package cannot be authenti .. code-block:: bash - sudo apt-get install lxc-docker + curl -s http://get.docker.io/builds/$(uname -s)/$(uname -m)/docker-master.tgz | tar -zxf- docker-master/docker + sudo cp docker-master/docker /usr/local/bin/docker Verify it worked From 2ac7298e4e4d8babcdd8ea0c9aeeb56c4857e035 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Mon, 6 May 2013 12:24:44 -0700 Subject: [PATCH 46/97] Bring back lego picture --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 7b1e18f390..5b80fe253d 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ Docker is an open-source implementation of the deployment engine which powers [d It benefits directly from the experience accumulated over several years of large-scale operation and support of hundreds of thousands of applications and databases. +![Docker L](docs/sources/static_files/lego_docker.jpg "Docker") + ## Better than VMs A common method for distributing applications and sandbox their execution is to use virtual machines, or VMs. Typical VM formats From 0fabd390a91ae34d85bb873d05ce7132266ce3a4 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 6 May 2013 13:11:08 -0700 Subject: [PATCH 47/97] Update ubuntulinux.rst --- docs/sources/installation/ubuntulinux.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/sources/installation/ubuntulinux.rst b/docs/sources/installation/ubuntulinux.rst index 3082e98da5..955e8eb3b0 100644 --- a/docs/sources/installation/ubuntulinux.rst +++ b/docs/sources/installation/ubuntulinux.rst @@ -48,8 +48,7 @@ Now install it, you will see another warning that the package cannot be authenti .. code-block:: bash - curl -s http://get.docker.io/builds/$(uname -s)/$(uname -m)/docker-master.tgz | tar -zxf- docker-master/docker - sudo cp docker-master/docker /usr/local/bin/docker + curl get.docker.io | sudo sh -x Verify it worked From 87cc8b6058c8cf185825a039d2320af578cb8f8a Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 6 May 2013 13:26:23 -0700 Subject: [PATCH 48/97] Update documentation, use docker-latest instead of docker-master --- contrib/install.sh | 4 ++-- docs/sources/installation/binaries.rst | 8 ++++---- docs/sources/installation/upgrading.rst | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/contrib/install.sh b/contrib/install.sh index d7c6e66466..7db577a9da 100755 --- a/contrib/install.sh +++ b/contrib/install.sh @@ -36,9 +36,9 @@ else fi echo "Downloading docker binary and uncompressing into /usr/local/bin..." -curl -s http://get.docker.io/builds/$(uname -s)/$(uname -m)/docker-master.tgz | +curl -s http://get.docker.io/builds/$(uname -s)/$(uname -m)/docker-latest.tgz | tar -C /usr/local/bin --strip-components=1 -zxf- \ -docker-master/docker +docker-latest/docker if [ -f /etc/init/dockerd.conf ] then diff --git a/docs/sources/installation/binaries.rst b/docs/sources/installation/binaries.rst index 3428f9f279..2607f3680f 100644 --- a/docs/sources/installation/binaries.rst +++ b/docs/sources/installation/binaries.rst @@ -26,9 +26,9 @@ Install the docker binary: :: - wget http://get.docker.io/builds/Linux/x86_64/docker-master.tgz - tar -xf docker-master.tgz - sudo cp ./docker-master /usr/local/bin + wget http://get.docker.io/builds/Linux/x86_64/docker-latest.tgz + tar -xf docker-latest.tgz + sudo cp ./docker-latest/docker /usr/local/bin Note: docker currently only supports 64-bit Linux hosts. @@ -50,4 +50,4 @@ Run your first container! -Continue with the :ref:`hello_world` example. \ No newline at end of file +Continue with the :ref:`hello_world` example. diff --git a/docs/sources/installation/upgrading.rst b/docs/sources/installation/upgrading.rst index 66825ac643..a5172b6d76 100644 --- a/docs/sources/installation/upgrading.rst +++ b/docs/sources/installation/upgrading.rst @@ -11,7 +11,7 @@ Get the latest docker binary: :: - wget http://get.docker.io/builds/$(uname -s)/$(uname -m)/docker-master.tgz + wget http://get.docker.io/builds/$(uname -s)/$(uname -m)/docker-latest.tgz @@ -19,7 +19,7 @@ Unpack it to your current dir :: - tar -xf docker-master.tgz + tar -xf docker-latest.tgz Stop your current daemon. How you stop your daemon depends on how you started it. @@ -38,4 +38,4 @@ Now start the daemon sudo ./docker -d & -Alternatively you can replace the docker binary in ``/usr/local/bin`` \ No newline at end of file +Alternatively you can replace the docker binary in ``/usr/local/bin`` From 8646f7f11cf9c59865fdf1a4800251e2010f1b5c Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Mon, 6 May 2013 14:11:38 -0700 Subject: [PATCH 49/97] + Website: new high-level overview --- docs/sources/index.html | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/docs/sources/index.html b/docs/sources/index.html index 1ba4184e28..1d5313cf5c 100644 --- a/docs/sources/index.html +++ b/docs/sources/index.html @@ -7,7 +7,7 @@ - Docker - the Linux container runtime + Docker - the Linux container engine @@ -71,18 +71,28 @@

Docker

-

The Linux container runtime

+

The Linux container engine

-

- Docker complements LXC with a high-level API which operates at the process level. - It runs unix processes with strong guarantees of isolation and repeatability across servers. -

-

- Docker is a great building block for automating distributed systems: large-scale web deployments, database clusters, continuous deployment systems, private PaaS, service-oriented architectures, etc. -

+

+ Docker is an open-source engine which automates the deployment of applications as highly portable, self-sufficient containers. +

+ +

+ Docker containers are both hardware-agnostic and platform-agnostic. This means that they can run anywhere, from your + laptop to the largest EC2 compute instance and everything in between - and they don't require that you use a particular + language, framework or packaging system. That makes them great building blocks for deploying and scaling web apps, databases + and backend services without depending on a particular stack or provider. +

+ +

+ Docker is an open-source implementation of the deployment engine which powers dotCloud, a popular Platform-as-a-Service. + It benefits directly from the experience accumulated over several years of large-scale operation and support of hundreds of thousands + of applications and databases. +

+
From 15b85d9d762106e03657ecde183303f6535ad90e Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Fri, 3 May 2013 15:03:01 -0700 Subject: [PATCH 50/97] packaging ubuntu; issue #510: Use goland-stable PPA package to build docker --- packaging/ubuntu/Vagrantfile | 11 +++++++---- packaging/ubuntu/changelog | 24 ++++++++++++------------ packaging/ubuntu/control | 2 +- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/packaging/ubuntu/Vagrantfile b/packaging/ubuntu/Vagrantfile index 0689eea1c2..0636b01b1e 100644 --- a/packaging/ubuntu/Vagrantfile +++ b/packaging/ubuntu/Vagrantfile @@ -1,12 +1,15 @@ -BUILDBOT_IP = '192.168.33.32' +GOPHERS_KEY = "308C15A29AD198E9" Vagrant::Config.run do |config| config.vm.box = 'precise64' config.vm.box_url = 'http://files.vagrantup.com/precise64.box' config.vm.share_folder 'v-data', '/data/docker', "#{File.dirname(__FILE__)}/../.." - config.vm.network :hostonly,BUILDBOT_IP + # Add docker PPA key to the local repository and install docker + pkg_cmd = "apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys #{GOPHERS_KEY}; " \ + "echo 'deb http://ppa.launchpad.net/gophers/go/ubuntu precise main' >/etc/apt/sources.list.d/gophers-go.list; " \ # Install ubuntu packaging dependencies and create ubuntu packages - config.vm.provision :shell, :inline => 'export DEBIAN_FRONTEND=noninteractive; apt-get -qq update; apt-get install -qq -y git debhelper autotools-dev devscripts golang' - config.vm.provision :shell, :inline => "export GPG_KEY='#{ENV['GPG_KEY']}'; cd /data/docker/packaging/ubuntu; make ubuntu" + pkg_cmd << "export DEBIAN_FRONTEND=noninteractive; apt-get -qq update; apt-get install -qq -y git debhelper autotools-dev devscripts golang-stable; " \ + "export GPG_KEY='#{ENV['GPG_KEY']}'; cd /data/docker/packaging/ubuntu; make ubuntu" + config.vm.provision :shell, :inline => pkg_cmd end diff --git a/packaging/ubuntu/changelog b/packaging/ubuntu/changelog index d8cd9f0e5a..b3e68558ff 100644 --- a/packaging/ubuntu/changelog +++ b/packaging/ubuntu/changelog @@ -1,21 +1,21 @@ lxc-docker (0.3.0-1) precise; urgency=low - - Registry: Implement the new registry - - Documentation: new example: sharing data between 2 couchdb databases - - Runtime: Fix the command existance check - - Runtime: strings.Split may return an empty string on no match - - Runtime: Fix an index out of range crash if cgroup memory is not - - Documentation: Various improvments - - Vagrant: Use only one deb line in /etc/apt + - Registry: Implement the new registry + - Documentation: new example: sharing data between 2 couchdb databases + - Runtime: Fix the command existance check + - Runtime: strings.Split may return an empty string on no match + - Runtime: Fix an index out of range crash if cgroup memory is not + - Documentation: Various improvments + - Vagrant: Use only one deb line in /etc/apt -- dotCloud Fri, 5 May 2013 00:00:00 -0700 lxc-docker (0.2.2-1) precise; urgency=low - - Support for data volumes ('docker run -v=PATH') - - Share data volumes between containers ('docker run -volumes-from') - - Improved documentation - - Upgrade to Go 1.0.3 - - Various upgrades to the dev environment for contributors + - Support for data volumes ('docker run -v=PATH') + - Share data volumes between containers ('docker run -volumes-from') + - Improved documentation + - Upgrade to Go 1.0.3 + - Various upgrades to the dev environment for contributors -- dotCloud Fri, 3 May 2013 00:00:00 -0700 diff --git a/packaging/ubuntu/control b/packaging/ubuntu/control index c52303a88b..b0560ebf70 100644 --- a/packaging/ubuntu/control +++ b/packaging/ubuntu/control @@ -2,7 +2,7 @@ Source: lxc-docker Section: misc Priority: extra Maintainer: Daniel Mizyrycki -Build-Depends: debhelper,autotools-dev,devscripts,golang +Build-Depends: debhelper,autotools-dev,devscripts,golang-stable Standards-Version: 3.9.3 Homepage: http://github.com/dotcloud/docker From f20b5e1323a6da49b060ccb18af12899296706e6 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 6 May 2013 15:58:04 -0700 Subject: [PATCH 51/97] Fix issue when login in with a different user and trying to push --- commands.go | 2 ++ registry.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/commands.go b/commands.go index cdc948e785..8895712a24 100644 --- a/commands.go +++ b/commands.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/dotcloud/docker/auth" "github.com/dotcloud/docker/rcli" + "github.com/shin-/cookiejar" "io" "log" "net/http" @@ -154,6 +155,7 @@ func (srv *Server) CmdLogin(stdin io.ReadCloser, stdout rcli.DockerConn, args .. if err != nil { fmt.Fprintf(stdout, "Error: %s\r\n", err) } else { + srv.runtime.graph.getHttpClient().Jar = cookiejar.NewCookieJar() srv.runtime.authConfig = newAuthConfig } if status != "" { diff --git a/registry.go b/registry.go index 1028d72725..f9bc756643 100644 --- a/registry.go +++ b/registry.go @@ -69,7 +69,7 @@ func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([] func (graph *Graph) getHttpClient() *http.Client { if graph.httpClient == nil { - graph.httpClient = new(http.Client) + graph.httpClient = &http.Client{} graph.httpClient.Jar = cookiejar.NewCookieJar() } return graph.httpClient From 27319da0d260b42acdbc3fd2a639ad3986d1ba64 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 11:03:01 -0700 Subject: [PATCH 52/97] Add build command --- builder.go | 111 ++++++++++++++++++++++++++++++++++++++++++++++++++++ commands.go | 28 +++++++++++++ 2 files changed, 139 insertions(+) create mode 100644 builder.go diff --git a/builder.go b/builder.go new file mode 100644 index 0000000000..240170ce9b --- /dev/null +++ b/builder.go @@ -0,0 +1,111 @@ +package docker + +import ( + "bufio" + "fmt" + "io" + "strings" +) + +type Builder struct { + runtime *Runtime +} + +func NewBuilder(runtime *Runtime) *Builder { + return &Builder{ + runtime: runtime, + } +} + +func (builder *Builder) run(image *Image, cmd string) (*Container, error) { + // FIXME: pass a NopWriter instead of nil + config, err := ParseRun([]string{"-d", image.Id, "/bin/sh", "-c", cmd}, nil, builder.runtime.capabilities) + if config.Image == "" { + return nil, fmt.Errorf("Image not specified") + } + if len(config.Cmd) == 0 { + return nil, fmt.Errorf("Command not specified") + } + if config.Tty { + return nil, fmt.Errorf("The tty mode is not supported within the builder") + } + + // Create new container + container, err := builder.runtime.Create(config) + if err != nil { + return nil, err + } + if err := container.Start(); err != nil { + return nil, err + } + return container, nil +} + +func (builder *Builder) runCommit(image *Image, cmd string) (*Image, error) { + c, err := builder.run(image, cmd) + if err != nil { + return nil, err + } + if result := c.Wait(); result != 0 { + return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", cmd, result) + } + img, err := builder.runtime.Commit(c.Id, "", "", "", "") + if err != nil { + return nil, err + } + return img, nil +} + +func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { + var image, base *Image + + file := bufio.NewReader(dockerfile) + for { + line, err := file.ReadString('\n') + if err != nil { + if err == io.EOF { + break + } + return err + } + line = strings.TrimSpace(line) + // Skip comments and empty line + if len(line) == 0 || line[0] == '#' { + continue + } + tmp := strings.SplitN(line, " ", 2) + if len(tmp) != 2 { + return fmt.Errorf("Invalid Dockerfile format") + } + switch tmp[0] { + case "from": + fmt.Fprintf(stdout, "FROM %s\n", tmp[1]) + image, err = builder.runtime.repositories.LookupImage(tmp[1]) + if err != nil { + return err + } + break + case "run": + fmt.Fprintf(stdout, "RUN %s\n", tmp[1]) + if image == nil { + return fmt.Errorf("Please provide a source image with `from` prior to run") + } + base, err = builder.runCommit(image, tmp[1]) + if err != nil { + return err + } + fmt.Fprintf(stdout, "===> %s\n", base.Id) + break + case "copy": + return fmt.Errorf("The copy operator has not yet been implemented") + default: + fmt.Fprintf(stdout, "Skipping unknown op %s\n", tmp[0]) + } + } + if base != nil { + fmt.Fprintf(stdout, "Build finished. image id: %s\n", base.Id) + } else { + fmt.Fprintf(stdout, "An error occured during the build\n") + } + return nil +} diff --git a/commands.go b/commands.go index cdc948e785..7116d919f3 100644 --- a/commands.go +++ b/commands.go @@ -11,6 +11,7 @@ import ( "net/http" "net/url" "path/filepath" + "os" "runtime" "strconv" "strings" @@ -34,6 +35,7 @@ func (srv *Server) Help() string { help := "Usage: docker COMMAND [arg...]\n\nA self-sufficient runtime for linux containers.\n\nCommands:\n" for _, cmd := range [][]string{ {"attach", "Attach to a running container"}, + {"build", "Build a container from Dockerfile"}, {"commit", "Create a new image from a container's changes"}, {"diff", "Inspect changes on a container's filesystem"}, {"export", "Stream the contents of a container as a tar archive"}, @@ -64,6 +66,32 @@ func (srv *Server) Help() string { return help } +func (srv *Server) CmdBuild(stdin io.ReadCloser, stdout rcli.DockerConn, args ...string) error { + stdout.Flush() + cmd := rcli.Subcmd(stdout, "build", "[Dockerfile|-]", "Build a container from Dockerfile") + if err := cmd.Parse(args); err != nil { + return nil + } + dockerfile := cmd.Arg(0) + if dockerfile == "" { + dockerfile = "Dockerfile" + } + + var file io.Reader + + if dockerfile != "-" { + f, err := os.Open(dockerfile) + if err != nil { + return err + } + defer f.Close() + file = f + } else { + file = stdin + } + return NewBuilder(srv.runtime).Build(file, stdout) +} + // 'docker login': login / register a user to registry service. func (srv *Server) CmdLogin(stdin io.ReadCloser, stdout rcli.DockerConn, args ...string) error { // Read a line on raw terminal with support for simple backspace From b8f66c0d14780bd8cb3b8916acc0287fcaf8cafa Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 12:02:00 -0700 Subject: [PATCH 53/97] Clear the containers/images upon failure --- builder.go | 51 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/builder.go b/builder.go index 240170ce9b..1b54df81ba 100644 --- a/builder.go +++ b/builder.go @@ -41,23 +41,25 @@ func (builder *Builder) run(image *Image, cmd string) (*Container, error) { return container, nil } -func (builder *Builder) runCommit(image *Image, cmd string) (*Image, error) { - c, err := builder.run(image, cmd) - if err != nil { - return nil, err +func (builder *Builder) clearTmp(containers, images map[string]struct{}) { + for c := range containers { + tmp := builder.runtime.Get(c) + builder.runtime.Destroy(tmp) + Debugf("Removing container %s", c) } - if result := c.Wait(); result != 0 { - return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", cmd, result) + for i := range images { + builder.runtime.graph.Delete(i) + Debugf("Removing image %s", i) } - img, err := builder.runtime.Commit(c.Id, "", "", "", "") - if err != nil { - return nil, err - } - return img, nil } func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { - var image, base *Image + var ( + image, base *Image + tmpContainers map[string]struct{} = make(map[string]struct{}) + tmpImages map[string]struct{} = make(map[string]struct{}) + ) + defer builder.clearTmp(tmpContainers, tmpImages) file := bufio.NewReader(dockerfile) for { @@ -90,10 +92,26 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { if image == nil { return fmt.Errorf("Please provide a source image with `from` prior to run") } - base, err = builder.runCommit(image, tmp[1]) + + // Create the container and start it + c, err := builder.run(image, tmp[1]) if err != nil { return err } + tmpContainers[c.Id] = struct{}{} + + // Wait for it to finish + if result := c.Wait(); result != 0 { + return fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", tmp[1], result) + } + + // Commit the container + base, err := builder.runtime.Commit(c.Id, "", "", "", "") + if err != nil { + return err + } + tmpImages[base.Id] = struct{}{} + fmt.Fprintf(stdout, "===> %s\n", base.Id) break case "copy": @@ -103,6 +121,13 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { } } if base != nil { + // The build is successful, keep the temporary containers and images + for i := range tmpImages { + delete(tmpImages, i) + } + for i := range tmpContainers { + delete(tmpContainers, i) + } fmt.Fprintf(stdout, "Build finished. image id: %s\n", base.Id) } else { fmt.Fprintf(stdout, "An error occured during the build\n") From 97215ca384373bfa65c08dc05454c14cef493d30 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 12:31:20 -0700 Subject: [PATCH 54/97] make builder.Run public it now runs only given arguments without sh -c --- builder.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/builder.go b/builder.go index 1b54df81ba..cbc8711aa7 100644 --- a/builder.go +++ b/builder.go @@ -17,9 +17,9 @@ func NewBuilder(runtime *Runtime) *Builder { } } -func (builder *Builder) run(image *Image, cmd string) (*Container, error) { +func (builder *Builder) Run(image *Image, cmd ...string) (*Container, error) { // FIXME: pass a NopWriter instead of nil - config, err := ParseRun([]string{"-d", image.Id, "/bin/sh", "-c", cmd}, nil, builder.runtime.capabilities) + config, err := ParseRun(append([]string{"-d", image.Id}, cmd...), nil, builder.runtime.capabilities) if config.Image == "" { return nil, fmt.Errorf("Image not specified") } @@ -94,7 +94,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { } // Create the container and start it - c, err := builder.run(image, tmp[1]) + c, err := builder.Run(image, "/bin/sh", "-c", tmp[1]) if err != nil { return err } From 7bccdc0d33d0589792b9b867b13c9e50dcf48358 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 13:35:57 -0700 Subject: [PATCH 55/97] Add a Builder.Commit method --- builder.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/builder.go b/builder.go index cbc8711aa7..b15a71d621 100644 --- a/builder.go +++ b/builder.go @@ -41,6 +41,10 @@ func (builder *Builder) Run(image *Image, cmd ...string) (*Container, error) { return container, nil } +func (builder *Builder) Commit(container *Container, repository, tag, comment, author string) (*Image, error) { + return builder.runtime.Commit(container.Id, repository, tag, comment, author) +} + func (builder *Builder) clearTmp(containers, images map[string]struct{}) { for c := range containers { tmp := builder.runtime.Get(c) @@ -106,7 +110,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { } // Commit the container - base, err := builder.runtime.Commit(c.Id, "", "", "", "") + base, err := builder.Commit(c, "", "", "", "") if err != nil { return err } From 97514831128a7bc7bc6a2f740275bf6336ede999 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 13:37:00 -0700 Subject: [PATCH 56/97] Add insert command in order to insert external files within an image --- commands.go | 43 +++++++++++++++++++++++++++++++++++++++++++ container.go | 12 ++++++++++++ 2 files changed, 55 insertions(+) diff --git a/commands.go b/commands.go index 7116d919f3..21c8b68abc 100644 --- a/commands.go +++ b/commands.go @@ -43,6 +43,7 @@ func (srv *Server) Help() string { {"images", "List images"}, {"import", "Create a new filesystem image from the contents of a tarball"}, {"info", "Display system-wide information"}, + {"insert", "Insert a file in an image"}, {"inspect", "Return low-level information on a container"}, {"kill", "Kill a running container"}, {"login", "Register or Login to the docker registry server"}, @@ -66,6 +67,48 @@ func (srv *Server) Help() string { return help } +func (srv *Server) CmdInsert(stdin io.ReadCloser, stdout rcli.DockerConn, args ...string) error { + stdout.Flush() + cmd := rcli.Subcmd(stdout, "insert", "IMAGE URL PATH", "Insert a file from URL in the IMAGE at PATH") + if err := cmd.Parse(args); err != nil { + return nil + } + if cmd.NArg() != 3 { + cmd.Usage() + return nil + } + imageId := cmd.Arg(0) + url := cmd.Arg(1) + path := cmd.Arg(2) + + img, err := srv.runtime.repositories.LookupImage(imageId) + if err != nil { + return err + } + file, err := Download(url, stdout) + if err != nil { + return err + } + defer file.Body.Close() + + b := NewBuilder(srv.runtime) + c, err := b.Run(img, "echo", "insert", url, path) + if err != nil { + return err + } + + if err := c.Inject(ProgressReader(file.Body, int(file.ContentLength), stdout, "Downloading %v/%v (%v)"), path); err != nil { + return err + } + // FIXME: Handle custom repo, tag comment, author + img, err = b.Commit(c, "", "", img.Comment, img.Author) + if err != nil { + return err + } + fmt.Fprintf(stdout, "%s\n", img) + return nil +} + func (srv *Server) CmdBuild(stdin io.ReadCloser, stdout rcli.DockerConn, args ...string) error { stdout.Flush() cmd := rcli.Subcmd(stdout, "build", "[Dockerfile|-]", "Build a container from Dockerfile") diff --git a/container.go b/container.go index 311e475ac1..ed3bdce94a 100644 --- a/container.go +++ b/container.go @@ -178,6 +178,18 @@ func (settings *NetworkSettings) PortMappingHuman() string { return strings.Join(mapping, ", ") } +// Inject the io.Reader at the given path. Note: do not close the reader +func (container *Container) Inject(file io.Reader, pth string) error { + dest, err := os.Open(path.Join(container.rwPath(), pth)) + if err != nil { + return err + } + if _, err := io.Copy(dest, file); err != nil { + return err + } + return nil +} + func (container *Container) Cmd() *exec.Cmd { return container.cmd } From 9db4972a70c2fb0921f984d67f816b81ea687a11 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 13:51:28 -0700 Subject: [PATCH 57/97] Make sure the destination directory exists when using docker insert --- builder.go | 2 +- commands.go | 2 +- container.go | 7 ++++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/builder.go b/builder.go index b15a71d621..717525831b 100644 --- a/builder.go +++ b/builder.go @@ -110,7 +110,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { } // Commit the container - base, err := builder.Commit(c, "", "", "", "") + base, err = builder.Commit(c, "", "", "", "") if err != nil { return err } diff --git a/commands.go b/commands.go index 21c8b68abc..9d848b95db 100644 --- a/commands.go +++ b/commands.go @@ -105,7 +105,7 @@ func (srv *Server) CmdInsert(stdin io.ReadCloser, stdout rcli.DockerConn, args . if err != nil { return err } - fmt.Fprintf(stdout, "%s\n", img) + fmt.Fprintf(stdout, "%s\n", img.Id) return nil } diff --git a/container.go b/container.go index ed3bdce94a..a4bee6a6fe 100644 --- a/container.go +++ b/container.go @@ -180,7 +180,12 @@ func (settings *NetworkSettings) PortMappingHuman() string { // Inject the io.Reader at the given path. Note: do not close the reader func (container *Container) Inject(file io.Reader, pth string) error { - dest, err := os.Open(path.Join(container.rwPath(), pth)) + // Make sure the directory exists + if err := os.MkdirAll(path.Join(container.rwPath(), path.Dir(pth)), 0755); err != nil { + return err + } + // FIXME: Handle permissions/already existing dest + dest, err := os.Create(path.Join(container.rwPath(), pth)) if err != nil { return err } From 0aebb254106be3086838cac145404c5d21c6bcd0 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 14:28:51 -0700 Subject: [PATCH 58/97] Implement the COPY operator within the builder --- builder.go | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/builder.go b/builder.go index 717525831b..105cb91e33 100644 --- a/builder.go +++ b/builder.go @@ -116,10 +116,39 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { } tmpImages[base.Id] = struct{}{} - fmt.Fprintf(stdout, "===> %s\n", base.Id) + fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) break case "copy": - return fmt.Errorf("The copy operator has not yet been implemented") + if image == nil { + return fmt.Errorf("Please provide a source image with `from` prior to copy") + } + tmp2 := strings.SplitN(tmp[1], " ", 2) + if len(tmp) != 2 { + return fmt.Errorf("Invalid COPY format") + } + fmt.Fprintf(stdout, "COPY %s to %s in %s\n", tmp2[0], tmp2[1], base.ShortId()) + + file, err := Download(tmp2[0], stdout) + if err != nil { + return err + } + defer file.Body.Close() + + c, err := builder.Run(base, "echo", "insert", tmp2[0], tmp2[1]) + if err != nil { + return err + } + + if err := c.Inject(file.Body, tmp2[1]); err != nil { + return err + } + + base, err = builder.Commit(c, "", "", "", "") + if err != nil { + return err + } + fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) + break default: fmt.Fprintf(stdout, "Skipping unknown op %s\n", tmp[0]) } @@ -132,7 +161,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { for i := range tmpContainers { delete(tmpContainers, i) } - fmt.Fprintf(stdout, "Build finished. image id: %s\n", base.Id) + fmt.Fprintf(stdout, "Build finished. image id: %s\n", base.ShortId()) } else { fmt.Fprintf(stdout, "An error occured during the build\n") } From 6f2125386a07a01eccce3bb4602aeecb8be3458b Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 15:14:10 -0700 Subject: [PATCH 59/97] Moving runtime.Create to builder.Create --- builder.go | 102 +++++++++++++++++++++++++++++++++++++++++----------- commands.go | 13 +++++-- runtime.go | 4 +-- 3 files changed, 93 insertions(+), 26 deletions(-) diff --git a/builder.go b/builder.go index 105cb91e33..66118a9879 100644 --- a/builder.go +++ b/builder.go @@ -4,38 +4,80 @@ import ( "bufio" "fmt" "io" + "os" + "path" "strings" + "time" ) type Builder struct { - runtime *Runtime + runtime *Runtime + repositories *TagStore } func NewBuilder(runtime *Runtime) *Builder { return &Builder{ - runtime: runtime, + runtime: runtime, + repositories: runtime.repositories, } } -func (builder *Builder) Run(image *Image, cmd ...string) (*Container, error) { - // FIXME: pass a NopWriter instead of nil - config, err := ParseRun(append([]string{"-d", image.Id}, cmd...), nil, builder.runtime.capabilities) - if config.Image == "" { - return nil, fmt.Errorf("Image not specified") - } - if len(config.Cmd) == 0 { - return nil, fmt.Errorf("Command not specified") - } - if config.Tty { - return nil, fmt.Errorf("The tty mode is not supported within the builder") - } - - // Create new container - container, err := builder.runtime.Create(config) +func (builder *Builder) Create(config *Config) (*Container, error) { + // Lookup image + img, err := builder.repositories.LookupImage(config.Image) if err != nil { return nil, err } - if err := container.Start(); err != nil { + // Generate id + id := GenerateId() + // Generate default hostname + // FIXME: the lxc template no longer needs to set a default hostname + if config.Hostname == "" { + config.Hostname = id[:12] + } + + container := &Container{ + // FIXME: we should generate the ID here instead of receiving it as an argument + Id: id, + Created: time.Now(), + Path: config.Cmd[0], + Args: config.Cmd[1:], //FIXME: de-duplicate from config + Config: config, + Image: img.Id, // Always use the resolved image id + NetworkSettings: &NetworkSettings{}, + // FIXME: do we need to store this in the container? + SysInitPath: sysInitPath, + } + container.root = builder.runtime.containerRoot(container.Id) + // Step 1: create the container directory. + // This doubles as a barrier to avoid race conditions. + if err := os.Mkdir(container.root, 0700); err != nil { + return nil, err + } + + // If custom dns exists, then create a resolv.conf for the container + if len(config.Dns) > 0 { + container.ResolvConfPath = path.Join(container.root, "resolv.conf") + f, err := os.Create(container.ResolvConfPath) + if err != nil { + return nil, err + } + defer f.Close() + for _, dns := range config.Dns { + if _, err := f.Write([]byte("nameserver " + dns + "\n")); err != nil { + return nil, err + } + } + } else { + container.ResolvConfPath = "/etc/resolv.conf" + } + + // Step 2: save the container json + if err := container.ToDisk(); err != nil { + return nil, err + } + // Step 3: register the container + if err := builder.runtime.Register(container); err != nil { return nil, err } return container, nil @@ -96,12 +138,19 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { if image == nil { return fmt.Errorf("Please provide a source image with `from` prior to run") } + config, err := ParseRun([]string{image.Id, "/bin/sh", "-c", tmp[1]}, nil, builder.runtime.capabilities) + if err != nil { + return err + } // Create the container and start it - c, err := builder.Run(image, "/bin/sh", "-c", tmp[1]) + c, err := builder.Create(config) if err != nil { return err } + if err := c.Start(); err != nil { + return err + } tmpContainers[c.Id] = struct{}{} // Wait for it to finish @@ -134,10 +183,23 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { } defer file.Body.Close() - c, err := builder.Run(base, "echo", "insert", tmp2[0], tmp2[1]) + config, err := ParseRun([]string{base.Id, "echo", "insert", tmp2[0], tmp2[1]}, nil, builder.runtime.capabilities) if err != nil { return err } + c, err := builder.Create(config) + if err != nil { + return err + } + + if err := c.Start(); err != nil { + return err + } + + // Wait for echo to finish + if result := c.Wait(); result != 0 { + return fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", tmp[1], result) + } if err := c.Inject(file.Body, tmp2[1]); err != nil { return err diff --git a/commands.go b/commands.go index 9d848b95db..aaa4dab3e5 100644 --- a/commands.go +++ b/commands.go @@ -91,8 +91,13 @@ func (srv *Server) CmdInsert(stdin io.ReadCloser, stdout rcli.DockerConn, args . } defer file.Body.Close() + config, err := ParseRun([]string{img.Id, "echo", "insert", url, path}, nil, srv.runtime.capabilities) + if err != nil { + return err + } + b := NewBuilder(srv.runtime) - c, err := b.Run(img, "echo", "insert", url, path) + c, err := b.Create(config) if err != nil { return err } @@ -1065,8 +1070,10 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout rcli.DockerConn, args ...s // or tell the client there is no options stdout.Flush() + b := NewBuilder(srv.runtime) + // Create new container - container, err := srv.runtime.Create(config) + container, err := b.Create(config) if err != nil { // If container not found, try to pull it if srv.runtime.graph.IsNotExist(err) { @@ -1074,7 +1081,7 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout rcli.DockerConn, args ...s if err = srv.CmdPull(stdin, stdout, config.Image); err != nil { return err } - if container, err = srv.runtime.Create(config); err != nil { + if container, err = b.Create(config); err != nil { return err } } else { diff --git a/runtime.go b/runtime.go index 79a7170c7d..a298d9b714 100644 --- a/runtime.go +++ b/runtime.go @@ -12,7 +12,6 @@ import ( "path" "sort" "strings" - "time" ) type Capabilities struct { @@ -116,7 +115,6 @@ func (runtime *Runtime) mergeConfig(userConf, imageConf *Config) { } func (runtime *Runtime) Create(config *Config) (*Container, error) { - // Lookup image img, err := runtime.repositories.LookupImage(config.Image) if err != nil { @@ -151,7 +149,6 @@ func (runtime *Runtime) Create(config *Config) (*Container, error) { // FIXME: do we need to store this in the container? SysInitPath: sysInitPath, } - container.root = runtime.containerRoot(container.Id) // Step 1: create the container directory. // This doubles as a barrier to avoid race conditions. @@ -187,6 +184,7 @@ func (runtime *Runtime) Create(config *Config) (*Container, error) { return container, nil } +======= end func (runtime *Runtime) Load(id string) (*Container, error) { container := &Container{root: runtime.containerRoot(id)} if err := container.FromDisk(); err != nil { From f7c5e92a2e1ec30f50b0affe952a0496c62195f5 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 15:24:14 -0700 Subject: [PATCH 60/97] Move runtime.Commit to builder.Commit --- builder.go | 23 ++++++++++++++++++++++- commands.go | 7 ++++++- runtime.go | 27 --------------------------- 3 files changed, 28 insertions(+), 29 deletions(-) diff --git a/builder.go b/builder.go index 66118a9879..1c354ea8a9 100644 --- a/builder.go +++ b/builder.go @@ -13,11 +13,13 @@ import ( type Builder struct { runtime *Runtime repositories *TagStore + graph *Graph } func NewBuilder(runtime *Runtime) *Builder { return &Builder{ runtime: runtime, + graph: runtime.graph, repositories: runtime.repositories, } } @@ -83,8 +85,27 @@ func (builder *Builder) Create(config *Config) (*Container, error) { return container, nil } +// Commit creates a new filesystem image from the current state of a container. +// The image can optionally be tagged into a repository func (builder *Builder) Commit(container *Container, repository, tag, comment, author string) (*Image, error) { - return builder.runtime.Commit(container.Id, repository, tag, comment, author) + // FIXME: freeze the container before copying it to avoid data corruption? + // FIXME: this shouldn't be in commands. + rwTar, err := container.ExportRw() + if err != nil { + return nil, err + } + // Create a new image from the container's base layers + a new layer from container changes + img, err := builder.graph.Create(rwTar, container, comment, author) + if err != nil { + return nil, err + } + // Register the image if needed + if repository != "" { + if err := builder.repositories.Set(repository, tag, img.Id, true); err != nil { + return img, err + } + } + return img, nil } func (builder *Builder) clearTmp(containers, images map[string]struct{}) { diff --git a/commands.go b/commands.go index aaa4dab3e5..39730afe56 100644 --- a/commands.go +++ b/commands.go @@ -852,7 +852,12 @@ func (srv *Server) CmdCommit(stdin io.ReadCloser, stdout io.Writer, args ...stri } } - img, err := srv.runtime.Commit(containerName, repository, tag, *flComment, *flAuthor, config) + container := srv.runtime.Get(containerName) + if container == nil { + return fmt.Errorf("No such container: %s", containerName) + } + + img, err := NewBuilder(srv.runtime).Commit(container, repository, tag, *flComment, *flAuthor, config) if err != nil { return err } diff --git a/runtime.go b/runtime.go index a298d9b714..ca27b7a7bf 100644 --- a/runtime.go +++ b/runtime.go @@ -309,33 +309,6 @@ func (runtime *Runtime) Destroy(container *Container) error { return nil } -// Commit creates a new filesystem image from the current state of a container. -// The image can optionally be tagged into a repository -func (runtime *Runtime) Commit(id, repository, tag, comment, author string, config *Config) (*Image, error) { - container := runtime.Get(id) - if container == nil { - return nil, fmt.Errorf("No such container: %s", id) - } - // FIXME: freeze the container before copying it to avoid data corruption? - // FIXME: this shouldn't be in commands. - rwTar, err := container.ExportRw() - if err != nil { - return nil, err - } - // Create a new image from the container's base layers + a new layer from container changes - img, err := runtime.graph.Create(rwTar, container, comment, author, config) - if err != nil { - return nil, err - } - // Register the image if needed - if repository != "" { - if err := runtime.repositories.Set(repository, tag, img.Id, true); err != nil { - return img, err - } - } - return img, nil -} - func (runtime *Runtime) restore() error { dir, err := ioutil.ReadDir(runtime.repository) if err != nil { From ff95f2b0ecf54fdfeffa1aa008e22cb5c7ec59d6 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 15:35:28 -0700 Subject: [PATCH 61/97] Update the unit tests to reflect the new API --- commands_test.go | 2 +- container_test.go | 67 +++++++++++++++++++++++++++-------------------- runtime_test.go | 21 +++++++++------ 3 files changed, 53 insertions(+), 37 deletions(-) diff --git a/commands_test.go b/commands_test.go index 83b480d52a..469364b4a2 100644 --- a/commands_test.go +++ b/commands_test.go @@ -339,7 +339,7 @@ func TestAttachDisconnect(t *testing.T) { srv := &Server{runtime: runtime} - container, err := runtime.Create( + container, err := NewBuilder(runtime).Create( &Config{ Image: GetTestImage(runtime).Id, Memory: 33554432, diff --git a/container_test.go b/container_test.go index c3a891cf4c..c06b8de876 100644 --- a/container_test.go +++ b/container_test.go @@ -20,7 +20,7 @@ func TestIdFormat(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container1, err := runtime.Create( + container1, err := NewBuilder(runtime).Create( &Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"/bin/sh", "-c", "echo hello world"}, @@ -44,7 +44,7 @@ func TestMultipleAttachRestart(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create( + container, err := NewBuilder(runtime).Create( &Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"/bin/sh", "-c", @@ -148,8 +148,10 @@ func TestDiff(t *testing.T) { } defer nuke(runtime) + builder := NewBuilder(runtime) + // Create a container and remove a file - container1, err := runtime.Create( + container1, err := builder.Create( &Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"/bin/rm", "/etc/passwd"}, @@ -190,7 +192,7 @@ func TestDiff(t *testing.T) { } // Create a new container from the commited image - container2, err := runtime.Create( + container2, err := builder.Create( &Config{ Image: img.Id, Cmd: []string{"cat", "/etc/passwd"}, @@ -301,7 +303,10 @@ func TestCommitRun(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container1, err := runtime.Create( + + builder := NewBuilder(runtime) + + container1, err := builder.Create( &Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"/bin/sh", "-c", "echo hello > /world"}, @@ -333,7 +338,7 @@ func TestCommitRun(t *testing.T) { // FIXME: Make a TestCommit that stops here and check docker.root/layers/img.id/world - container2, err := runtime.Create( + container2, err := builder.Create( &Config{ Image: img.Id, Cmd: []string{"cat", "/world"}, @@ -380,7 +385,7 @@ func TestStart(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create( + container, err := NewBuilder(runtime).Create( &Config{ Image: GetTestImage(runtime).Id, Memory: 33554432, @@ -419,7 +424,7 @@ func TestRun(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create( + container, err := NewBuilder(runtime).Create( &Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"ls", "-al"}, @@ -447,7 +452,7 @@ func TestOutput(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create( + container, err := NewBuilder(runtime).Create( &Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"echo", "-n", "foobar"}, @@ -472,7 +477,7 @@ func TestKillDifferentUser(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create(&Config{ + container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"tail", "-f", "/etc/resolv.conf"}, User: "daemon", @@ -520,7 +525,7 @@ func TestKill(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create(&Config{ + container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"cat", "/dev/zero"}, }, @@ -566,7 +571,9 @@ func TestExitCode(t *testing.T) { } defer nuke(runtime) - trueContainer, err := runtime.Create(&Config{ + builder := NewBuilder(runtime) + + trueContainer, err := builder.Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"/bin/true", ""}, }) @@ -581,7 +588,7 @@ func TestExitCode(t *testing.T) { t.Errorf("Unexpected exit code %d (expected 0)", trueContainer.State.ExitCode) } - falseContainer, err := runtime.Create(&Config{ + falseContainer, err := builder.Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"/bin/false", ""}, }) @@ -603,7 +610,7 @@ func TestRestart(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create(&Config{ + container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"echo", "-n", "foobar"}, }, @@ -636,7 +643,7 @@ func TestRestartStdin(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create(&Config{ + container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"cat"}, @@ -715,8 +722,10 @@ func TestUser(t *testing.T) { } defer nuke(runtime) + builder := NewBuilder(runtime) + // Default user must be root - container, err := runtime.Create(&Config{ + container, err := builder.Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"id"}, }, @@ -734,7 +743,7 @@ func TestUser(t *testing.T) { } // Set a username - container, err = runtime.Create(&Config{ + container, err = builder.Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"id"}, @@ -754,7 +763,7 @@ func TestUser(t *testing.T) { } // Set a UID - container, err = runtime.Create(&Config{ + container, err = builder.Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"id"}, @@ -774,7 +783,7 @@ func TestUser(t *testing.T) { } // Set a different user by uid - container, err = runtime.Create(&Config{ + container, err = builder.Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"id"}, @@ -796,7 +805,7 @@ func TestUser(t *testing.T) { } // Set a different user by username - container, err = runtime.Create(&Config{ + container, err = builder.Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"id"}, @@ -823,7 +832,9 @@ func TestMultipleContainers(t *testing.T) { } defer nuke(runtime) - container1, err := runtime.Create(&Config{ + builder := NewBuilder(runtime) + + container1, err := builder.Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"cat", "/dev/zero"}, }, @@ -833,7 +844,7 @@ func TestMultipleContainers(t *testing.T) { } defer runtime.Destroy(container1) - container2, err := runtime.Create(&Config{ + container2, err := builder.Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"cat", "/dev/zero"}, }, @@ -879,7 +890,7 @@ func TestStdin(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create(&Config{ + container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"cat"}, @@ -926,7 +937,7 @@ func TestTty(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create(&Config{ + container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"cat"}, @@ -973,7 +984,7 @@ func TestEnv(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create(&Config{ + container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"/usr/bin/env"}, }, @@ -1047,7 +1058,7 @@ func TestLXCConfig(t *testing.T) { memMin := 33554432 memMax := 536870912 mem := memMin + rand.Intn(memMax-memMin) - container, err := runtime.Create(&Config{ + container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"/bin/true"}, @@ -1074,7 +1085,7 @@ func BenchmarkRunSequencial(b *testing.B) { } defer nuke(runtime) for i := 0; i < b.N; i++ { - container, err := runtime.Create(&Config{ + container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"echo", "-n", "foo"}, }, @@ -1109,7 +1120,7 @@ func BenchmarkRunParallel(b *testing.B) { complete := make(chan error) tasks = append(tasks, complete) go func(i int, complete chan error) { - container, err := runtime.Create(&Config{ + container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"echo", "-n", "foo"}, }, diff --git a/runtime_test.go b/runtime_test.go index e9be838c0e..8e21f57bc5 100644 --- a/runtime_test.go +++ b/runtime_test.go @@ -118,7 +118,7 @@ func TestRuntimeCreate(t *testing.T) { if len(runtime.List()) != 0 { t.Errorf("Expected 0 containers, %v found", len(runtime.List())) } - container, err := runtime.Create(&Config{ + container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"ls", "-al"}, }, @@ -165,7 +165,7 @@ func TestDestroy(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container, err := runtime.Create(&Config{ + container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"ls", "-al"}, }, @@ -212,7 +212,10 @@ func TestGet(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container1, err := runtime.Create(&Config{ + + builder := NewBuilder(runtime) + + container1, err := builder.Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"ls", "-al"}, }, @@ -222,7 +225,7 @@ func TestGet(t *testing.T) { } defer runtime.Destroy(container1) - container2, err := runtime.Create(&Config{ + container2, err := builder.Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"ls", "-al"}, }, @@ -232,7 +235,7 @@ func TestGet(t *testing.T) { } defer runtime.Destroy(container2) - container3, err := runtime.Create(&Config{ + container3, err := builder.Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"ls", "-al"}, }, @@ -262,7 +265,7 @@ func TestAllocatePortLocalhost(t *testing.T) { if err != nil { t.Fatal(err) } - container, err := runtime.Create(&Config{ + container, err := NewBuilder(runtime).Create(&Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"sh", "-c", "echo well hello there | nc -l -p 5555"}, PortSpecs: []string{"5555"}, @@ -325,8 +328,10 @@ func TestRestore(t *testing.T) { t.Fatal(err) } + builder := NewBuilder(runtime1) + // Create a container with one instance of docker - container1, err := runtime1.Create(&Config{ + container1, err := builder.Create(&Config{ Image: GetTestImage(runtime1).Id, Cmd: []string{"ls", "-al"}, }, @@ -337,7 +342,7 @@ func TestRestore(t *testing.T) { defer runtime1.Destroy(container1) // Create a second container meant to be killed - container2, err := runtime1.Create(&Config{ + container2, err := builder.Create(&Config{ Image: GetTestImage(runtime1).Id, Cmd: []string{"/bin/cat"}, OpenStdin: true, From 74b9e851f6f407ede9d46ab4960e5e134ff666c7 Mon Sep 17 00:00:00 2001 From: Nate Jones Date: Thu, 25 Apr 2013 08:08:05 -0700 Subject: [PATCH 62/97] use new image as base of next command --- builder.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/builder.go b/builder.go index 1c354ea8a9..faed9c7794 100644 --- a/builder.go +++ b/builder.go @@ -187,6 +187,10 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { tmpImages[base.Id] = struct{}{} fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) + + // use the base as the new image + image = base + break case "copy": if image == nil { From dade95844feaa2446e472767e1e2763996241dee Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 25 Apr 2013 11:20:45 -0700 Subject: [PATCH 63/97] Make Builder.Build return the builded image --- builder.go | 40 ++++++++++++++++++++-------------------- commands.go | 7 ++++++- utils.go | 7 +++++++ 3 files changed, 33 insertions(+), 21 deletions(-) diff --git a/builder.go b/builder.go index faed9c7794..6c389b34b4 100644 --- a/builder.go +++ b/builder.go @@ -120,7 +120,7 @@ func (builder *Builder) clearTmp(containers, images map[string]struct{}) { } } -func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { +func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, error) { var ( image, base *Image tmpContainers map[string]struct{} = make(map[string]struct{}) @@ -135,7 +135,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { if err == io.EOF { break } - return err + return nil, err } line = strings.TrimSpace(line) // Skip comments and empty line @@ -144,45 +144,45 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { } tmp := strings.SplitN(line, " ", 2) if len(tmp) != 2 { - return fmt.Errorf("Invalid Dockerfile format") + return nil, fmt.Errorf("Invalid Dockerfile format") } switch tmp[0] { case "from": fmt.Fprintf(stdout, "FROM %s\n", tmp[1]) image, err = builder.runtime.repositories.LookupImage(tmp[1]) if err != nil { - return err + return nil, err } break case "run": fmt.Fprintf(stdout, "RUN %s\n", tmp[1]) if image == nil { - return fmt.Errorf("Please provide a source image with `from` prior to run") + return nil, fmt.Errorf("Please provide a source image with `from` prior to run") } config, err := ParseRun([]string{image.Id, "/bin/sh", "-c", tmp[1]}, nil, builder.runtime.capabilities) if err != nil { - return err + return nil, err } // Create the container and start it c, err := builder.Create(config) if err != nil { - return err + return nil, err } if err := c.Start(); err != nil { - return err + return nil, err } tmpContainers[c.Id] = struct{}{} // Wait for it to finish if result := c.Wait(); result != 0 { - return fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", tmp[1], result) + return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", tmp[1], result) } // Commit the container base, err = builder.Commit(c, "", "", "", "") if err != nil { - return err + return nil, err } tmpImages[base.Id] = struct{}{} @@ -194,45 +194,45 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { break case "copy": if image == nil { - return fmt.Errorf("Please provide a source image with `from` prior to copy") + return nil, fmt.Errorf("Please provide a source image with `from` prior to copy") } tmp2 := strings.SplitN(tmp[1], " ", 2) if len(tmp) != 2 { - return fmt.Errorf("Invalid COPY format") + return nil, fmt.Errorf("Invalid COPY format") } fmt.Fprintf(stdout, "COPY %s to %s in %s\n", tmp2[0], tmp2[1], base.ShortId()) file, err := Download(tmp2[0], stdout) if err != nil { - return err + return nil, err } defer file.Body.Close() config, err := ParseRun([]string{base.Id, "echo", "insert", tmp2[0], tmp2[1]}, nil, builder.runtime.capabilities) if err != nil { - return err + return nil, err } c, err := builder.Create(config) if err != nil { - return err + return nil, err } if err := c.Start(); err != nil { - return err + return nil, err } // Wait for echo to finish if result := c.Wait(); result != 0 { - return fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", tmp[1], result) + return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", tmp[1], result) } if err := c.Inject(file.Body, tmp2[1]); err != nil { - return err + return nil, err } base, err = builder.Commit(c, "", "", "", "") if err != nil { - return err + return nil, err } fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) break @@ -252,5 +252,5 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { } else { fmt.Fprintf(stdout, "An error occured during the build\n") } - return nil + return base, nil } diff --git a/commands.go b/commands.go index 39730afe56..3975d67671 100644 --- a/commands.go +++ b/commands.go @@ -137,7 +137,12 @@ func (srv *Server) CmdBuild(stdin io.ReadCloser, stdout rcli.DockerConn, args .. } else { file = stdin } - return NewBuilder(srv.runtime).Build(file, stdout) + img, err := NewBuilder(srv.runtime).Build(file, stdout) + if err != nil { + return err + } + fmt.Fprintf(stdout, "%s\n", img.ShortId()) + return nil } // 'docker login': login / register a user to registry service. diff --git a/utils.go b/utils.go index 229b938830..047a29abef 100644 --- a/utils.go +++ b/utils.go @@ -155,6 +155,13 @@ func SelfPath() string { return path } +type nopWriter struct { +} + +func (w *nopWriter) Write(buf []byte) (int, error) { + return len(buf), nil +} + type nopWriteCloser struct { io.Writer } From e337949cb0ce8eda0673e005e670c12e997a3336 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 25 Apr 2013 11:20:56 -0700 Subject: [PATCH 64/97] Add builder_test.go --- builder_test.go | 88 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 builder_test.go diff --git a/builder_test.go b/builder_test.go new file mode 100644 index 0000000000..345fd2c594 --- /dev/null +++ b/builder_test.go @@ -0,0 +1,88 @@ +package docker + +import ( + "strings" + "testing" +) + +const Dockerfile = ` +# VERSION 0.1 +# DOCKER-VERSION 0.1.6 + +from docker-ut +run sh -c 'echo root:testpass > /tmp/passwd' +run mkdir -p /var/run/sshd +copy https://raw.github.com/dotcloud/docker/master/CHANGELOG.md /tmp/CHANGELOG.md +` + +func TestBuild(t *testing.T) { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) + + builder := NewBuilder(runtime) + + img, err := builder.Build(strings.NewReader(Dockerfile), &nopWriter{}) + if err != nil { + t.Fatal(err) + } + + container, err := builder.Create( + &Config{ + Image: img.Id, + Cmd: []string{"cat", "/tmp/passwd"}, + }, + ) + if err != nil { + t.Fatal(err) + } + defer runtime.Destroy(container) + + output, err := container.Output() + if err != nil { + t.Fatal(err) + } + if string(output) != "root:testpass\n" { + t.Fatalf("Unexpected output. Read '%s', expected '%s'", output, "root:testpass\n") + } + + container2, err := builder.Create( + &Config{ + Image: img.Id, + Cmd: []string{"ls", "-d", "/var/run/sshd"}, + }, + ) + if err != nil { + t.Fatal(err) + } + defer runtime.Destroy(container2) + + output, err = container2.Output() + if err != nil { + t.Fatal(err) + } + if string(output) != "/var/run/sshd\n" { + t.Fatal("/var/run/sshd has not been created") + } + + container3, err := builder.Create( + &Config{ + Image: img.Id, + Cmd: []string{"cat", "/tmp/CHANGELOG.md"}, + }, + ) + if err != nil { + t.Fatal(err) + } + defer runtime.Destroy(container3) + + output, err = container3.Output() + if err != nil { + t.Fatal(err) + } + if len(output) == 0 { + t.Fatal("/tmp/CHANGELOG.md has not been copied") + } +} From 4390a3182f9ea4fc1e215917c2c6494d6fc453ac Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Sat, 27 Apr 2013 21:45:05 -0700 Subject: [PATCH 65/97] Fix image pipe with Builder COPY --- builder.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/builder.go b/builder.go index 6c389b34b4..5ede4ae5b2 100644 --- a/builder.go +++ b/builder.go @@ -235,6 +235,9 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e return nil, err } fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) + + image = base + break default: fmt.Fprintf(stdout, "Skipping unknown op %s\n", tmp[0]) From 034c7a7a5e3838efb1fd06f544ae9a8b21e1e1b9 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 30 Apr 2013 18:03:15 -0700 Subject: [PATCH 66/97] Remove the open from CmdBuild --- commands.go | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/commands.go b/commands.go index 3975d67671..08f9f7e18e 100644 --- a/commands.go +++ b/commands.go @@ -35,7 +35,7 @@ func (srv *Server) Help() string { help := "Usage: docker COMMAND [arg...]\n\nA self-sufficient runtime for linux containers.\n\nCommands:\n" for _, cmd := range [][]string{ {"attach", "Attach to a running container"}, - {"build", "Build a container from Dockerfile"}, + {"build", "Build a container from Dockerfile via stdin"}, {"commit", "Create a new image from a container's changes"}, {"diff", "Inspect changes on a container's filesystem"}, {"export", "Stream the contents of a container as a tar archive"}, @@ -116,28 +116,11 @@ func (srv *Server) CmdInsert(stdin io.ReadCloser, stdout rcli.DockerConn, args . func (srv *Server) CmdBuild(stdin io.ReadCloser, stdout rcli.DockerConn, args ...string) error { stdout.Flush() - cmd := rcli.Subcmd(stdout, "build", "[Dockerfile|-]", "Build a container from Dockerfile") + cmd := rcli.Subcmd(stdout, "build", "-", "Build a container from Dockerfile via stdin") if err := cmd.Parse(args); err != nil { return nil } - dockerfile := cmd.Arg(0) - if dockerfile == "" { - dockerfile = "Dockerfile" - } - - var file io.Reader - - if dockerfile != "-" { - f, err := os.Open(dockerfile) - if err != nil { - return err - } - defer f.Close() - file = f - } else { - file = stdin - } - img, err := NewBuilder(srv.runtime).Build(file, stdout) + img, err := NewBuilder(srv.runtime).Build(stdin, stdout) if err != nil { return err } From bbb634a98068171b6b6f4bbb9c7160284ef252e4 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 1 May 2013 13:37:32 -0700 Subject: [PATCH 67/97] Add doc for the builder --- docs/sources/builder/basics.rst | 91 ++++++++++++++++++++++ docs/sources/builder/index.rst | 14 ++++ docs/sources/commandline/cli.rst | 1 + docs/sources/commandline/command/build.rst | 9 +++ docs/sources/index.rst | 3 +- 5 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 docs/sources/builder/basics.rst create mode 100644 docs/sources/builder/index.rst create mode 100644 docs/sources/commandline/command/build.rst diff --git a/docs/sources/builder/basics.rst b/docs/sources/builder/basics.rst new file mode 100644 index 0000000000..a233515ef2 --- /dev/null +++ b/docs/sources/builder/basics.rst @@ -0,0 +1,91 @@ +============== +Docker Builder +============== + +.. contents:: Table of Contents + +1. Format +========= + +The Docker builder format is quite simple: + + ``instruction arguments`` + +The first instruction must be `FROM` + +All instruction are to be placed in a file named `Dockerfile` + +In order to place comments within a Dockerfile, simply prefix the line with "`#`" + +2. Instructions +=============== + +Docker builder comes with a set of instructions: + +1. FROM: Set from what image to build +2. RUN: Execute a command +3. INSERT: Insert a remote file (http) into the image + +2.1 FROM +-------- + ``FROM `` + +The `FROM` instruction must be the first one in order for Builder to know from where to run commands. + +`FROM` can also be used in order to build multiple images within a single Dockerfile + +2.2 RUN +------- + ``RUN `` + +The `RUN` instruction is the main one, it allows you to execute any commands on the `FROM` image and to save the results. +You can use as many `RUN` as you want within a Dockerfile, the commands will be executed on the result of the previous command. + +2.3 INSERT +---------- + + ``INSERT `` + +The `INSERT` instruction will download the file at the given url and place it within the image at the given path. + +.. note:: + The path must include the file name. + +3. Dockerfile Examples +====================== + +:: + + # Nginx + # + # VERSION 0.0.1 + # DOCKER-VERSION 0.2 + + from ubuntu + + # make sure the package repository is up to date + run echo "deb http://archive.ubuntu.com/ubuntu precise main universe" > /etc/apt/sources.list + run apt-get update + + run apt-get install -y inotify-tools nginx apache openssh-server + insert https://raw.github.com/creack/docker-vps/master/nginx-wrapper.sh /usr/sbin/nginx-wrapper + +:: + + # Firefox over VNC + # + # VERSION 0.3 + # DOCKER-VERSION 0.2 + + from ubuntu + # make sure the package repository is up to date + run echo "deb http://archive.ubuntu.com/ubuntu precise main universe" > /etc/apt/sources.list + run apt-get update + + # Install vnc, xvfb in order to create a 'fake' display and firefox + run apt-get install -y x11vnc xvfb firefox + run mkdir /.vnc + # Setup a password + run x11vnc -storepasswd 1234 ~/.vnc/passwd + # Autostart firefox (might not be the best way to do it, but it does the trick) + run bash -c 'echo "firefox" >> /.bashrc' diff --git a/docs/sources/builder/index.rst b/docs/sources/builder/index.rst new file mode 100644 index 0000000000..170be1a5ab --- /dev/null +++ b/docs/sources/builder/index.rst @@ -0,0 +1,14 @@ +:title: docker documentation +:description: Documentation for docker builder +:keywords: docker, builder, dockerfile + + +Builder +======= + +Contents: + +.. toctree:: + :maxdepth: 2 + + basics diff --git a/docs/sources/commandline/cli.rst b/docs/sources/commandline/cli.rst index 2657b91777..46ea3e4a7f 100644 --- a/docs/sources/commandline/cli.rst +++ b/docs/sources/commandline/cli.rst @@ -27,6 +27,7 @@ Available Commands :maxdepth: 1 command/attach + command/build command/commit command/diff command/export diff --git a/docs/sources/commandline/command/build.rst b/docs/sources/commandline/command/build.rst new file mode 100644 index 0000000000..6415f11f7b --- /dev/null +++ b/docs/sources/commandline/command/build.rst @@ -0,0 +1,9 @@ +=========================================== +``build`` -- Build a container from Dockerfile via stdin +=========================================== + +:: + + Usage: docker build - + Example: cat Dockerfile | docker build - + Build a new image from the Dockerfile passed via stdin diff --git a/docs/sources/index.rst b/docs/sources/index.rst index e6a1482ccd..9a272d2a34 100644 --- a/docs/sources/index.rst +++ b/docs/sources/index.rst @@ -17,7 +17,8 @@ This documentation has the following resources: commandline/index registry/index index/index + builder/index faq -.. image:: http://www.docker.io/_static/lego_docker.jpg \ No newline at end of file +.. image:: http://www.docker.io/_static/lego_docker.jpg From 6bfb652f5b1bce989868f8269c3d736f3faf259a Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 1 May 2013 13:45:35 -0700 Subject: [PATCH 68/97] Change dockerbulder format, no more tabs and COPY becomes INSERT to avoid conflict with contrib script --- builder.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/builder.go b/builder.go index 5ede4ae5b2..e252d3d8c0 100644 --- a/builder.go +++ b/builder.go @@ -142,7 +142,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e if len(line) == 0 || line[0] == '#' { continue } - tmp := strings.SplitN(line, " ", 2) + tmp := strings.SplitN(line, " ", 2) if len(tmp) != 2 { return nil, fmt.Errorf("Invalid Dockerfile format") } @@ -192,13 +192,13 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e image = base break - case "copy": + case "insert": if image == nil { return nil, fmt.Errorf("Please provide a source image with `from` prior to copy") } tmp2 := strings.SplitN(tmp[1], " ", 2) if len(tmp) != 2 { - return nil, fmt.Errorf("Invalid COPY format") + return nil, fmt.Errorf("Invalid INSERT format") } fmt.Fprintf(stdout, "COPY %s to %s in %s\n", tmp2[0], tmp2[1], base.ShortId()) @@ -240,7 +240,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e break default: - fmt.Fprintf(stdout, "Skipping unknown op %s\n", tmp[0]) + fmt.Fprintf(stdout, "Skipping unknown instruction %s\n", instruction) } } if base != nil { From 4386edff0bc940543c61aa91d1f2144802ce1a08 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 1 May 2013 13:45:50 -0700 Subject: [PATCH 69/97] Better varibale names --- builder.go | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/builder.go b/builder.go index e252d3d8c0..62a7731b4b 100644 --- a/builder.go +++ b/builder.go @@ -146,20 +146,22 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e if len(tmp) != 2 { return nil, fmt.Errorf("Invalid Dockerfile format") } - switch tmp[0] { + instruction := tmp[0] + arguments := tmp[1] + switch strings.ToLower(instruction) { case "from": - fmt.Fprintf(stdout, "FROM %s\n", tmp[1]) - image, err = builder.runtime.repositories.LookupImage(tmp[1]) + fmt.Fprintf(stdout, "FROM %s\n", arguments) + image, err = builder.runtime.repositories.LookupImage(arguments) if err != nil { return nil, err } break case "run": - fmt.Fprintf(stdout, "RUN %s\n", tmp[1]) + fmt.Fprintf(stdout, "RUN %s\n", arguments) if image == nil { return nil, fmt.Errorf("Please provide a source image with `from` prior to run") } - config, err := ParseRun([]string{image.Id, "/bin/sh", "-c", tmp[1]}, nil, builder.runtime.capabilities) + config, err := ParseRun([]string{image.Id, "/bin/sh", "-c", arguments}, nil, builder.runtime.capabilities) if err != nil { return nil, err } @@ -176,7 +178,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e // Wait for it to finish if result := c.Wait(); result != 0 { - return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", tmp[1], result) + return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", arguments, result) } // Commit the container @@ -196,19 +198,21 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e if image == nil { return nil, fmt.Errorf("Please provide a source image with `from` prior to copy") } - tmp2 := strings.SplitN(tmp[1], " ", 2) + tmp = strings.SplitN(arguments, " ", 2) if len(tmp) != 2 { return nil, fmt.Errorf("Invalid INSERT format") } - fmt.Fprintf(stdout, "COPY %s to %s in %s\n", tmp2[0], tmp2[1], base.ShortId()) + sourceUrl := tmp[0] + destPath := tmp[1] + fmt.Fprintf(stdout, "COPY %s to %s in %s\n", sourceUrl, destPath, base.ShortId()) - file, err := Download(tmp2[0], stdout) + file, err := Download(sourceUrl, stdout) if err != nil { return nil, err } defer file.Body.Close() - config, err := ParseRun([]string{base.Id, "echo", "insert", tmp2[0], tmp2[1]}, nil, builder.runtime.capabilities) + config, err := ParseRun([]string{base.Id, "echo", "insert", sourceUrl, destPath}, nil, builder.runtime.capabilities) if err != nil { return nil, err } @@ -223,10 +227,10 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e // Wait for echo to finish if result := c.Wait(); result != 0 { - return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", tmp[1], result) + return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", arguments, result) } - if err := c.Inject(file.Body, tmp2[1]); err != nil { + if err := c.Inject(file.Body, destPath); err != nil { return nil, err } From 6c168a8986261c95ae5e036f008375f77afbe78e Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 1 May 2013 14:36:45 -0700 Subject: [PATCH 70/97] Rebase master (autorun) --- builder.go | 53 +++++++++++++++++++++-- commands.go | 2 +- container_test.go | 7 +-- runtime.go | 107 ---------------------------------------------- 4 files changed, 54 insertions(+), 115 deletions(-) diff --git a/builder.go b/builder.go index 62a7731b4b..9ecf5f4e8e 100644 --- a/builder.go +++ b/builder.go @@ -24,12 +24,57 @@ func NewBuilder(runtime *Runtime) *Builder { } } +func (builder *Builder) mergeConfig(userConf, imageConf *Config) { + if userConf.Hostname != "" { + userConf.Hostname = imageConf.Hostname + } + if userConf.User != "" { + userConf.User = imageConf.User + } + if userConf.Memory == 0 { + userConf.Memory = imageConf.Memory + } + if userConf.MemorySwap == 0 { + userConf.MemorySwap = imageConf.MemorySwap + } + if userConf.PortSpecs == nil || len(userConf.PortSpecs) == 0 { + userConf.PortSpecs = imageConf.PortSpecs + } + if !userConf.Tty { + userConf.Tty = userConf.Tty + } + if !userConf.OpenStdin { + userConf.OpenStdin = imageConf.OpenStdin + } + if !userConf.StdinOnce { + userConf.StdinOnce = imageConf.StdinOnce + } + if userConf.Env == nil || len(userConf.Env) == 0 { + userConf.Env = imageConf.Env + } + if userConf.Cmd == nil || len(userConf.Cmd) == 0 { + userConf.Cmd = imageConf.Cmd + } + if userConf.Dns == nil || len(userConf.Dns) == 0 { + userConf.Dns = imageConf.Dns + } +} + func (builder *Builder) Create(config *Config) (*Container, error) { // Lookup image img, err := builder.repositories.LookupImage(config.Image) if err != nil { return nil, err } + + if img.Config != nil { + builder.mergeConfig(config, img.Config) + } + + if config.Cmd == nil { + return nil, fmt.Errorf("No command specified") + } + // Generate id id := GenerateId() // Generate default hostname @@ -87,7 +132,7 @@ func (builder *Builder) Create(config *Config) (*Container, error) { // Commit creates a new filesystem image from the current state of a container. // The image can optionally be tagged into a repository -func (builder *Builder) Commit(container *Container, repository, tag, comment, author string) (*Image, error) { +func (builder *Builder) Commit(container *Container, repository, tag, comment, author string, config *Config) (*Image, error) { // FIXME: freeze the container before copying it to avoid data corruption? // FIXME: this shouldn't be in commands. rwTar, err := container.ExportRw() @@ -95,7 +140,7 @@ func (builder *Builder) Commit(container *Container, repository, tag, comment, a return nil, err } // Create a new image from the container's base layers + a new layer from container changes - img, err := builder.graph.Create(rwTar, container, comment, author) + img, err := builder.graph.Create(rwTar, container, comment, author, config) if err != nil { return nil, err } @@ -182,7 +227,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e } // Commit the container - base, err = builder.Commit(c, "", "", "", "") + base, err = builder.Commit(c, "", "", "", "", nil) if err != nil { return nil, err } @@ -234,7 +279,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e return nil, err } - base, err = builder.Commit(c, "", "", "", "") + base, err = builder.Commit(c, "", "", "", "", nil) if err != nil { return nil, err } diff --git a/commands.go b/commands.go index 08f9f7e18e..c72e24aa61 100644 --- a/commands.go +++ b/commands.go @@ -106,7 +106,7 @@ func (srv *Server) CmdInsert(stdin io.ReadCloser, stdout rcli.DockerConn, args . return err } // FIXME: Handle custom repo, tag comment, author - img, err = b.Commit(c, "", "", img.Comment, img.Author) + img, err = b.Commit(c, "", "", img.Comment, img.Author, nil) if err != nil { return err } diff --git a/container_test.go b/container_test.go index c06b8de876..5b63b2a0e7 100644 --- a/container_test.go +++ b/container_test.go @@ -225,7 +225,9 @@ func TestCommitAutoRun(t *testing.T) { t.Fatal(err) } defer nuke(runtime) - container1, err := runtime.Create( + + builder := NewBuilder(runtime) + container1, err := builder.Create( &Config{ Image: GetTestImage(runtime).Id, Cmd: []string{"/bin/sh", "-c", "echo hello > /world"}, @@ -256,8 +258,7 @@ func TestCommitAutoRun(t *testing.T) { } // FIXME: Make a TestCommit that stops here and check docker.root/layers/img.id/world - - container2, err := runtime.Create( + container2, err := builder.Create( &Config{ Image: img.Id, }, diff --git a/runtime.go b/runtime.go index ca27b7a7bf..5958aa1811 100644 --- a/runtime.go +++ b/runtime.go @@ -78,113 +78,6 @@ func (runtime *Runtime) containerRoot(id string) string { return path.Join(runtime.repository, id) } -func (runtime *Runtime) mergeConfig(userConf, imageConf *Config) { - if userConf.Hostname == "" { - userConf.Hostname = imageConf.Hostname - } - if userConf.User == "" { - userConf.User = imageConf.User - } - if userConf.Memory == 0 { - userConf.Memory = imageConf.Memory - } - if userConf.MemorySwap == 0 { - userConf.MemorySwap = imageConf.MemorySwap - } - if userConf.PortSpecs == nil || len(userConf.PortSpecs) == 0 { - userConf.PortSpecs = imageConf.PortSpecs - } - if !userConf.Tty { - userConf.Tty = userConf.Tty - } - if !userConf.OpenStdin { - userConf.OpenStdin = imageConf.OpenStdin - } - if !userConf.StdinOnce { - userConf.StdinOnce = imageConf.StdinOnce - } - if userConf.Env == nil || len(userConf.Env) == 0 { - userConf.Env = imageConf.Env - } - if userConf.Cmd == nil || len(userConf.Cmd) == 0 { - userConf.Cmd = imageConf.Cmd - } - if userConf.Dns == nil || len(userConf.Dns) == 0 { - userConf.Dns = imageConf.Dns - } -} - -func (runtime *Runtime) Create(config *Config) (*Container, error) { - // Lookup image - img, err := runtime.repositories.LookupImage(config.Image) - if err != nil { - return nil, err - } - - if img.Config != nil { - runtime.mergeConfig(config, img.Config) - } - - if config.Cmd == nil || len(config.Cmd) == 0 { - return nil, fmt.Errorf("No command specified") - } - - // Generate id - id := GenerateId() - // Generate default hostname - // FIXME: the lxc template no longer needs to set a default hostname - if config.Hostname == "" { - config.Hostname = id[:12] - } - - container := &Container{ - // FIXME: we should generate the ID here instead of receiving it as an argument - Id: id, - Created: time.Now(), - Path: config.Cmd[0], - Args: config.Cmd[1:], //FIXME: de-duplicate from config - Config: config, - Image: img.Id, // Always use the resolved image id - NetworkSettings: &NetworkSettings{}, - // FIXME: do we need to store this in the container? - SysInitPath: sysInitPath, - } - container.root = runtime.containerRoot(container.Id) - // Step 1: create the container directory. - // This doubles as a barrier to avoid race conditions. - if err := os.Mkdir(container.root, 0700); err != nil { - return nil, err - } - - // If custom dns exists, then create a resolv.conf for the container - if len(config.Dns) > 0 { - container.ResolvConfPath = path.Join(container.root, "resolv.conf") - f, err := os.Create(container.ResolvConfPath) - if err != nil { - return nil, err - } - defer f.Close() - for _, dns := range config.Dns { - if _, err := f.Write([]byte("nameserver " + dns + "\n")); err != nil { - return nil, err - } - } - } else { - container.ResolvConfPath = "/etc/resolv.conf" - } - - // Step 2: save the container json - if err := container.ToDisk(); err != nil { - return nil, err - } - // Step 3: register the container - if err := runtime.Register(container); err != nil { - return nil, err - } - return container, nil -} - -======= end func (runtime *Runtime) Load(id string) (*Container, error) { container := &Container{root: runtime.containerRoot(id)} if err := container.FromDisk(); err != nil { From 15ea5a479a78d1011cda9bd55fd442d810234eb6 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 1 May 2013 14:37:26 -0700 Subject: [PATCH 71/97] Update the TestBuild with new format --- builder_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/builder_test.go b/builder_test.go index 345fd2c594..d7fa921e3d 100644 --- a/builder_test.go +++ b/builder_test.go @@ -7,12 +7,12 @@ import ( const Dockerfile = ` # VERSION 0.1 -# DOCKER-VERSION 0.1.6 +# DOCKER-VERSION 0.2 -from docker-ut -run sh -c 'echo root:testpass > /tmp/passwd' -run mkdir -p /var/run/sshd -copy https://raw.github.com/dotcloud/docker/master/CHANGELOG.md /tmp/CHANGELOG.md +from docker-ut +run sh -c 'echo root:testpass > /tmp/passwd' +run mkdir -p /var/run/sshd +insert https://raw.github.com/dotcloud/docker/master/CHANGELOG.md /tmp/CHANGELOG.md ` func TestBuild(t *testing.T) { From 4ebec08add53bd9b3e10c331168daa6579ebd6db Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 1 May 2013 15:05:52 -0700 Subject: [PATCH 72/97] Trim the splited builder lines --- builder.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/builder.go b/builder.go index 9ecf5f4e8e..feaa81256c 100644 --- a/builder.go +++ b/builder.go @@ -191,8 +191,8 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e if len(tmp) != 2 { return nil, fmt.Errorf("Invalid Dockerfile format") } - instruction := tmp[0] - arguments := tmp[1] + instruction := strings.Trim(tmp[0], " ") + arguments := strings.Trim(tmp[1], " ") switch strings.ToLower(instruction) { case "from": fmt.Fprintf(stdout, "FROM %s\n", arguments) @@ -247,8 +247,8 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e if len(tmp) != 2 { return nil, fmt.Errorf("Invalid INSERT format") } - sourceUrl := tmp[0] - destPath := tmp[1] + sourceUrl := strings.Trim(tmp[0], "") + destPath := strings.Trim(tmp[1], " ") fmt.Fprintf(stdout, "COPY %s to %s in %s\n", sourceUrl, destPath, base.ShortId()) file, err := Download(sourceUrl, stdout) From 924b61328cd63fc8b1da77d9cb4f1e6b85eb1592 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 1 May 2013 15:06:23 -0700 Subject: [PATCH 73/97] Make the FROM instruction pull the image if not existing --- builder.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/builder.go b/builder.go index feaa81256c..8d94c40f95 100644 --- a/builder.go +++ b/builder.go @@ -198,8 +198,27 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e fmt.Fprintf(stdout, "FROM %s\n", arguments) image, err = builder.runtime.repositories.LookupImage(arguments) if err != nil { - return nil, err + if builder.runtime.graph.IsNotExist(err) { + if builder.runtime.graph.LookupRemoteImage(arguments, builder.runtime.authConfig) { + if err := builder.runtime.graph.PullImage(stdout, arguments, builder.runtime.authConfig); err != nil { + return nil, err + } + } else { + // FIXME: Allow pull repo:tag + if err := builder.runtime.graph.PullRepository(stdout, arguments, "", builder.runtime.repositories, builder.runtime.authConfig); err != nil { + return nil, err + } + } + + image, err = builder.runtime.repositories.LookupImage(arguments) + if err != nil { + return nil, err + } + } else { + return nil, err + } } + break case "run": fmt.Fprintf(stdout, "RUN %s\n", arguments) From 6d6a03dfba37a8f1bd7304467db86ebe248ed2b5 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 1 May 2013 15:06:40 -0700 Subject: [PATCH 74/97] More consistent docker build test --- builder_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder_test.go b/builder_test.go index d7fa921e3d..08b7dd58cc 100644 --- a/builder_test.go +++ b/builder_test.go @@ -9,7 +9,7 @@ const Dockerfile = ` # VERSION 0.1 # DOCKER-VERSION 0.2 -from docker-ut +from ` + unitTestImageName + ` run sh -c 'echo root:testpass > /tmp/passwd' run mkdir -p /var/run/sshd insert https://raw.github.com/dotcloud/docker/master/CHANGELOG.md /tmp/CHANGELOG.md From 92e98c66af7543f9a9d6c1a223333201b39aa6dc Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 1 May 2013 16:26:26 -0700 Subject: [PATCH 75/97] Implement MAINTAINER to builder --- builder.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/builder.go b/builder.go index 8d94c40f95..6ab3a6f84e 100644 --- a/builder.go +++ b/builder.go @@ -168,6 +168,7 @@ func (builder *Builder) clearTmp(containers, images map[string]struct{}) { func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, error) { var ( image, base *Image + maintainer string tmpContainers map[string]struct{} = make(map[string]struct{}) tmpImages map[string]struct{} = make(map[string]struct{}) ) @@ -219,6 +220,10 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e } } + break + case "mainainer": + fmt.Fprintf(stdout, "MAINTAINER %s\n", arguments) + maintainer = arguments break case "run": fmt.Fprintf(stdout, "RUN %s\n", arguments) @@ -246,7 +251,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e } // Commit the container - base, err = builder.Commit(c, "", "", "", "", nil) + base, err = builder.Commit(c, "", "", "", maintainer, nil) if err != nil { return nil, err } @@ -266,7 +271,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e if len(tmp) != 2 { return nil, fmt.Errorf("Invalid INSERT format") } - sourceUrl := strings.Trim(tmp[0], "") + sourceUrl := strings.Trim(tmp[0], " ") destPath := strings.Trim(tmp[1], " ") fmt.Fprintf(stdout, "COPY %s to %s in %s\n", sourceUrl, destPath, base.ShortId()) @@ -298,7 +303,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e return nil, err } - base, err = builder.Commit(c, "", "", "", "", nil) + base, err = builder.Commit(c, "", "", "", maintainer, nil) if err != nil { return nil, err } From ae1e655fb1d95b5828a2b0cd67d19d090d04ff74 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 1 May 2013 16:26:46 -0700 Subject: [PATCH 76/97] Implement EXPOSE to builder --- builder.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/builder.go b/builder.go index 6ab3a6f84e..e49f7d4527 100644 --- a/builder.go +++ b/builder.go @@ -262,6 +262,35 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e // use the base as the new image image = base + break + case "expose": + ports := strings.Split(arguments, " ") + + fmt.Fprintf(stdout, "EXPOSE %v\n", ports) + if image == nil { + return nil, fmt.Errorf("Please provide a source image with `from` prior to copy") + } + + // Create the container and start it + c, err := builder.Create(&Config{Image: image.Id, Cmd: []string{"", ""}}) + if err != nil { + return nil, err + } + if err := c.Start(); err != nil { + return nil, err + } + tmpContainers[c.Id] = struct{}{} + + // Commit the container + base, err = builder.Commit(c, "", "", "", maintainer, &Config{PortSpecs: ports}) + if err != nil { + return nil, err + } + tmpImages[base.Id] = struct{}{} + + fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) + + image = base break case "insert": if image == nil { From 2bc4ad940259bd46f0ff4d3c9462893004188416 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Sun, 5 May 2013 06:27:15 -0700 Subject: [PATCH 77/97] Rebase fix --- commands.go | 1 - 1 file changed, 1 deletion(-) diff --git a/commands.go b/commands.go index c72e24aa61..442d7f55ee 100644 --- a/commands.go +++ b/commands.go @@ -11,7 +11,6 @@ import ( "net/http" "net/url" "path/filepath" - "os" "runtime" "strconv" "strings" From 62a1850c16180ac83418e044e315ba8776f6a651 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 6 May 2013 16:40:45 -0700 Subject: [PATCH 78/97] Make the autopull compatible with new registry --- builder.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/builder.go b/builder.go index e49f7d4527..72989a8a0a 100644 --- a/builder.go +++ b/builder.go @@ -200,21 +200,23 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e image, err = builder.runtime.repositories.LookupImage(arguments) if err != nil { if builder.runtime.graph.IsNotExist(err) { - if builder.runtime.graph.LookupRemoteImage(arguments, builder.runtime.authConfig) { - if err := builder.runtime.graph.PullImage(stdout, arguments, builder.runtime.authConfig); err != nil { - return nil, err - } - } else { - // FIXME: Allow pull repo:tag - if err := builder.runtime.graph.PullRepository(stdout, arguments, "", builder.runtime.repositories, builder.runtime.authConfig); err != nil { - return nil, err - } + + var tag, remote string + if strings.Contains(remote, ":") { + remoteParts := strings.Split(remote, ":") + tag = remoteParts[1] + remote = remoteParts[0] + } + + if err := builder.runtime.graph.PullRepository(stdout, remote, tag, builder.runtime.repositories, builder.runtime.authConfig); err != nil { + return nil, err } image, err = builder.runtime.repositories.LookupImage(arguments) if err != nil { return nil, err } + } else { return nil, err } From e2880950c5ea721be00a814d93ea256feb9f98bb Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 11:03:01 -0700 Subject: [PATCH 79/97] Add build command --- commands.go | 1 + 1 file changed, 1 insertion(+) diff --git a/commands.go b/commands.go index 442d7f55ee..8d8046c0f5 100644 --- a/commands.go +++ b/commands.go @@ -10,6 +10,7 @@ import ( "log" "net/http" "net/url" + "os" "path/filepath" "runtime" "strconv" From ebb59c1125d194716f26b8cc53863fa9910c4b3f Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 30 Apr 2013 18:03:15 -0700 Subject: [PATCH 80/97] Remove the open from CmdBuild --- commands.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/commands.go b/commands.go index 8d8046c0f5..d9756c9d88 100644 --- a/commands.go +++ b/commands.go @@ -10,8 +10,6 @@ import ( "log" "net/http" "net/url" - "os" - "path/filepath" "runtime" "strconv" "strings" From d92166cc7937afa39e2ed537c601555a0a5b84eb Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Sat, 4 May 2013 20:58:45 -0700 Subject: [PATCH 81/97] Fix merge issue --- commands.go | 1 + 1 file changed, 1 insertion(+) diff --git a/commands.go b/commands.go index d9756c9d88..442d7f55ee 100644 --- a/commands.go +++ b/commands.go @@ -10,6 +10,7 @@ import ( "log" "net/http" "net/url" + "path/filepath" "runtime" "strconv" "strings" From 96069de4e0818f9d513f6ea495e03d8e67eb1e98 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 11:03:01 -0700 Subject: [PATCH 82/97] Add build command --- builder.go | 308 +++++++--------------------------------------------- commands.go | 1 + 2 files changed, 39 insertions(+), 270 deletions(-) diff --git a/builder.go b/builder.go index 72989a8a0a..4de71e017b 100644 --- a/builder.go +++ b/builder.go @@ -4,175 +4,60 @@ import ( "bufio" "fmt" "io" - "os" - "path" "strings" - "time" ) type Builder struct { - runtime *Runtime - repositories *TagStore - graph *Graph + runtime *Runtime } func NewBuilder(runtime *Runtime) *Builder { return &Builder{ - runtime: runtime, - graph: runtime.graph, - repositories: runtime.repositories, + runtime: runtime, } } -func (builder *Builder) mergeConfig(userConf, imageConf *Config) { - if userConf.Hostname != "" { - userConf.Hostname = imageConf.Hostname +func (builder *Builder) run(image *Image, cmd string) (*Container, error) { + // FIXME: pass a NopWriter instead of nil + config, err := ParseRun([]string{"-d", image.Id, "/bin/sh", "-c", cmd}, nil, builder.runtime.capabilities) + if config.Image == "" { + return nil, fmt.Errorf("Image not specified") } - if userConf.User != "" { - userConf.User = imageConf.User + if len(config.Cmd) == 0 { + return nil, fmt.Errorf("Command not specified") } - if userConf.Memory == 0 { - userConf.Memory = imageConf.Memory + if config.Tty { + return nil, fmt.Errorf("The tty mode is not supported within the builder") } - if userConf.MemorySwap == 0 { - userConf.MemorySwap = imageConf.MemorySwap - } - if userConf.PortSpecs == nil || len(userConf.PortSpecs) == 0 { - userConf.PortSpecs = imageConf.PortSpecs - } - if !userConf.Tty { - userConf.Tty = userConf.Tty - } - if !userConf.OpenStdin { - userConf.OpenStdin = imageConf.OpenStdin - } - if !userConf.StdinOnce { - userConf.StdinOnce = imageConf.StdinOnce - } - if userConf.Env == nil || len(userConf.Env) == 0 { - userConf.Env = imageConf.Env - } - if userConf.Cmd == nil || len(userConf.Cmd) == 0 { - userConf.Cmd = imageConf.Cmd - } - if userConf.Dns == nil || len(userConf.Dns) == 0 { - userConf.Dns = imageConf.Dns - } -} -func (builder *Builder) Create(config *Config) (*Container, error) { - // Lookup image - img, err := builder.repositories.LookupImage(config.Image) + // Create new container + container, err := builder.runtime.Create(config) if err != nil { return nil, err } - - if img.Config != nil { - builder.mergeConfig(config, img.Config) - } - - if config.Cmd == nil { - return nil, fmt.Errorf("No command specified") - } - - // Generate id - id := GenerateId() - // Generate default hostname - // FIXME: the lxc template no longer needs to set a default hostname - if config.Hostname == "" { - config.Hostname = id[:12] - } - - container := &Container{ - // FIXME: we should generate the ID here instead of receiving it as an argument - Id: id, - Created: time.Now(), - Path: config.Cmd[0], - Args: config.Cmd[1:], //FIXME: de-duplicate from config - Config: config, - Image: img.Id, // Always use the resolved image id - NetworkSettings: &NetworkSettings{}, - // FIXME: do we need to store this in the container? - SysInitPath: sysInitPath, - } - container.root = builder.runtime.containerRoot(container.Id) - // Step 1: create the container directory. - // This doubles as a barrier to avoid race conditions. - if err := os.Mkdir(container.root, 0700); err != nil { - return nil, err - } - - // If custom dns exists, then create a resolv.conf for the container - if len(config.Dns) > 0 { - container.ResolvConfPath = path.Join(container.root, "resolv.conf") - f, err := os.Create(container.ResolvConfPath) - if err != nil { - return nil, err - } - defer f.Close() - for _, dns := range config.Dns { - if _, err := f.Write([]byte("nameserver " + dns + "\n")); err != nil { - return nil, err - } - } - } else { - container.ResolvConfPath = "/etc/resolv.conf" - } - - // Step 2: save the container json - if err := container.ToDisk(); err != nil { - return nil, err - } - // Step 3: register the container - if err := builder.runtime.Register(container); err != nil { + if err := container.Start(); err != nil { return nil, err } return container, nil } -// Commit creates a new filesystem image from the current state of a container. -// The image can optionally be tagged into a repository -func (builder *Builder) Commit(container *Container, repository, tag, comment, author string, config *Config) (*Image, error) { - // FIXME: freeze the container before copying it to avoid data corruption? - // FIXME: this shouldn't be in commands. - rwTar, err := container.ExportRw() +func (builder *Builder) runCommit(image *Image, cmd string) (*Image, error) { + c, err := builder.run(image, cmd) if err != nil { return nil, err } - // Create a new image from the container's base layers + a new layer from container changes - img, err := builder.graph.Create(rwTar, container, comment, author, config) + if result := c.Wait(); result != 0 { + return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", cmd, result) + } + img, err := builder.runtime.Commit(c.Id, "", "", "", "") if err != nil { return nil, err } - // Register the image if needed - if repository != "" { - if err := builder.repositories.Set(repository, tag, img.Id, true); err != nil { - return img, err - } - } return img, nil } -func (builder *Builder) clearTmp(containers, images map[string]struct{}) { - for c := range containers { - tmp := builder.runtime.Get(c) - builder.runtime.Destroy(tmp) - Debugf("Removing container %s", c) - } - for i := range images { - builder.runtime.graph.Delete(i) - Debugf("Removing image %s", i) - } -} - -func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, error) { - var ( - image, base *Image - maintainer string - tmpContainers map[string]struct{} = make(map[string]struct{}) - tmpImages map[string]struct{} = make(map[string]struct{}) - ) - defer builder.clearTmp(tmpContainers, tmpImages) +func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { + var image, base *Image file := bufio.NewReader(dockerfile) for { @@ -181,23 +66,21 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e if err == io.EOF { break } - return nil, err + return err } line = strings.TrimSpace(line) // Skip comments and empty line if len(line) == 0 || line[0] == '#' { continue } - tmp := strings.SplitN(line, " ", 2) + tmp := strings.SplitN(line, " ", 2) if len(tmp) != 2 { - return nil, fmt.Errorf("Invalid Dockerfile format") + return fmt.Errorf("Invalid Dockerfile format") } - instruction := strings.Trim(tmp[0], " ") - arguments := strings.Trim(tmp[1], " ") - switch strings.ToLower(instruction) { + switch tmp[0] { case "from": - fmt.Fprintf(stdout, "FROM %s\n", arguments) - image, err = builder.runtime.repositories.LookupImage(arguments) + fmt.Fprintf(stdout, "FROM %s\n", tmp[1]) + image, err = builder.runtime.repositories.LookupImage(tmp[1]) if err != nil { if builder.runtime.graph.IsNotExist(err) { @@ -221,143 +104,28 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e return nil, err } } - - break - case "mainainer": - fmt.Fprintf(stdout, "MAINTAINER %s\n", arguments) - maintainer = arguments break case "run": - fmt.Fprintf(stdout, "RUN %s\n", arguments) + fmt.Fprintf(stdout, "RUN %s\n", tmp[1]) if image == nil { - return nil, fmt.Errorf("Please provide a source image with `from` prior to run") + return fmt.Errorf("Please provide a source image with `from` prior to run") } - config, err := ParseRun([]string{image.Id, "/bin/sh", "-c", arguments}, nil, builder.runtime.capabilities) + base, err = builder.runCommit(image, tmp[1]) if err != nil { - return nil, err + return err } - - // Create the container and start it - c, err := builder.Create(config) - if err != nil { - return nil, err - } - if err := c.Start(); err != nil { - return nil, err - } - tmpContainers[c.Id] = struct{}{} - - // Wait for it to finish - if result := c.Wait(); result != 0 { - return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", arguments, result) - } - - // Commit the container - base, err = builder.Commit(c, "", "", "", maintainer, nil) - if err != nil { - return nil, err - } - tmpImages[base.Id] = struct{}{} - - fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) - - // use the base as the new image - image = base - - break - case "expose": - ports := strings.Split(arguments, " ") - - fmt.Fprintf(stdout, "EXPOSE %v\n", ports) - if image == nil { - return nil, fmt.Errorf("Please provide a source image with `from` prior to copy") - } - - // Create the container and start it - c, err := builder.Create(&Config{Image: image.Id, Cmd: []string{"", ""}}) - if err != nil { - return nil, err - } - if err := c.Start(); err != nil { - return nil, err - } - tmpContainers[c.Id] = struct{}{} - - // Commit the container - base, err = builder.Commit(c, "", "", "", maintainer, &Config{PortSpecs: ports}) - if err != nil { - return nil, err - } - tmpImages[base.Id] = struct{}{} - - fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) - - image = base - break - case "insert": - if image == nil { - return nil, fmt.Errorf("Please provide a source image with `from` prior to copy") - } - tmp = strings.SplitN(arguments, " ", 2) - if len(tmp) != 2 { - return nil, fmt.Errorf("Invalid INSERT format") - } - sourceUrl := strings.Trim(tmp[0], " ") - destPath := strings.Trim(tmp[1], " ") - fmt.Fprintf(stdout, "COPY %s to %s in %s\n", sourceUrl, destPath, base.ShortId()) - - file, err := Download(sourceUrl, stdout) - if err != nil { - return nil, err - } - defer file.Body.Close() - - config, err := ParseRun([]string{base.Id, "echo", "insert", sourceUrl, destPath}, nil, builder.runtime.capabilities) - if err != nil { - return nil, err - } - c, err := builder.Create(config) - if err != nil { - return nil, err - } - - if err := c.Start(); err != nil { - return nil, err - } - - // Wait for echo to finish - if result := c.Wait(); result != 0 { - return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", arguments, result) - } - - if err := c.Inject(file.Body, destPath); err != nil { - return nil, err - } - - base, err = builder.Commit(c, "", "", "", maintainer, nil) - if err != nil { - return nil, err - } - fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) - - image = base - + fmt.Fprintf(stdout, "===> %s\n", base.Id) break + case "copy": + return fmt.Errorf("The copy operator has not yet been implemented") default: - fmt.Fprintf(stdout, "Skipping unknown instruction %s\n", instruction) + fmt.Fprintf(stdout, "Skipping unknown op %s\n", tmp[0]) } } if base != nil { - // The build is successful, keep the temporary containers and images - for i := range tmpImages { - delete(tmpImages, i) - } - for i := range tmpContainers { - delete(tmpContainers, i) - } - fmt.Fprintf(stdout, "Build finished. image id: %s\n", base.ShortId()) + fmt.Fprintf(stdout, "Build finished. image id: %s\n", base.Id) } else { fmt.Fprintf(stdout, "An error occured during the build\n") } - return base, nil + return nil } diff --git a/commands.go b/commands.go index 442d7f55ee..c72e24aa61 100644 --- a/commands.go +++ b/commands.go @@ -11,6 +11,7 @@ import ( "net/http" "net/url" "path/filepath" + "os" "runtime" "strconv" "strings" From f911ccc27bcbcf281a0d690c5d7b05709c8947d4 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 15:14:10 -0700 Subject: [PATCH 83/97] Moving runtime.Create to builder.Create --- builder.go | 149 +++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 128 insertions(+), 21 deletions(-) diff --git a/builder.go b/builder.go index 4de71e017b..789f8642eb 100644 --- a/builder.go +++ b/builder.go @@ -4,38 +4,80 @@ import ( "bufio" "fmt" "io" + "os" + "path" "strings" + "time" ) type Builder struct { - runtime *Runtime + runtime *Runtime + repositories *TagStore } func NewBuilder(runtime *Runtime) *Builder { return &Builder{ - runtime: runtime, + runtime: runtime, + repositories: runtime.repositories, } } -func (builder *Builder) run(image *Image, cmd string) (*Container, error) { - // FIXME: pass a NopWriter instead of nil - config, err := ParseRun([]string{"-d", image.Id, "/bin/sh", "-c", cmd}, nil, builder.runtime.capabilities) - if config.Image == "" { - return nil, fmt.Errorf("Image not specified") - } - if len(config.Cmd) == 0 { - return nil, fmt.Errorf("Command not specified") - } - if config.Tty { - return nil, fmt.Errorf("The tty mode is not supported within the builder") - } - - // Create new container - container, err := builder.runtime.Create(config) +func (builder *Builder) Create(config *Config) (*Container, error) { + // Lookup image + img, err := builder.repositories.LookupImage(config.Image) if err != nil { return nil, err } - if err := container.Start(); err != nil { + // Generate id + id := GenerateId() + // Generate default hostname + // FIXME: the lxc template no longer needs to set a default hostname + if config.Hostname == "" { + config.Hostname = id[:12] + } + + container := &Container{ + // FIXME: we should generate the ID here instead of receiving it as an argument + Id: id, + Created: time.Now(), + Path: config.Cmd[0], + Args: config.Cmd[1:], //FIXME: de-duplicate from config + Config: config, + Image: img.Id, // Always use the resolved image id + NetworkSettings: &NetworkSettings{}, + // FIXME: do we need to store this in the container? + SysInitPath: sysInitPath, + } + container.root = builder.runtime.containerRoot(container.Id) + // Step 1: create the container directory. + // This doubles as a barrier to avoid race conditions. + if err := os.Mkdir(container.root, 0700); err != nil { + return nil, err + } + + // If custom dns exists, then create a resolv.conf for the container + if len(config.Dns) > 0 { + container.ResolvConfPath = path.Join(container.root, "resolv.conf") + f, err := os.Create(container.ResolvConfPath) + if err != nil { + return nil, err + } + defer f.Close() + for _, dns := range config.Dns { + if _, err := f.Write([]byte("nameserver " + dns + "\n")); err != nil { + return nil, err + } + } + } else { + container.ResolvConfPath = "/etc/resolv.conf" + } + + // Step 2: save the container json + if err := container.ToDisk(); err != nil { + return nil, err + } + // Step 3: register the container + if err := builder.runtime.Register(container); err != nil { return nil, err } return container, nil @@ -110,14 +152,79 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { if image == nil { return fmt.Errorf("Please provide a source image with `from` prior to run") } - base, err = builder.runCommit(image, tmp[1]) + config, err := ParseRun([]string{image.Id, "/bin/sh", "-c", tmp[1]}, nil, builder.runtime.capabilities) if err != nil { return err } - fmt.Fprintf(stdout, "===> %s\n", base.Id) + + // Create the container and start it + c, err := builder.Create(config) + if err != nil { + return err + } + if err := c.Start(); err != nil { + return err + } + tmpContainers[c.Id] = struct{}{} + + // Wait for it to finish + if result := c.Wait(); result != 0 { + return fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", tmp[1], result) + } + + // Commit the container + base, err = builder.Commit(c, "", "", "", "") + if err != nil { + return err + } + tmpImages[base.Id] = struct{}{} + + fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) break case "copy": - return fmt.Errorf("The copy operator has not yet been implemented") + if image == nil { + return fmt.Errorf("Please provide a source image with `from` prior to copy") + } + tmp2 := strings.SplitN(tmp[1], " ", 2) + if len(tmp) != 2 { + return fmt.Errorf("Invalid COPY format") + } + fmt.Fprintf(stdout, "COPY %s to %s in %s\n", tmp2[0], tmp2[1], base.ShortId()) + + file, err := Download(tmp2[0], stdout) + if err != nil { + return err + } + defer file.Body.Close() + + config, err := ParseRun([]string{base.Id, "echo", "insert", tmp2[0], tmp2[1]}, nil, builder.runtime.capabilities) + if err != nil { + return err + } + c, err := builder.Create(config) + if err != nil { + return err + } + + if err := c.Start(); err != nil { + return err + } + + // Wait for echo to finish + if result := c.Wait(); result != 0 { + return fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", tmp[1], result) + } + + if err := c.Inject(file.Body, tmp2[1]); err != nil { + return err + } + + base, err = builder.Commit(c, "", "", "", "") + if err != nil { + return err + } + fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) + break default: fmt.Fprintf(stdout, "Skipping unknown op %s\n", tmp[0]) } From 9959e2cd637d09e8c37891cebcb1764c825ef7d5 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 1 May 2013 14:36:45 -0700 Subject: [PATCH 84/97] Rebase master (autorun) --- builder.go | 63 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 8 deletions(-) diff --git a/builder.go b/builder.go index 789f8642eb..5ac4fa3c36 100644 --- a/builder.go +++ b/builder.go @@ -22,12 +22,57 @@ func NewBuilder(runtime *Runtime) *Builder { } } +func (builder *Builder) mergeConfig(userConf, imageConf *Config) { + if userConf.Hostname != "" { + userConf.Hostname = imageConf.Hostname + } + if userConf.User != "" { + userConf.User = imageConf.User + } + if userConf.Memory == 0 { + userConf.Memory = imageConf.Memory + } + if userConf.MemorySwap == 0 { + userConf.MemorySwap = imageConf.MemorySwap + } + if userConf.PortSpecs == nil || len(userConf.PortSpecs) == 0 { + userConf.PortSpecs = imageConf.PortSpecs + } + if !userConf.Tty { + userConf.Tty = userConf.Tty + } + if !userConf.OpenStdin { + userConf.OpenStdin = imageConf.OpenStdin + } + if !userConf.StdinOnce { + userConf.StdinOnce = imageConf.StdinOnce + } + if userConf.Env == nil || len(userConf.Env) == 0 { + userConf.Env = imageConf.Env + } + if userConf.Cmd == nil || len(userConf.Cmd) == 0 { + userConf.Cmd = imageConf.Cmd + } + if userConf.Dns == nil || len(userConf.Dns) == 0 { + userConf.Dns = imageConf.Dns + } +} + func (builder *Builder) Create(config *Config) (*Container, error) { // Lookup image img, err := builder.repositories.LookupImage(config.Image) if err != nil { return nil, err } + + if img.Config != nil { + builder.mergeConfig(config, img.Config) + } + + if config.Cmd == nil { + return nil, fmt.Errorf("No command specified") + } + // Generate id id := GenerateId() // Generate default hostname @@ -83,15 +128,17 @@ func (builder *Builder) Create(config *Config) (*Container, error) { return container, nil } -func (builder *Builder) runCommit(image *Image, cmd string) (*Image, error) { - c, err := builder.run(image, cmd) +// Commit creates a new filesystem image from the current state of a container. +// The image can optionally be tagged into a repository +func (builder *Builder) Commit(container *Container, repository, tag, comment, author string, config *Config) (*Image, error) { + // FIXME: freeze the container before copying it to avoid data corruption? + // FIXME: this shouldn't be in commands. + rwTar, err := container.ExportRw() if err != nil { return nil, err } - if result := c.Wait(); result != 0 { - return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", cmd, result) - } - img, err := builder.runtime.Commit(c.Id, "", "", "", "") + // Create a new image from the container's base layers + a new layer from container changes + img, err := builder.graph.Create(rwTar, container, comment, author, config) if err != nil { return nil, err } @@ -173,7 +220,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { } // Commit the container - base, err = builder.Commit(c, "", "", "", "") + base, err = builder.Commit(c, "", "", "", "", nil) if err != nil { return err } @@ -219,7 +266,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { return err } - base, err = builder.Commit(c, "", "", "", "") + base, err = builder.Commit(c, "", "", "", "", nil) if err != nil { return err } From a46fc3a59e585e005ec67b05211ca9de02f0c4ff Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 2 May 2013 00:49:23 -0700 Subject: [PATCH 85/97] Implement caching for docker builder --- builder.go | 75 +++++++++++++++++++++++++++++++++++++++++++++++++----- utils.go | 47 ++++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 7 deletions(-) diff --git a/builder.go b/builder.go index 5ac4fa3c36..e7f5e330f7 100644 --- a/builder.go +++ b/builder.go @@ -145,8 +145,55 @@ func (builder *Builder) Commit(container *Container, repository, tag, comment, a return img, nil } -func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { - var image, base *Image +func (builder *Builder) clearTmp(containers, images map[string]struct{}) { + for c := range containers { + tmp := builder.runtime.Get(c) + builder.runtime.Destroy(tmp) + Debugf("Removing container %s", c) + } + for i := range images { + builder.runtime.graph.Delete(i) + Debugf("Removing image %s", i) + } +} + +func (builder *Builder) getCachedImage(image *Image, config *Config) (*Image, error) { + // Retrieve all images + images, err := builder.graph.All() + if err != nil { + return nil, err + } + + // Store the tree in a map of map (map[parentId][childId]) + imageMap := make(map[string]map[string]struct{}) + for _, img := range images { + if _, exists := imageMap[img.Parent]; !exists { + imageMap[img.Parent] = make(map[string]struct{}) + } + imageMap[img.Parent][img.Id] = struct{}{} + } + + // Loop on the children of the given image and check the config + for elem := range imageMap[image.Id] { + img, err := builder.graph.Get(elem) + if err != nil { + return nil, err + } + if CompareConfig(&img.ContainerConfig, config) { + return img, nil + } + } + return nil, nil +} + +func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, error) { + var ( + image, base *Image + maintainer string + tmpContainers map[string]struct{} = make(map[string]struct{}) + tmpImages map[string]struct{} = make(map[string]struct{}) + ) + defer builder.clearTmp(tmpContainers, tmpImages) file := bufio.NewReader(dockerfile) for { @@ -204,6 +251,14 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { return err } + if cache, err := builder.getCachedImage(image, config); err != nil { + return nil, err + } else if cache != nil { + image = cache + fmt.Fprintf(stdout, "===> %s\n", image.ShortId()) + break + } + // Create the container and start it c, err := builder.Create(config) if err != nil { @@ -276,10 +331,16 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) error { fmt.Fprintf(stdout, "Skipping unknown op %s\n", tmp[0]) } } - if base != nil { - fmt.Fprintf(stdout, "Build finished. image id: %s\n", base.Id) - } else { - fmt.Fprintf(stdout, "An error occured during the build\n") + if image != nil { + // The build is successful, keep the temporary containers and images + for i := range tmpImages { + delete(tmpImages, i) + } + for i := range tmpContainers { + delete(tmpContainers, i) + } + fmt.Fprintf(stdout, "Build finished. image id: %s\n", image.ShortId()) + return image, nil } - return nil + return nil, fmt.Errorf("An error occured during the build\n") } diff --git a/utils.go b/utils.go index 047a29abef..095be2f4bf 100644 --- a/utils.go +++ b/utils.go @@ -474,3 +474,50 @@ func FindCgroupMountpoint(cgroupType string) (string, error) { return "", fmt.Errorf("cgroup mountpoint not found for %s", cgroupType) } + +// Compare two Config struct. Do not compare the "Image" nor "Hostname" fields +// If OpenStdin is set, then it differs +func CompareConfig(a, b *Config) bool { + if a == nil || b == nil || + a.OpenStdin || b.OpenStdin { + return false + } + if a.AttachStdout != b.AttachStdout || + a.AttachStderr != b.AttachStderr || + a.User != b.User || + a.Memory != b.Memory || + a.MemorySwap != b.MemorySwap || + a.OpenStdin != b.OpenStdin || + a.Tty != b.Tty { + return false + } + if len(a.Cmd) != len(b.Cmd) || + len(a.Dns) != len(b.Dns) || + len(a.Env) != len(b.Env) || + len(a.PortSpecs) != len(b.PortSpecs) { + return false + } + + for i := 0; i < len(a.Cmd); i++ { + if a.Cmd[i] != b.Cmd[i] { + return false + } + } + for i := 0; i < len(a.Dns); i++ { + if a.Dns[i] != b.Dns[i] { + return false + } + } + for i := 0; i < len(a.Env); i++ { + if a.Env[i] != b.Env[i] { + return false + } + } + for i := 0; i < len(a.PortSpecs); i++ { + if a.PortSpecs[i] != b.PortSpecs[i] { + return false + } + } + + return true +} From 756df27e45f2e5d9033058f60afa9547239af1d6 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 2 May 2013 01:18:48 -0700 Subject: [PATCH 86/97] Add compatibility with contrib builder --- builder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builder.go b/builder.go index e7f5e330f7..19c24c901e 100644 --- a/builder.go +++ b/builder.go @@ -204,7 +204,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e } return err } - line = strings.TrimSpace(line) + line = strings.Replace(strings.TrimSpace(line), " ", " ", 1) // Skip comments and empty line if len(line) == 0 || line[0] == '#' { continue @@ -328,7 +328,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) break default: - fmt.Fprintf(stdout, "Skipping unknown op %s\n", tmp[0]) + fmt.Fprintf(stdout, "Skipping unknown instruction %s\n", strings.ToUpper(instruction)) } } if image != nil { From 35c59f4e05fb8dc3a5939d6af8c9313ebda6a5dd Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 6 May 2013 16:58:09 -0700 Subject: [PATCH 87/97] Rebase fix --- builder.go | 132 ++++++++++++++++++++++++++++++++++++---------------- commands.go | 1 - 2 files changed, 93 insertions(+), 40 deletions(-) diff --git a/builder.go b/builder.go index 19c24c901e..84fb01eb49 100644 --- a/builder.go +++ b/builder.go @@ -13,11 +13,13 @@ import ( type Builder struct { runtime *Runtime repositories *TagStore + graph *Graph } func NewBuilder(runtime *Runtime) *Builder { return &Builder{ runtime: runtime, + graph: runtime.graph, repositories: runtime.repositories, } } @@ -142,6 +144,12 @@ func (builder *Builder) Commit(container *Container, repository, tag, comment, a if err != nil { return nil, err } + // Register the image if needed + if repository != "" { + if err := builder.repositories.Set(repository, tag, img.Id, true); err != nil { + return img, err + } + } return img, nil } @@ -202,29 +210,33 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e if err == io.EOF { break } - return err + return nil, err } line = strings.Replace(strings.TrimSpace(line), " ", " ", 1) // Skip comments and empty line if len(line) == 0 || line[0] == '#' { continue } - tmp := strings.SplitN(line, " ", 2) + tmp := strings.SplitN(line, " ", 2) if len(tmp) != 2 { - return fmt.Errorf("Invalid Dockerfile format") + return nil, fmt.Errorf("Invalid Dockerfile format") } - switch tmp[0] { + instruction := strings.Trim(tmp[0], " ") + arguments := strings.Trim(tmp[1], " ") + switch strings.ToLower(instruction) { case "from": - fmt.Fprintf(stdout, "FROM %s\n", tmp[1]) - image, err = builder.runtime.repositories.LookupImage(tmp[1]) + fmt.Fprintf(stdout, "FROM %s\n", arguments) + image, err = builder.runtime.repositories.LookupImage(arguments) if err != nil { if builder.runtime.graph.IsNotExist(err) { var tag, remote string - if strings.Contains(remote, ":") { - remoteParts := strings.Split(remote, ":") + if strings.Contains(arguments, ":") { + remoteParts := strings.Split(arguments, ":") tag = remoteParts[1] remote = remoteParts[0] + } else { + remote = arguments } if err := builder.runtime.graph.PullRepository(stdout, remote, tag, builder.runtime.repositories, builder.runtime.authConfig); err != nil { @@ -235,20 +247,24 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e if err != nil { return nil, err } - } else { return nil, err } } + + break + case "mainainer": + fmt.Fprintf(stdout, "MAINTAINER %s\n", arguments) + maintainer = arguments break case "run": - fmt.Fprintf(stdout, "RUN %s\n", tmp[1]) + fmt.Fprintf(stdout, "RUN %s\n", arguments) if image == nil { - return fmt.Errorf("Please provide a source image with `from` prior to run") + return nil, fmt.Errorf("Please provide a source image with `from` prior to run") } - config, err := ParseRun([]string{image.Id, "/bin/sh", "-c", tmp[1]}, nil, builder.runtime.capabilities) + config, err := ParseRun([]string{image.Id, "/bin/sh", "-c", arguments}, nil, builder.runtime.capabilities) if err != nil { - return err + return nil, err } if cache, err := builder.getCachedImage(image, config); err != nil { @@ -262,70 +278,108 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e // Create the container and start it c, err := builder.Create(config) if err != nil { - return err + return nil, err } if err := c.Start(); err != nil { - return err + return nil, err } tmpContainers[c.Id] = struct{}{} // Wait for it to finish if result := c.Wait(); result != 0 { - return fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", tmp[1], result) + return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", arguments, result) } // Commit the container - base, err = builder.Commit(c, "", "", "", "", nil) + base, err = builder.Commit(c, "", "", "", maintainer, nil) if err != nil { - return err + return nil, err } tmpImages[base.Id] = struct{}{} fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) - break - case "copy": - if image == nil { - return fmt.Errorf("Please provide a source image with `from` prior to copy") - } - tmp2 := strings.SplitN(tmp[1], " ", 2) - if len(tmp) != 2 { - return fmt.Errorf("Invalid COPY format") - } - fmt.Fprintf(stdout, "COPY %s to %s in %s\n", tmp2[0], tmp2[1], base.ShortId()) - file, err := Download(tmp2[0], stdout) + // use the base as the new image + image = base + + break + case "expose": + ports := strings.Split(arguments, " ") + + fmt.Fprintf(stdout, "EXPOSE %v\n", ports) + if image == nil { + return nil, fmt.Errorf("Please provide a source image with `from` prior to copy") + } + + // Create the container and start it + c, err := builder.Create(&Config{Image: image.Id, Cmd: []string{"", ""}}) if err != nil { - return err + return nil, err + } + if err := c.Start(); err != nil { + return nil, err + } + tmpContainers[c.Id] = struct{}{} + + // Commit the container + base, err = builder.Commit(c, "", "", "", maintainer, &Config{PortSpecs: ports}) + if err != nil { + return nil, err + } + tmpImages[base.Id] = struct{}{} + + fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) + + image = base + break + case "insert": + if image == nil { + return nil, fmt.Errorf("Please provide a source image with `from` prior to copy") + } + tmp = strings.SplitN(arguments, " ", 2) + if len(tmp) != 2 { + return nil, fmt.Errorf("Invalid INSERT format") + } + sourceUrl := strings.Trim(tmp[0], " ") + destPath := strings.Trim(tmp[1], " ") + fmt.Fprintf(stdout, "COPY %s to %s in %s\n", sourceUrl, destPath, base.ShortId()) + + file, err := Download(sourceUrl, stdout) + if err != nil { + return nil, err } defer file.Body.Close() - config, err := ParseRun([]string{base.Id, "echo", "insert", tmp2[0], tmp2[1]}, nil, builder.runtime.capabilities) + config, err := ParseRun([]string{base.Id, "echo", "insert", sourceUrl, destPath}, nil, builder.runtime.capabilities) if err != nil { - return err + return nil, err } c, err := builder.Create(config) if err != nil { - return err + return nil, err } if err := c.Start(); err != nil { - return err + return nil, err } // Wait for echo to finish if result := c.Wait(); result != 0 { - return fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", tmp[1], result) + return nil, fmt.Errorf("!!! '%s' return non-zero exit code '%d'. Aborting.", arguments, result) } - if err := c.Inject(file.Body, tmp2[1]); err != nil { - return err + if err := c.Inject(file.Body, destPath); err != nil { + return nil, err } - base, err = builder.Commit(c, "", "", "", "", nil) + base, err = builder.Commit(c, "", "", "", maintainer, nil) if err != nil { - return err + return nil, err } fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) + + image = base + break default: fmt.Fprintf(stdout, "Skipping unknown instruction %s\n", strings.ToUpper(instruction)) diff --git a/commands.go b/commands.go index c72e24aa61..442d7f55ee 100644 --- a/commands.go +++ b/commands.go @@ -11,7 +11,6 @@ import ( "net/http" "net/url" "path/filepath" - "os" "runtime" "strconv" "strings" From 602786cd60746cd2baf7bb14fb6acb60f9554f50 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 15:14:10 -0700 Subject: [PATCH 88/97] Moving runtime.Create to builder.Create --- builder.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/builder.go b/builder.go index 84fb01eb49..24e6dc1e67 100644 --- a/builder.go +++ b/builder.go @@ -274,6 +274,10 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e fmt.Fprintf(stdout, "===> %s\n", image.ShortId()) break } + config, err := ParseRun([]string{image.Id, "/bin/sh", "-c", tmp[1]}, nil, builder.runtime.capabilities) + if err != nil { + return err + } // Create the container and start it c, err := builder.Create(config) @@ -283,6 +287,9 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e if err := c.Start(); err != nil { return nil, err } + if err := c.Start(); err != nil { + return err + } tmpContainers[c.Id] = struct{}{} // Wait for it to finish From a64ebe5feb84d65adbc23ee9d2119f5eab836079 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 2 May 2013 02:51:14 -0700 Subject: [PATCH 89/97] Allow to stack multiple EXPOSE instructions --- builder.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/builder.go b/builder.go index 24e6dc1e67..3c76712226 100644 --- a/builder.go +++ b/builder.go @@ -197,6 +197,7 @@ func (builder *Builder) getCachedImage(image *Image, config *Config) (*Image, er func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, error) { var ( image, base *Image + config *Config maintainer string tmpContainers map[string]struct{} = make(map[string]struct{}) tmpImages map[string]struct{} = make(map[string]struct{}) @@ -251,6 +252,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e return nil, err } } + config = &Config{} break case "mainainer": @@ -328,8 +330,10 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e } tmpContainers[c.Id] = struct{}{} + config.PortSpecs = append(ports, config.PortSpecs...) + // Commit the container - base, err = builder.Commit(c, "", "", "", maintainer, &Config{PortSpecs: ports}) + base, err = builder.Commit(c, "", "", "", maintainer, config) if err != nil { return nil, err } From db4417b601feedb03cc70b79f64456e817ce0c05 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 2 May 2013 03:00:56 -0700 Subject: [PATCH 90/97] Implement the CMD instruction in order to allow autorun --- builder.go | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/builder.go b/builder.go index 3c76712226..980d001904 100644 --- a/builder.go +++ b/builder.go @@ -2,6 +2,7 @@ package docker import ( "bufio" + "encoding/json" "fmt" "io" "os" @@ -311,6 +312,35 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e // use the base as the new image image = base + break + case "cmd": + fmt.Fprintf(stdout, "CMD %s\n", arguments) + + // Create the container and start it + c, err := builder.Create(&Config{Image: image.Id, Cmd: []string{"", ""}}) + if err != nil { + return nil, err + } + if err := c.Start(); err != nil { + return nil, err + } + tmpContainers[c.Id] = struct{}{} + + cmd := []string{} + if err := json.Unmarshal([]byte(arguments), &cmd); err != nil { + return nil, err + } + config.Cmd = cmd + + // Commit the container + base, err = builder.Commit(c, "", "", "", maintainer, config) + if err != nil { + return nil, err + } + tmpImages[base.Id] = struct{}{} + + fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) + image = base break case "expose": ports := strings.Split(arguments, " ") @@ -340,7 +370,6 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e tmpImages[base.Id] = struct{}{} fmt.Fprintf(stdout, "===> %s\n", base.ShortId()) - image = base break case "insert": From 979db00d9af4741572ece9c67061526d5d4a6616 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 2 May 2013 03:58:58 -0700 Subject: [PATCH 91/97] Fix typo in builder --- builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder.go b/builder.go index 980d001904..f22d4864e6 100644 --- a/builder.go +++ b/builder.go @@ -256,7 +256,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e config = &Config{} break - case "mainainer": + case "maintainer": fmt.Fprintf(stdout, "MAINTAINER %s\n", arguments) maintainer = arguments break From 3439cd9cea8056cce62d46dd5cf59d6d5b50ef84 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 6 May 2013 17:07:56 -0700 Subject: [PATCH 92/97] Rebase fix --- builder.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/builder.go b/builder.go index f22d4864e6..61ea03311c 100644 --- a/builder.go +++ b/builder.go @@ -277,10 +277,6 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e fmt.Fprintf(stdout, "===> %s\n", image.ShortId()) break } - config, err := ParseRun([]string{image.Id, "/bin/sh", "-c", tmp[1]}, nil, builder.runtime.capabilities) - if err != nil { - return err - } // Create the container and start it c, err := builder.Create(config) @@ -290,9 +286,6 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e if err := c.Start(); err != nil { return nil, err } - if err := c.Start(); err != nil { - return err - } tmpContainers[c.Id] = struct{}{} // Wait for it to finish From dae2828957b6b962109a9f7ad061e9ddd90b8833 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 24 Apr 2013 15:14:10 -0700 Subject: [PATCH 93/97] Moving runtime.Create to builder.Create --- builder.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/builder.go b/builder.go index 61ea03311c..f22d4864e6 100644 --- a/builder.go +++ b/builder.go @@ -277,6 +277,10 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e fmt.Fprintf(stdout, "===> %s\n", image.ShortId()) break } + config, err := ParseRun([]string{image.Id, "/bin/sh", "-c", tmp[1]}, nil, builder.runtime.capabilities) + if err != nil { + return err + } // Create the container and start it c, err := builder.Create(config) @@ -286,6 +290,9 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e if err := c.Start(); err != nil { return nil, err } + if err := c.Start(); err != nil { + return err + } tmpContainers[c.Id] = struct{}{} // Wait for it to finish From 49b05eb24a3281a8d57791b09f5b82f7dafc87f8 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 2 May 2013 04:12:42 -0700 Subject: [PATCH 94/97] Update docker builder doc --- docs/sources/builder/basics.rst | 38 +++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/docs/sources/builder/basics.rst b/docs/sources/builder/basics.rst index a233515ef2..2282cccf29 100644 --- a/docs/sources/builder/basics.rst +++ b/docs/sources/builder/basics.rst @@ -34,14 +34,39 @@ The `FROM` instruction must be the first one in order for Builder to know from w `FROM` can also be used in order to build multiple images within a single Dockerfile -2.2 RUN +2.2 MAINTAINER +-------------- + ``MAINTAINER `` + +The `MAINTAINER` instruction allow you to set the Author field of the generated images. +This instruction is never automatically reset. + +2.3 RUN ------- ``RUN `` The `RUN` instruction is the main one, it allows you to execute any commands on the `FROM` image and to save the results. You can use as many `RUN` as you want within a Dockerfile, the commands will be executed on the result of the previous command. -2.3 INSERT + +2.4 CMD +------- + ``CMD `` + +The `CMD` instruction sets the command to be executed when running the image. +It is equivalent to do `docker commit -run '{"Cmd": }'` outside the builder. + +.. note:: + Do not confuse `RUN` with `CMD`. `RUN` actually run a command and save the result, `CMD` does not execute anything. + +2.5 EXPOSE +---------- + ``EXPOSE [...]`` + +The `EXPOSE` instruction sets ports to be publicly exposed when running the image. +This is equivalent to do `docker commit -run '{"PortSpecs": ["", ""]}'` outside the builder. + +2.6 INSERT ---------- ``INSERT `` @@ -51,6 +76,7 @@ The `INSERT` instruction will download the file at the given url and place it wi .. note:: The path must include the file name. + 3. Dockerfile Examples ====================== @@ -61,8 +87,9 @@ The `INSERT` instruction will download the file at the given url and place it wi # VERSION 0.0.1 # DOCKER-VERSION 0.2 - from ubuntu - + from ubuntu + maintainer Guillaume J. Charmes "guillaume@dotcloud.com" + # make sure the package repository is up to date run echo "deb http://archive.ubuntu.com/ubuntu precise main universe" > /etc/apt/sources.list run apt-get update @@ -89,3 +116,6 @@ The `INSERT` instruction will download the file at the given url and place it wi run x11vnc -storepasswd 1234 ~/.vnc/passwd # Autostart firefox (might not be the best way to do it, but it does the trick) run bash -c 'echo "firefox" >> /.bashrc' + + expose 5900 + cmd ["x11vnc", "-forever", "-usepw", "-create"] From 7757be1f452c625d26d1b086384003ab6131e6d4 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 6 May 2013 17:12:56 -0700 Subject: [PATCH 95/97] Rebase fix --- builder.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/builder.go b/builder.go index f22d4864e6..61ea03311c 100644 --- a/builder.go +++ b/builder.go @@ -277,10 +277,6 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e fmt.Fprintf(stdout, "===> %s\n", image.ShortId()) break } - config, err := ParseRun([]string{image.Id, "/bin/sh", "-c", tmp[1]}, nil, builder.runtime.capabilities) - if err != nil { - return err - } // Create the container and start it c, err := builder.Create(config) @@ -290,9 +286,6 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e if err := c.Start(); err != nil { return nil, err } - if err := c.Start(); err != nil { - return err - } tmpContainers[c.Id] = struct{}{} // Wait for it to finish From e45aef0c82044be6a427502d4bbb0979074c2ba1 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 6 May 2013 18:39:56 -0700 Subject: [PATCH 96/97] Implement ENV within docker builder --- builder.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/builder.go b/builder.go index 61ea03311c..e835c6d390 100644 --- a/builder.go +++ b/builder.go @@ -200,6 +200,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e image, base *Image config *Config maintainer string + env map[string]string = make(map[string]string) tmpContainers map[string]struct{} = make(map[string]struct{}) tmpImages map[string]struct{} = make(map[string]struct{}) ) @@ -270,6 +271,10 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e return nil, err } + for key, value := range env { + config.Env = append(config.Env, fmt.Sprintf("%s=%s", key, value)) + } + if cache, err := builder.getCachedImage(image, config); err != nil { return nil, err } else if cache != nil { @@ -278,11 +283,21 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e break } + Debugf("Env -----> %v ------ %v\n", config.Env, env) + // Create the container and start it c, err := builder.Create(config) if err != nil { return nil, err } + + if os.Getenv("DEBUG") != "" { + out, _ := c.StdoutPipe() + err2, _ := c.StderrPipe() + go io.Copy(os.Stdout, out) + go io.Copy(os.Stdout, err2) + } + if err := c.Start(); err != nil { return nil, err } @@ -305,6 +320,21 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e // use the base as the new image image = base + break + case "env": + tmp := strings.SplitN(arguments, " ", 2) + if len(tmp) != 2 { + return nil, fmt.Errorf("Invalid ENV format") + } + key := strings.Trim(tmp[0], " ") + value := strings.Trim(tmp[1], " ") + fmt.Fprintf(stdout, "ENV %s %s\n", key, value) + env[key] = value + if image != nil { + fmt.Fprintf(stdout, "===> %s\n", image.ShortId()) + } else { + fmt.Fprintf(stdout, "===> \n") + } break case "cmd": fmt.Fprintf(stdout, "CMD %s\n", arguments) From 4c7c177e4ea4a166e656306e1d845cdb9e2f37ce Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 6 May 2013 18:54:27 -0700 Subject: [PATCH 97/97] Add the ENV instruciton to the docker builder documentation --- docs/sources/builder/basics.rst | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/sources/builder/basics.rst b/docs/sources/builder/basics.rst index 2282cccf29..0d726e93c1 100644 --- a/docs/sources/builder/basics.rst +++ b/docs/sources/builder/basics.rst @@ -66,7 +66,16 @@ It is equivalent to do `docker commit -run '{"Cmd": }'` outside the bui The `EXPOSE` instruction sets ports to be publicly exposed when running the image. This is equivalent to do `docker commit -run '{"PortSpecs": ["", ""]}'` outside the builder. -2.6 INSERT +2.6 ENV +------- + ``ENV `` + +The `ENV` instruction set as environment variable `` with the value ``. This value will be passed to all future ``RUN`` instructions. + +.. note:: + The environment variables are local to the Dockerfile, they will not be set as autorun. + +2.7 INSERT ---------- ``INSERT ``