From 17a806c8a0b6add2aa773dfca272acefee9b638c Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 12 Dec 2013 22:39:35 +0000 Subject: [PATCH] Port 'docker images' to the engine API Docker-DCO-1.1-Signed-off-by: Solomon Hykes (github: shykes) --- api.go | 30 ++++------- engine/env.go | 74 +++++++++++++++++++++++++ engine/streams.go | 16 ++++++ engine/table_test.go | 28 ++++++++++ integration/api_test.go | 64 ++++++++++------------ integration/runtime_test.go | 17 +++--- integration/server_test.go | 105 +++++++++++++----------------------- integration/sorter_test.go | 16 ++---- integration/utils_test.go | 21 +++++++- server.go | 74 ++++++++++++++----------- 10 files changed, 270 insertions(+), 175 deletions(-) create mode 100644 engine/table_test.go diff --git a/api.go b/api.go index c768ef9344..10dab5fae2 100644 --- a/api.go +++ b/api.go @@ -181,27 +181,19 @@ func getImagesJSON(srv *Server, version float64, w http.ResponseWriter, r *http. if err := parseForm(r); err != nil { return err } - - all, err := getBoolParam(r.Form.Get("all")) - if err != nil { + fmt.Printf("getImagesJSON\n") + job := srv.Eng.Job("images") + job.Setenv("filter", r.Form.Get("filter")) + job.Setenv("all", r.Form.Get("all")) + // FIXME: 1.7 clients expect a single json list + job.Stdout.Add(w) + w.WriteHeader(http.StatusOK) + fmt.Printf("running images job\n") + if err := job.Run(); err != nil { return err } - filter := r.Form.Get("filter") - - outs, err := srv.Images(all, filter) - if err != nil { - return err - } - - if version < 1.7 { - outs2 := []APIImagesOld{} - for _, ctnr := range outs { - outs2 = append(outs2, ctnr.ToLegacy()...) - } - - return writeJSON(w, http.StatusOK, outs2) - } - return writeJSON(w, http.StatusOK, outs) + fmt.Printf("job has been run\n") + return nil } func getImagesViz(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { diff --git a/engine/env.go b/engine/env.go index a65c8438d2..f4cc124240 100644 --- a/engine/env.go +++ b/engine/env.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io" + "sort" "strconv" "strings" ) @@ -232,3 +233,76 @@ func (env *Env) Map() map[string]string { } return m } + +type Table struct { + Data []*Env + sortKey string +} + +func NewTable(sortKey string, sizeHint int) *Table { + return &Table{ + make([]*Env, 0, sizeHint), + sortKey, + } +} + +func (t *Table) Add(env *Env) { + t.Data = append(t.Data, env) +} + +func (t *Table) Len() int { + return len(t.Data) +} + +func (t *Table) Less(a, b int) bool { + return t.lessBy(a, b, t.sortKey) +} + +func (t *Table) lessBy(a, b int, by string) bool { + keyA := t.Data[a].Get(by) + keyB := t.Data[b].Get(by) + intA, errA := strconv.ParseInt(keyA, 10, 64) + intB, errB := strconv.ParseInt(keyB, 10, 64) + if errA == nil && errB == nil { + return intA < intB + } + return keyA < keyB +} + +func (t *Table) Swap(a, b int) { + tmp := t.Data[a] + t.Data[a] = t.Data[b] + t.Data[b] = tmp +} + +func (t *Table) Sort() { + sort.Sort(t) +} + +func (t *Table) WriteTo(dst io.Writer) (n int64, err error) { + for _, env := range t.Data { + bytes, err := env.WriteTo(dst) + if err != nil { + return -1, err + } + if _, err := dst.Write([]byte{'\n'}); err != nil { + return -1, err + } + n += bytes + 1 + } + return n, nil +} + +func (t *Table) ReadFrom(src io.Reader) (n int64, err error) { + decoder := NewDecoder(src) + for { + env, err := decoder.Decode() + if err == io.EOF { + return 0, nil + } else if err != nil { + return -1, err + } + t.Add(env) + } + return 0, nil +} diff --git a/engine/streams.go b/engine/streams.go index 7cd4a60cf7..824f0a4ab2 100644 --- a/engine/streams.go +++ b/engine/streams.go @@ -190,3 +190,19 @@ func (o *Output) AddEnv() (dst *Env, err error) { }() return dst, nil } + +func (o *Output) AddTable() (dst *Table, err error) { + src, err := o.AddPipe() + if err != nil { + return nil, err + } + dst = NewTable("", 0) + o.tasks.Add(1) + go func() { + defer o.tasks.Done() + if _, err := dst.ReadFrom(src); err != nil { + return + } + }() + return dst, nil +} diff --git a/engine/table_test.go b/engine/table_test.go new file mode 100644 index 0000000000..0a81d690eb --- /dev/null +++ b/engine/table_test.go @@ -0,0 +1,28 @@ +package engine + +import ( + "testing" + "bytes" + "encoding/json" +) + +func TestTableWriteTo(t *testing.T) { + table := NewTable("", 0) + e := &Env{} + e.Set("foo", "bar") + table.Add(e) + var buf bytes.Buffer + if _, err := table.WriteTo(&buf); err != nil { + t.Fatal(err) + } + output := make(map[string]string) + if err := json.Unmarshal(buf.Bytes(), &output); err != nil { + t.Fatal(err) + } + if len(output) != 1 { + t.Fatalf("Incorrect output: %v", output) + } + if val, exists := output["foo"]; !exists || val != "bar" { + t.Fatalf("Inccorect output: %v", output) + } +} diff --git a/integration/api_test.go b/integration/api_test.go index ff42afac5a..de54078dea 100644 --- a/integration/api_test.go +++ b/integration/api_test.go @@ -60,11 +60,14 @@ func TestGetInfo(t *testing.T) { defer mkRuntimeFromEngine(eng, t).Nuke() srv := mkServerFromEngine(eng, t) - initialImages, err := srv.Images(false, "") + job := eng.Job("images") + initialImages, err := job.Stdout.AddTable() if err != nil { t.Fatal(err) } - + if err := job.Run(); err != nil { + t.Fatal(err) + } req, err := http.NewRequest("GET", "/info", nil) if err != nil { t.Fatal(err) @@ -85,8 +88,8 @@ func TestGetInfo(t *testing.T) { t.Fatal(err) } out.Close() - if images := i.GetInt("Images"); images != len(initialImages) { - t.Errorf("Expected images: %d, %d found", len(initialImages), images) + if images := i.GetInt("Images"); images != initialImages.Len() { + t.Errorf("Expected images: %d, %d found", initialImages.Len(), images) } expected := "application/json" if result := r.HeaderMap.Get("Content-Type"); result != expected { @@ -145,12 +148,14 @@ func TestGetImagesJSON(t *testing.T) { defer mkRuntimeFromEngine(eng, t).Nuke() srv := mkServerFromEngine(eng, t) - // all=0 - - initialImages, err := srv.Images(false, "") + job := eng.Job("images") + initialImages, err := job.Stdout.AddTable() if err != nil { t.Fatal(err) } + if err := job.Run(); err != nil { + t.Fatal(err) + } req, err := http.NewRequest("GET", "/images/json?all=0", nil) if err != nil { @@ -164,18 +169,18 @@ func TestGetImagesJSON(t *testing.T) { } assertHttpNotError(r, t) - images := []docker.APIImages{} - if err := json.Unmarshal(r.Body.Bytes(), &images); err != nil { + images := engine.NewTable("Created", 0) + if _, err := images.ReadFrom(r.Body); err != nil { t.Fatal(err) } - if len(images) != len(initialImages) { - t.Errorf("Expected %d image, %d found", len(initialImages), len(images)) + if images.Len() != initialImages.Len() { + t.Errorf("Expected %d image, %d found", initialImages.Len(), images.Len()) } found := false - for _, img := range images { - if strings.Contains(img.RepoTags[0], unitTestImageName) { + for _, img := range images.Data { + if strings.Contains(img.GetList("RepoTags")[0], unitTestImageName) { found = true break } @@ -188,10 +193,7 @@ func TestGetImagesJSON(t *testing.T) { // all=1 - initialImages, err = srv.Images(true, "") - if err != nil { - t.Fatal(err) - } + initialImages = getAllImages(eng, t) req2, err := http.NewRequest("GET", "/images/json?all=true", nil) if err != nil { @@ -207,8 +209,8 @@ func TestGetImagesJSON(t *testing.T) { t.Fatal(err) } - if len(images2) != len(initialImages) { - t.Errorf("Expected %d image, %d found", len(initialImages), len(images2)) + if len(images2) != initialImages.Len() { + t.Errorf("Expected %d image, %d found", initialImages.Len(), len(images2)) } found = false @@ -1126,21 +1128,16 @@ func TestDeleteImages(t *testing.T) { defer mkRuntimeFromEngine(eng, t).Nuke() srv := mkServerFromEngine(eng, t) - initialImages, err := srv.Images(false, "") - if err != nil { - t.Fatal(err) - } + initialImages := getImages(eng, t, true, "") if err := eng.Job("tag", unitTestImageName, "test", "test").Run(); err != nil { t.Fatal(err) } - images, err := srv.Images(false, "") - if err != nil { - t.Fatal(err) - } - if len(images[0].RepoTags) != len(initialImages[0].RepoTags)+1 { - t.Errorf("Expected %d images, %d found", len(initialImages)+1, len(images)) + images := getImages(eng, t, true, "") + + if images.Len() != initialImages.Len()+1 { + t.Errorf("Expected %d images, %d found", initialImages.Len()+1, images.Len()) } req, err := http.NewRequest("DELETE", "/images/"+unitTestImageID, nil) @@ -1177,13 +1174,10 @@ func TestDeleteImages(t *testing.T) { if len(outs) != 1 { t.Fatalf("Expected %d event (untagged), got %d", 1, len(outs)) } - images, err = srv.Images(false, "") - if err != nil { - t.Fatal(err) - } + images = getImages(eng, t, false, "") - if len(images[0].RepoTags) != len(initialImages[0].RepoTags) { - t.Errorf("Expected %d image, %d found", len(initialImages), len(images)) + if images.Len() != initialImages.Len() { + t.Errorf("Expected %d image, %d found", initialImages.Len(), images.Len()) } } diff --git a/integration/runtime_test.go b/integration/runtime_test.go index cdd4818934..f3d8384082 100644 --- a/integration/runtime_test.go +++ b/integration/runtime_test.go @@ -51,14 +51,17 @@ func cleanup(eng *engine.Engine, t *testing.T) error { container.Kill() runtime.Destroy(container) } - srv := mkServerFromEngine(eng, t) - images, err := srv.Images(true, "") + job := eng.Job("images") + images, err := job.Stdout.AddTable() if err != nil { - return err + t.Fatal(err) } - for _, image := range images { - if image.ID != unitTestImageID { - srv.ImageDelete(image.ID, false) + if err := job.Run(); err != nil { + t.Fatal(err) + } + for _, image := range images.Data { + if image.Get("ID") != unitTestImageID { + mkServerFromEngine(eng, t).ImageDelete(image.Get("ID"), false) } } return nil @@ -158,7 +161,7 @@ func spawnGlobalDaemon() { Host: testDaemonAddr, } job := eng.Job("serveapi", listenURL.String()) - job.SetenvBool("Logging", os.Getenv("DEBUG") != "") + job.SetenvBool("Logging", true) if err := job.Run(); err != nil { log.Fatalf("Unable to spawn the test daemon: %s", err) } diff --git a/integration/server_test.go b/integration/server_test.go index a29c2acbf3..106bff9c2b 100644 --- a/integration/server_test.go +++ b/integration/server_test.go @@ -14,10 +14,7 @@ func TestImageTagImageDelete(t *testing.T) { srv := mkServerFromEngine(eng, t) - initialImages, err := srv.Images(false, "") - if err != nil { - t.Fatal(err) - } + initialImages := getAllImages(eng, t) if err := eng.Job("tag", unitTestImageName, "utest", "tag1").Run(); err != nil { t.Fatal(err) } @@ -30,52 +27,43 @@ func TestImageTagImageDelete(t *testing.T) { t.Fatal(err) } - images, err := srv.Images(false, "") - if err != nil { - t.Fatal(err) - } + images := getAllImages(eng, t) - if len(images[0].RepoTags) != len(initialImages[0].RepoTags)+3 { - t.Errorf("Expected %d images, %d found", len(initialImages)+3, len(images)) + nExpected := len(initialImages.Data[0].GetList("RepoTags")) + 3 + nActual := len(images.Data[0].GetList("RepoTags")) + if nExpected != nActual { + t.Errorf("Expected %d images, %d found", nExpected, nActual) } if _, err := srv.ImageDelete("utest/docker:tag2", true); err != nil { t.Fatal(err) } - images, err = srv.Images(false, "") - if err != nil { - t.Fatal(err) - } + images = getAllImages(eng, t) - if len(images[0].RepoTags) != len(initialImages[0].RepoTags)+2 { - t.Errorf("Expected %d images, %d found", len(initialImages)+2, len(images)) + nExpected = len(initialImages.Data[0].GetList("RepoTags")) + 2 + nActual = len(images.Data[0].GetList("RepoTags")) + if nExpected != nActual { + t.Errorf("Expected %d images, %d found", nExpected, nActual) } if _, err := srv.ImageDelete("utest:5000/docker:tag3", true); err != nil { t.Fatal(err) } - images, err = srv.Images(false, "") - if err != nil { - t.Fatal(err) - } + images = getAllImages(eng, t) - if len(images[0].RepoTags) != len(initialImages[0].RepoTags)+1 { - t.Errorf("Expected %d images, %d found", len(initialImages)+1, len(images)) - } + nExpected = len(initialImages.Data[0].GetList("RepoTags")) + 1 + nActual = len(images.Data[0].GetList("RepoTags")) if _, err := srv.ImageDelete("utest:tag1", true); err != nil { t.Fatal(err) } - images, err = srv.Images(false, "") - if err != nil { - t.Fatal(err) - } + images = getAllImages(eng, t) - if len(images) != len(initialImages) { - t.Errorf("Expected %d image, %d found", len(initialImages), len(images)) + if images.Len() != initialImages.Len() { + t.Errorf("Expected %d image, %d found", initialImages.Len(), images.Len()) } } @@ -250,10 +238,7 @@ func TestRmi(t *testing.T) { srv := mkServerFromEngine(eng, t) defer mkRuntimeFromEngine(eng, t).Nuke() - initialImages, err := srv.Images(false, "") - if err != nil { - t.Fatal(err) - } + initialImages := getAllImages(eng, t) config, hostConfig, _, err := docker.ParseRun([]string{unitTestImageID, "echo", "test"}, nil) if err != nil { @@ -308,13 +293,10 @@ func TestRmi(t *testing.T) { t.Fatal(err) } - images, err := srv.Images(false, "") - if err != nil { - t.Fatal(err) - } + images := getAllImages(eng, t) - if len(images)-len(initialImages) != 2 { - t.Fatalf("Expected 2 new images, found %d.", len(images)-len(initialImages)) + if images.Len()-initialImages.Len() != 2 { + t.Fatalf("Expected 2 new images, found %d.", images.Len()-initialImages.Len()) } _, err = srv.ImageDelete(imageID, true) @@ -322,20 +304,17 @@ func TestRmi(t *testing.T) { t.Fatal(err) } - images, err = srv.Images(false, "") - if err != nil { - t.Fatal(err) + images = getAllImages(eng, t) + + if images.Len()-initialImages.Len() != 1 { + t.Fatalf("Expected 1 new image, found %d.", images.Len()-initialImages.Len()) } - if len(images)-len(initialImages) != 1 { - t.Fatalf("Expected 1 new image, found %d.", len(images)-len(initialImages)) - } - - for _, image := range images { - if strings.Contains(unitTestImageID, image.ID) { + for _, image := range images.Data { + if strings.Contains(unitTestImageID, image.Get("ID")) { continue } - if image.RepoTags[0] == ":" { + if image.GetList("RepoTags")[0] == ":" { t.Fatalf("Expected tagged image, got untagged one.") } } @@ -359,39 +338,27 @@ func TestImagesFilter(t *testing.T) { t.Fatal(err) } - images, err := srv.Images(false, "utest*/*") - if err != nil { - t.Fatal(err) - } + images := getImages(eng, t, false, "utest*/*") - if len(images[0].RepoTags) != 2 { + if len(images.Data[0].GetList("RepoTags")) != 2 { t.Fatal("incorrect number of matches returned") } - images, err = srv.Images(false, "utest") - if err != nil { - t.Fatal(err) - } + images = getImages(eng, t, false, "utest") - if len(images[0].RepoTags) != 1 { + if len(images.Data[0].GetList("RepoTags")) != 1 { t.Fatal("incorrect number of matches returned") } - images, err = srv.Images(false, "utest*") - if err != nil { - t.Fatal(err) - } + images = getImages(eng, t, false, "utest*") - if len(images[0].RepoTags) != 1 { + if len(images.Data[0].GetList("RepoTags")) != 1 { t.Fatal("incorrect number of matches returned") } - images, err = srv.Images(false, "*5000*/*") - if err != nil { - t.Fatal(err) - } + images = getImages(eng, t, false, "*5000*/*") - if len(images[0].RepoTags) != 1 { + if len(images.Data[0].GetList("RepoTags")) != 1 { t.Fatal("incorrect number of matches returned") } } diff --git a/integration/sorter_test.go b/integration/sorter_test.go index 77848c7ddf..1c089a2997 100644 --- a/integration/sorter_test.go +++ b/integration/sorter_test.go @@ -17,13 +17,10 @@ func TestServerListOrderedImagesByCreationDate(t *testing.T) { t.Fatal(err) } - images, err := srv.Images(true, "") - if err != nil { - t.Fatal(err) - } + images := getImages(eng, t, true, "") - if images[0].Created < images[1].Created { - t.Error("Expected []APIImges to be ordered by most recent creation date.") + if images.Data[0].GetInt("Created") < images.Data[1].GetInt("Created") { + t.Error("Expected images to be ordered by most recent creation date.") } } @@ -44,12 +41,9 @@ func TestServerListOrderedImagesByCreationDateAndTag(t *testing.T) { t.Fatal(err) } - images, err := srv.Images(true, "") - if err != nil { - t.Fatal(err) - } + images := getImages(eng, t, true, "") - if images[0].RepoTags[0] != "repo:zed" && images[0].RepoTags[0] != "repo:bar" { + if images.Data[0].GetList("RepoTags")[0] != "repo:zed" && images.Data[0].GetList("RepoTags")[0] != "repo:bar" { t.Errorf("Expected []APIImges to be ordered by most recent creation date. %s", images) } } diff --git a/integration/utils_test.go b/integration/utils_test.go index 85ba13d698..4ab1c96cca 100644 --- a/integration/utils_test.go +++ b/integration/utils_test.go @@ -186,8 +186,6 @@ func NewTestEngine(t utils.Fataler) *engine.Engine { if err != nil { t.Fatal(err) } - eng.Stdout = ioutil.Discard - eng.Stderr = ioutil.Discard // Load default plugins // (This is manually copied and modified from main() until we have a more generic plugin system) job := eng.Job("initapi") @@ -329,3 +327,22 @@ func fakeTar() (io.Reader, error) { tw.Close() return buf, nil } + +func getAllImages(eng *engine.Engine, t *testing.T) *engine.Table { + return getImages(eng, t, true, "") +} + +func getImages(eng *engine.Engine, t *testing.T, all bool, filter string) *engine.Table { + job := eng.Job("images") + job.SetenvBool("all", all) + job.Setenv("filter", filter) + images, err := job.Stdout.AddTable() + if err != nil { + t.Fatal(err) + } + if err := job.Run(); err != nil { + t.Fatal(err) + } + return images + +} diff --git a/server.go b/server.go index c1fda6aeb9..556b342c16 100644 --- a/server.go +++ b/server.go @@ -127,6 +127,10 @@ func jobInitApi(job *engine.Job) engine.Status { job.Error(err) return engine.StatusErr } + if err := job.Eng.Register("images", srv.Images); err != nil { + job.Error(err) + return engine.StatusErr + } return engine.StatusOK } @@ -568,23 +572,26 @@ func (srv *Server) ImagesViz(out io.Writer) error { return nil } -func (srv *Server) Images(all bool, filter string) ([]APIImages, error) { +func (srv *Server) Images(job *engine.Job) engine.Status { + fmt.Printf("Images()\n") + srv.Eng.Job("version").Run() var ( allImages map[string]*Image err error ) - if all { + if job.GetenvBool("all") { allImages, err = srv.runtime.graph.Map() } else { allImages, err = srv.runtime.graph.Heads() } if err != nil { - return nil, err + job.Errorf("%s", err) + return engine.StatusErr } - lookup := make(map[string]APIImages) + lookup := make(map[string]*engine.Env) for name, repository := range srv.runtime.repositories.Repositories { - if filter != "" { - if match, _ := path.Match(filter, name); !match { + if job.Getenv("filter") != "" { + if match, _ := path.Match(job.Getenv("filter"), name); !match { continue } } @@ -596,48 +603,51 @@ func (srv *Server) Images(all bool, filter string) ([]APIImages, error) { } if out, exists := lookup[id]; exists { - out.RepoTags = append(out.RepoTags, fmt.Sprintf("%s:%s", name, tag)) - - lookup[id] = out + repotag := fmt.Sprintf("%s:%s", name, tag) + out.SetList("RepoTags", append(out.GetList("RepoTags"), repotag)) } else { - var out APIImages - + out := &engine.Env{} delete(allImages, id) - - out.ParentId = image.Parent - out.RepoTags = []string{fmt.Sprintf("%s:%s", name, tag)} - out.ID = image.ID - out.Created = image.Created.Unix() - out.Size = image.Size - out.VirtualSize = image.getParentsSize(0) + image.Size - + out.Set("ParentId", image.Parent) + out.SetList("RepoTags", []string{fmt.Sprintf("%s:%s", name, tag)}) + out.Set("ID", image.ID) + out.SetInt64("Created", image.Created.Unix()) + out.SetInt64("Size", image.Size) + out.SetInt64("VirtualSize", image.getParentsSize(0)+image.Size) lookup[id] = out } } } - outs := make([]APIImages, 0, len(lookup)) + outs := engine.NewTable("Created", len(lookup)) for _, value := range lookup { - outs = append(outs, value) + outs.Add(value) } // Display images which aren't part of a repository/tag - if filter == "" { + if job.Getenv("filter") == "" { for _, image := range allImages { - var out APIImages - out.ID = image.ID - out.ParentId = image.Parent - out.RepoTags = []string{":"} - out.Created = image.Created.Unix() - out.Size = image.Size - out.VirtualSize = image.getParentsSize(0) + image.Size - outs = append(outs, out) + out := &engine.Env{} + out.Set("ParentId", image.Parent) + out.SetList("RepoTags", []string{":"}) + out.Set("ID", image.ID) + out.SetInt64("Created", image.Created.Unix()) + out.SetInt64("Size", image.Size) + out.SetInt64("VirtualSize", image.getParentsSize(0)+image.Size) + outs.Add(out) } } - sortImagesByCreationAndTag(outs) - return outs, nil + outs.Sort() + job.Logf("Sending %d images to stdout", outs.Len()) + if n, err := outs.WriteTo(job.Stdout); err != nil { + job.Errorf("%s", err) + return engine.StatusErr + } else { + job.Logf("%d bytes sent", n) + } + return engine.StatusOK } func (srv *Server) DockerInfo(job *engine.Job) engine.Status {