diff --git a/graph/export.go b/graph/export.go index c356a23225..5c7fbcf1ce 100644 --- a/graph/export.go +++ b/graph/export.go @@ -114,6 +114,8 @@ func (s *TagStore) ImageExport(imageExportConfig *ImageExportConfig) error { // FIXME: this should be a top-level function, not a class method func (s *TagStore) exportImage(name, tempdir string) error { for n := name; n != ""; { + img, err := s.LookupImage(n) + // temporary directory tmpImageDir := filepath.Join(tempdir, n) if err := os.Mkdir(tmpImageDir, os.FileMode(0755)); err != nil { @@ -130,15 +132,17 @@ func (s *TagStore) exportImage(name, tempdir string) error { return err } + imageInspectRaw, err := json.Marshal(img) + if err != nil { + return err + } + // serialize json json, err := os.Create(filepath.Join(tmpImageDir, "json")) if err != nil { return err } - imageInspectRaw, err := s.LookupRaw(n) - if err != nil { - return err - } + written, err := json.Write(imageInspectRaw) if err != nil { return err @@ -156,11 +160,6 @@ func (s *TagStore) exportImage(name, tempdir string) error { return err } - // find parent - img, err := s.LookupImage(n) - if err != nil { - return err - } n = img.Parent } return nil diff --git a/graph/graph.go b/graph/graph.go index be911b0482..fd73bfc3e8 100644 --- a/graph/graph.go +++ b/graph/graph.go @@ -32,6 +32,26 @@ import ( "github.com/vbatts/tar-split/tar/storage" ) +// v1ImageDescriptor is a non-content-addressable image descriptor +type v1ImageDescriptor struct { + img *image.Image +} + +// ID returns the image ID specified in the image structure. +func (img v1ImageDescriptor) ID() string { + return img.img.ID +} + +// Parent returns the parent ID specified in the image structure. +func (img v1ImageDescriptor) Parent() string { + return img.img.Parent +} + +// MarshalConfig renders the image structure into JSON. +func (img v1ImageDescriptor) MarshalConfig() ([]byte, error) { + return json.Marshal(img.img) +} + // The type is used to protect pulling or building related image // layers from deleteing when filtered by dangling=true // The key of layers is the images ID which is pulling or building @@ -86,10 +106,12 @@ type Graph struct { // file names for ./graph// const ( - jsonFileName = "json" - layersizeFileName = "layersize" - digestFileName = "checksum" - tarDataFileName = "tar-data.json.gz" + jsonFileName = "json" + layersizeFileName = "layersize" + digestFileName = "checksum" + tarDataFileName = "tar-data.json.gz" + v1CompatibilityFileName = "v1Compatibility" + parentFileName = "parent" ) var ( @@ -214,23 +236,36 @@ func (graph *Graph) Create(layerData archive.ArchiveReader, containerID, contain img.ContainerConfig = *containerConfig } - if err := graph.Register(img, layerData); err != nil { + if err := graph.Register(v1ImageDescriptor{img}, layerData); err != nil { return nil, err } return img, nil } // Register imports a pre-existing image into the graph. -func (graph *Graph) Register(img *image.Image, layerData archive.ArchiveReader) (err error) { +// Returns nil if the image is already registered. +func (graph *Graph) Register(im image.ImageDescriptor, layerData archive.ArchiveReader) (err error) { + imgID := im.ID() - if err := image.ValidateID(img.ID); err != nil { + if err := image.ValidateID(imgID); err != nil { return err } // We need this entire operation to be atomic within the engine. Note that // this doesn't mean Register is fully safe yet. - graph.imageMutex.Lock(img.ID) - defer graph.imageMutex.Unlock(img.ID) + graph.imageMutex.Lock(imgID) + defer graph.imageMutex.Unlock(imgID) + + return graph.register(im, layerData) +} + +func (graph *Graph) register(im image.ImageDescriptor, layerData archive.ArchiveReader) (err error) { + imgID := im.ID() + + // Skip register if image is already registered + if graph.Exists(imgID) { + return nil + } // The returned `error` must be named in this function's signature so that // `err` is not shadowed in this deferred cleanup. @@ -238,19 +273,14 @@ func (graph *Graph) Register(img *image.Image, layerData archive.ArchiveReader) // If any error occurs, remove the new dir from the driver. // Don't check for errors since the dir might not have been created. if err != nil { - graph.driver.Remove(img.ID) + graph.driver.Remove(imgID) } }() - // (This is a convenience to save time. Race conditions are taken care of by os.Rename) - if graph.Exists(img.ID) { - return fmt.Errorf("Image %s already exists", img.ID) - } - // Ensure that the image root does not exist on the filesystem // when it is not registered in the graph. // This is common when you switch from one graph driver to another - if err := os.RemoveAll(graph.imageRoot(img.ID)); err != nil && !os.IsNotExist(err) { + if err := os.RemoveAll(graph.imageRoot(imgID)); err != nil && !os.IsNotExist(err) { return err } @@ -258,7 +288,7 @@ func (graph *Graph) Register(img *image.Image, layerData archive.ArchiveReader) // (the graph is the source of truth). // Ignore errors, since we don't know if the driver correctly returns ErrNotExist. // (FIXME: make that mandatory for drivers). - graph.driver.Remove(img.ID) + graph.driver.Remove(imgID) tmp, err := graph.mktemp("") defer os.RemoveAll(tmp) @@ -266,20 +296,33 @@ func (graph *Graph) Register(img *image.Image, layerData archive.ArchiveReader) return fmt.Errorf("mktemp failed: %s", err) } + parent := im.Parent() + // Create root filesystem in the driver - if err := createRootFilesystemInDriver(graph, img, layerData); err != nil { + if err := createRootFilesystemInDriver(graph, imgID, parent, layerData); err != nil { return err } // Apply the diff/layer - if err := graph.storeImage(img, layerData, tmp); err != nil { + config, err := im.MarshalConfig() + if err != nil { + return err + } + if err := graph.storeImage(imgID, parent, config, layerData, tmp); err != nil { return err } // Commit - if err := os.Rename(tmp, graph.imageRoot(img.ID)); err != nil { + if err := os.Rename(tmp, graph.imageRoot(imgID)); err != nil { return err } - graph.idIndex.Add(img.ID) + graph.idIndex.Add(imgID) + return nil +} + +func createRootFilesystemInDriver(graph *Graph, id, parent string, layerData archive.ArchiveReader) error { + if err := graph.driver.Create(id, parent); err != nil { + return fmt.Errorf("Driver %s failed to create image rootfs %s: %s", graph.driver, id, err) + } return nil } @@ -462,6 +505,21 @@ func (graph *Graph) loadImage(id string) (*image.Image, error) { if err := dec.Decode(img); err != nil { return nil, err } + + if img.ID == "" { + img.ID = id + } + + if img.Parent == "" && img.ParentID != "" && img.ParentID.Validate() == nil { + img.Parent = img.ParentID.Hex() + } + + // compatibilityID for parent + parent, err := ioutil.ReadFile(filepath.Join(root, parentFileName)) + if err == nil && len(parent) > 0 { + img.Parent = string(parent) + } + if err := image.ValidateID(img.ID); err != nil { return nil, err } @@ -496,7 +554,13 @@ func (graph *Graph) saveSize(root string, size int) error { } // SetDigest sets the digest for the image layer to the provided value. -func (graph *Graph) SetDigest(id string, dgst digest.Digest) error { +func (graph *Graph) SetLayerDigest(id string, dgst digest.Digest) error { + graph.imageMutex.Lock(id) + defer graph.imageMutex.Unlock(id) + + return graph.setLayerDigest(id, dgst) +} +func (graph *Graph) setLayerDigest(id string, dgst digest.Digest) error { root := graph.imageRoot(id) if err := ioutil.WriteFile(filepath.Join(root, digestFileName), []byte(dgst.String()), 0600); err != nil { return fmt.Errorf("Error storing digest in %s/%s: %s", root, digestFileName, err) @@ -505,7 +569,14 @@ func (graph *Graph) SetDigest(id string, dgst digest.Digest) error { } // GetDigest gets the digest for the provide image layer id. -func (graph *Graph) GetDigest(id string) (digest.Digest, error) { +func (graph *Graph) GetLayerDigest(id string) (digest.Digest, error) { + graph.imageMutex.Lock(id) + defer graph.imageMutex.Unlock(id) + + return graph.getLayerDigest(id) +} + +func (graph *Graph) getLayerDigest(id string) (digest.Digest, error) { root := graph.imageRoot(id) cs, err := ioutil.ReadFile(filepath.Join(root, digestFileName)) if err != nil { @@ -517,6 +588,76 @@ func (graph *Graph) GetDigest(id string) (digest.Digest, error) { return digest.ParseDigest(string(cs)) } +// SetV1CompatibilityConfig stores the v1Compatibility JSON data associated +// with the image in the manifest to the disk +func (graph *Graph) SetV1CompatibilityConfig(id string, data []byte) error { + graph.imageMutex.Lock(id) + defer graph.imageMutex.Unlock(id) + + return graph.setV1CompatibilityConfig(id, data) +} +func (graph *Graph) setV1CompatibilityConfig(id string, data []byte) error { + root := graph.imageRoot(id) + return ioutil.WriteFile(filepath.Join(root, v1CompatibilityFileName), data, 0600) +} + +// GetV1CompatibilityConfig reads the v1Compatibility JSON data for the image +// from the disk +func (graph *Graph) GetV1CompatibilityConfig(id string) ([]byte, error) { + graph.imageMutex.Lock(id) + defer graph.imageMutex.Unlock(id) + + return graph.getV1CompatibilityConfig(id) +} + +func (graph *Graph) getV1CompatibilityConfig(id string) ([]byte, error) { + root := graph.imageRoot(id) + return ioutil.ReadFile(filepath.Join(root, v1CompatibilityFileName)) +} + +// GenerateV1CompatibilityChain makes sure v1Compatibility JSON data exists +// for the image. If it doesn't it generates and stores it for the image and +// all of it's parents based on the image config JSON. +func (graph *Graph) GenerateV1CompatibilityChain(id string) ([]byte, error) { + graph.imageMutex.Lock(id) + defer graph.imageMutex.Unlock(id) + + if v1config, err := graph.getV1CompatibilityConfig(id); err == nil { + return v1config, nil + } + + // generate new, store it to disk + img, err := graph.Get(id) + if err != nil { + return nil, err + } + + digestPrefix := string(digest.Canonical) + ":" + img.ID = strings.TrimPrefix(img.ID, digestPrefix) + + if img.Parent != "" { + parentConfig, err := graph.GenerateV1CompatibilityChain(img.Parent) + if err != nil { + return nil, err + } + var parent struct{ ID string } + err = json.Unmarshal(parentConfig, &parent) + if err != nil { + return nil, err + } + img.Parent = parent.ID + } + + json, err := json.Marshal(img) + if err != nil { + return nil, err + } + if err := graph.setV1CompatibilityConfig(id, json); err != nil { + return nil, err + } + return json, nil +} + // RawJSON returns the JSON representation for an image as a byte array. func (graph *Graph) RawJSON(id string) ([]byte, error) { root := graph.imageRoot(id) @@ -533,11 +674,11 @@ func jsonPath(root string) string { return filepath.Join(root, jsonFileName) } -func (graph *Graph) disassembleAndApplyTarLayer(img *image.Image, layerData archive.ArchiveReader, root string) error { +func (graph *Graph) disassembleAndApplyTarLayer(id, parent string, layerData archive.ArchiveReader, root string) (size int64, err error) { // this is saving the tar-split metadata mf, err := os.OpenFile(filepath.Join(root, tarDataFileName), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600)) if err != nil { - return err + return 0, err } mfz := gzip.NewWriter(mf) metaPacker := storage.NewJSONPacker(mfz) @@ -546,21 +687,21 @@ func (graph *Graph) disassembleAndApplyTarLayer(img *image.Image, layerData arch inflatedLayerData, err := archive.DecompressStream(layerData) if err != nil { - return err + return 0, err } // we're passing nil here for the file putter, because the ApplyDiff will // handle the extraction of the archive rdr, err := asm.NewInputTarStream(inflatedLayerData, metaPacker, nil) if err != nil { - return err + return 0, err } - if img.Size, err = graph.driver.ApplyDiff(img.ID, img.Parent, archive.ArchiveReader(rdr)); err != nil { - return err + if size, err = graph.driver.ApplyDiff(id, parent, archive.ArchiveReader(rdr)); err != nil { + return 0, err } - return nil + return } func (graph *Graph) assembleTarLayer(img *image.Image) (archive.Archive, error) { diff --git a/graph/graph_test.go b/graph/graph_test.go index d7dd591a3c..3782b7db92 100644 --- a/graph/graph_test.go +++ b/graph/graph_test.go @@ -73,7 +73,7 @@ func TestInterruptedRegister(t *testing.T) { Created: time.Now(), } w.CloseWithError(errors.New("But I'm not a tarball!")) // (Nobody's perfect, darling) - graph.Register(image, badArchive) + graph.Register(v1ImageDescriptor{image}, badArchive) if _, err := graph.Get(image.ID); err == nil { t.Fatal("Image should not exist after Register is interrupted") } @@ -82,7 +82,7 @@ func TestInterruptedRegister(t *testing.T) { if err != nil { t.Fatal(err) } - if err := graph.Register(image, goodArchive); err != nil { + if err := graph.Register(v1ImageDescriptor{image}, goodArchive); err != nil { t.Fatal(err) } } @@ -130,7 +130,7 @@ func TestRegister(t *testing.T) { Comment: "testing", Created: time.Now(), } - err = graph.Register(image, archive) + err = graph.Register(v1ImageDescriptor{image}, archive) if err != nil { t.Fatal(err) } @@ -212,7 +212,7 @@ func TestDelete(t *testing.T) { t.Fatal(err) } // Test delete twice (pull -> rm -> pull -> rm) - if err := graph.Register(img1, archive); err != nil { + if err := graph.Register(v1ImageDescriptor{img1}, archive); err != nil { t.Fatal(err) } if err := graph.Delete(img1.ID); err != nil { @@ -246,9 +246,19 @@ func TestByParent(t *testing.T) { Created: time.Now(), Parent: parentImage.ID, } - _ = graph.Register(parentImage, archive1) - _ = graph.Register(childImage1, archive2) - _ = graph.Register(childImage2, archive3) + + err := graph.Register(v1ImageDescriptor{parentImage}, archive1) + if err != nil { + t.Fatal(err) + } + err = graph.Register(v1ImageDescriptor{childImage1}, archive2) + if err != nil { + t.Fatal(err) + } + err = graph.Register(v1ImageDescriptor{childImage2}, archive3) + if err != nil { + t.Fatal(err) + } byParent := graph.ByParent() numChildren := len(byParent[parentImage.ID]) diff --git a/graph/graph_unix.go b/graph/graph_unix.go index 351a84ccad..a6ca741134 100644 --- a/graph/graph_unix.go +++ b/graph/graph_unix.go @@ -3,8 +3,7 @@ package graph import ( - "encoding/json" - "fmt" + "io/ioutil" "os" "path/filepath" "strings" @@ -74,13 +73,6 @@ func SetupInitLayer(initLayer string) error { return nil } -func createRootFilesystemInDriver(graph *Graph, img *image.Image, layerData archive.ArchiveReader) error { - if err := graph.driver.Create(img.ID, img.Parent); err != nil { - return fmt.Errorf("Driver %s failed to create image rootfs %s: %s", graph.driver, img.ID, err) - } - return nil -} - func (graph *Graph) restoreBaseImages() ([]string, error) { return nil, nil } @@ -88,26 +80,35 @@ func (graph *Graph) restoreBaseImages() ([]string, error) { // storeImage stores file system layer data for the given image to the // graph's storage driver. Image metadata is stored in a file // at the specified root directory. -func (graph *Graph) storeImage(img *image.Image, layerData archive.ArchiveReader, root string) (err error) { +func (graph *Graph) storeImage(id, parent string, config []byte, layerData archive.ArchiveReader, root string) (err error) { + var size int64 // Store the layer. If layerData is not nil, unpack it into the new layer if layerData != nil { - if err := graph.disassembleAndApplyTarLayer(img, layerData, root); err != nil { + if size, err = graph.disassembleAndApplyTarLayer(id, parent, layerData, root); err != nil { return err } } - if err := graph.saveSize(root, int(img.Size)); err != nil { + if err := graph.saveSize(root, int(size)); err != nil { return err } - f, err := os.OpenFile(jsonPath(root), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600)) + if err := ioutil.WriteFile(jsonPath(root), config, 0600); err != nil { + return err + } + + // If image is pointing to a parent via CompatibilityID write the reference to disk + img, err := image.NewImgJSON(config) if err != nil { return err } - defer f.Close() - - return json.NewEncoder(f).Encode(img) + if img.ParentID.Validate() == nil && parent != img.ParentID.Hex() { + if err := ioutil.WriteFile(filepath.Join(root, parentFileName), []byte(parent), 0600); err != nil { + return err + } + } + return nil } // TarLayer returns a tar archive of the image's filesystem layer. diff --git a/graph/graph_windows.go b/graph/graph_windows.go index 1422705a75..33f4d877bc 100644 --- a/graph/graph_windows.go +++ b/graph/graph_windows.go @@ -3,9 +3,8 @@ package graph import ( - "encoding/json" - "fmt" - "os" + "io/ioutil" + "path/filepath" "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/graphdriver/windows" @@ -19,41 +18,6 @@ func SetupInitLayer(initLayer string) error { return nil } -func createRootFilesystemInDriver(graph *Graph, img *image.Image, layerData archive.ArchiveReader) error { - if wd, ok := graph.driver.(*windows.WindowsGraphDriver); ok { - if img.Container != "" && layerData == nil { - logrus.Debugf("Copying from container %s.", img.Container) - - var ids []string - if img.Parent != "" { - parentImg, err := graph.Get(img.Parent) - if err != nil { - return err - } - - ids, err = graph.ParentLayerIds(parentImg) - if err != nil { - return err - } - } - - if err := wd.CopyDiff(img.Container, img.ID, wd.LayerIdsToPaths(ids)); err != nil { - return fmt.Errorf("Driver %s failed to copy image rootfs %s: %s", graph.driver, img.Container, err) - } - } else if img.Parent == "" { - if err := graph.driver.Create(img.ID, img.Parent); err != nil { - return fmt.Errorf("Driver %s failed to create image rootfs %s: %s", graph.driver, img.ID, err) - } - } - } else { - // This fallback allows the use of VFS during daemon development. - if err := graph.driver.Create(img.ID, img.Parent); err != nil { - return fmt.Errorf("Driver %s failed to create image rootfs %s: %s", graph.driver, img.ID, err) - } - } - return nil -} - func (graph *Graph) restoreBaseImages() ([]string, error) { // TODO Windows. This needs implementing (@swernli) return nil, nil @@ -71,42 +35,28 @@ func (graph *Graph) ParentLayerIds(img *image.Image) (ids []string, err error) { // storeImage stores file system layer data for the given image to the // graph's storage driver. Image metadata is stored in a file // at the specified root directory. -func (graph *Graph) storeImage(img *image.Image, layerData archive.ArchiveReader, root string) (err error) { - +func (graph *Graph) storeImage(id, parent string, config []byte, layerData archive.ArchiveReader, root string) (err error) { + var size int64 if wd, ok := graph.driver.(*windows.WindowsGraphDriver); ok { // Store the layer. If layerData is not nil and this isn't a base image, // unpack it into the new layer - if layerData != nil && img.Parent != "" { + if layerData != nil && parent != "" { var ids []string - if img.Parent != "" { - parentImg, err := graph.Get(img.Parent) - if err != nil { - return err - } - - ids, err = graph.ParentLayerIds(parentImg) - if err != nil { - return err - } + parentImg, err := graph.Get(parent) + if err != nil { + return err } - if img.Size, err = wd.Import(img.ID, layerData, wd.LayerIdsToPaths(ids)); err != nil { + ids, err = graph.ParentLayerIds(parentImg) + if err != nil { + return err + } + + if size, err = wd.Import(id, layerData, wd.LayerIdsToPaths(ids)); err != nil { return err } } - if err := graph.saveSize(root, int(img.Size)); err != nil { - return err - } - - f, err := os.OpenFile(jsonPath(root), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600)) - if err != nil { - return err - } - - defer f.Close() - - return json.NewEncoder(f).Encode(img) } else { // We keep this functionality here so that we can still work with the // VFS driver during development. This will not be used for actual running @@ -115,23 +65,32 @@ func (graph *Graph) storeImage(img *image.Image, layerData archive.ArchiveReader // Store the layer. If layerData is not nil, unpack it into the new layer if layerData != nil { - if err := graph.disassembleAndApplyTarLayer(img, layerData, root); err != nil { + if size, err = graph.disassembleAndApplyTarLayer(id, parent, layerData, root); err != nil { return err } } - if err := graph.saveSize(root, int(img.Size)); err != nil { + if err := graph.saveSize(root, size); err != nil { return err } - f, err := os.OpenFile(jsonPath(root), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600)) + if err := ioutil.WriteFile(jsonPath(root), config, 0600); err != nil { + return err + } + + // If image is pointing to a parent via CompatibilityID write the reference to disk + img, err := image.NewImgJSON(config) if err != nil { return err } - defer f.Close() + if img.ParentID.Validate() == nil && parent != img.ParentID.Hex() { + if err := ioutil.WriteFile(filepath.Join(root, parentFileName), []byte(parent), 0600); err != nil { + return err + } + } - return json.NewEncoder(f).Encode(img) + return nil } } diff --git a/graph/load.go b/graph/load.go index 9c19012795..c2010d3b9b 100644 --- a/graph/load.go +++ b/graph/load.go @@ -125,7 +125,7 @@ func (s *TagStore) recursiveLoad(address, tmpImageDir string) error { } } } - if err := s.graph.Register(img, layer); err != nil { + if err := s.graph.Register(v1ImageDescriptor{img}, layer); err != nil { return err } } diff --git a/graph/pull_v1.go b/graph/pull_v1.go index f8df392042..5fb0dac139 100644 --- a/graph/pull_v1.go +++ b/graph/pull_v1.go @@ -123,7 +123,7 @@ func (p *v1Puller) pullRepository(askedTag string) error { defer func() { p.graph.Release(sessionID, imgIDs...) }() - for _, image := range repoData.ImgList { + for _, imgData := range repoData.ImgList { downloadImage := func(img *registry.ImgData) { if askedTag != "" && img.Tag != askedTag { errors <- nil @@ -136,6 +136,11 @@ func (p *v1Puller) pullRepository(askedTag string) error { return } + if err := image.ValidateID(img.ID); err != nil { + errors <- err + return + } + // ensure no two downloads of the same image happen at the same time if c, err := p.poolAdd("pull", "img:"+img.ID); err != nil { if c != nil { @@ -196,7 +201,7 @@ func (p *v1Puller) pullRepository(askedTag string) error { errors <- nil } - go downloadImage(image) + go downloadImage(imgData) } var lastError error @@ -304,7 +309,7 @@ func (p *v1Puller) pullImage(imgID, endpoint string, token []string) (bool, erro layersDownloaded = true defer layer.Close() - err = p.graph.Register(img, + err = p.graph.Register(v1ImageDescriptor{img}, progressreader.New(progressreader.Config{ In: layer, Out: out, diff --git a/graph/pull_v2.go b/graph/pull_v2.go index 1dbb9fe3f4..6de715cfb7 100644 --- a/graph/pull_v2.go +++ b/graph/pull_v2.go @@ -6,6 +6,7 @@ import ( "io" "io/ioutil" "os" + "sync" "github.com/Sirupsen/logrus" "github.com/docker/distribution" @@ -73,7 +74,8 @@ func (p *v2Puller) pullV2Repository(tag string) (err error) { } - c, err := p.poolAdd("pull", taggedName) + poolKey := "v2:" + taggedName + c, err := p.poolAdd("pull", poolKey) if err != nil { if c != nil { // Another pull of the same repository is already taking place; just wait for it to finish @@ -83,7 +85,7 @@ func (p *v2Puller) pullV2Repository(tag string) (err error) { } return err } - defer p.poolRemove("pull", taggedName) + defer p.poolRemove("pull", poolKey) var layersDownloaded bool for _, tag := range tags { @@ -103,7 +105,7 @@ func (p *v2Puller) pullV2Repository(tag string) (err error) { // downloadInfo is used to pass information from download to extractor type downloadInfo struct { - img *image.Image + img contentAddressableDescriptor tmpFile *os.File digest digest.Digest layer distribution.ReadSeekCloser @@ -112,28 +114,83 @@ type downloadInfo struct { out io.Writer // Download progress is written here. } +// contentAddressableDescriptor is used to pass image data from a manifest to the +// graph. +type contentAddressableDescriptor struct { + id string + parent string + strongID digest.Digest + compatibilityID string + config []byte + v1Compatibility []byte +} + +func newContentAddressableImage(v1Compatibility []byte, blobSum digest.Digest, parent digest.Digest) (contentAddressableDescriptor, error) { + img := contentAddressableDescriptor{ + v1Compatibility: v1Compatibility, + } + + var err error + img.config, err = image.MakeImageConfig(v1Compatibility, blobSum, parent) + if err != nil { + return img, err + } + img.strongID, err = image.StrongID(img.config) + if err != nil { + return img, err + } + + unmarshalledConfig, err := image.NewImgJSON(v1Compatibility) + if err != nil { + return img, err + } + + img.compatibilityID = unmarshalledConfig.ID + img.id = img.strongID.Hex() + + return img, nil +} + +// ID returns the actual ID to be used for the downloaded image. This may be +// a computed ID. +func (img contentAddressableDescriptor) ID() string { + return img.id +} + +// Parent returns the parent ID to be used for the image. This may be a +// computed ID. +func (img contentAddressableDescriptor) Parent() string { + return img.parent +} + +// MarshalConfig renders the image structure into JSON. +func (img contentAddressableDescriptor) MarshalConfig() ([]byte, error) { + return img.config, nil +} + type errVerification struct{} func (errVerification) Error() string { return "verification failed" } func (p *v2Puller) download(di *downloadInfo) { - logrus.Debugf("pulling blob %q to %s", di.digest, di.img.ID) + logrus.Debugf("pulling blob %q to %s", di.digest, di.img.id) out := di.out - if c, err := p.poolAdd("pull", "img:"+di.img.ID); err != nil { + poolKey := "v2img:" + di.img.id + if c, err := p.poolAdd("pull", poolKey); err != nil { if c != nil { - out.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.ID), "Layer already being pulled by another client. Waiting.", nil)) + out.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.id), "Layer already being pulled by another client. Waiting.", nil)) <-c - out.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.ID), "Download complete", nil)) + out.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.id), "Download complete", nil)) } else { - logrus.Debugf("Image (id: %s) pull is already running, skipping: %v", di.img.ID, err) + logrus.Debugf("Image (id: %s) pull is already running, skipping: %v", di.img.id, err) } di.err <- nil return } - defer p.poolRemove("pull", "img:"+di.img.ID) + defer p.poolRemove("pull", poolKey) tmpFile, err := ioutil.TempFile("", "GetImageBlob") if err != nil { di.err <- err @@ -170,12 +227,12 @@ func (p *v2Puller) download(di *downloadInfo) { Formatter: p.sf, Size: int(di.size), NewLines: false, - ID: stringid.TruncateID(di.img.ID), + ID: stringid.TruncateID(di.img.id), Action: "Downloading", }) io.Copy(tmpFile, reader) - out.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.ID), "Verifying Checksum", nil)) + out.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.id), "Verifying Checksum", nil)) if !verifier.Verified() { err = fmt.Errorf("filesystem layer verification failed for digest %s", di.digest) @@ -184,9 +241,9 @@ func (p *v2Puller) download(di *downloadInfo) { return } - out.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.ID), "Download complete", nil)) + out.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.id), "Download complete", nil)) - logrus.Debugf("Downloaded %s to tempfile %s", di.img.ID, tmpFile.Name()) + logrus.Debugf("Downloaded %s to tempfile %s", di.img.id, tmpFile.Name()) di.tmpFile = tmpFile di.layer = layerDownload @@ -214,6 +271,17 @@ func (p *v2Puller) pullV2Tag(tag, taggedName string) (verified bool, err error) logrus.Printf("Image manifest for %s has been verified", taggedName) } + // remove duplicate layers and check parent chain validity + err = fixManifestLayers(&manifest.Manifest) + if err != nil { + return false, err + } + + imgs, err := p.getImageInfos(manifest.Manifest) + if err != nil { + return false, err + } + // By using a pipeWriter for each of the downloads to write their progress // to, we can avoid an issue where this function returns an error but // leaves behind running download goroutines. By splitting the writer @@ -234,7 +302,7 @@ func (p *v2Puller) pullV2Tag(tag, taggedName string) (verified bool, err error) // until all current readers/writers are done using the pipe then // set the error. All successive reads/writes will return with this // error. - pipeWriter.CloseWithError(errors.New("download canceled")) + pipeWriter.CloseWithError(fmt.Errorf("download canceled %+v", err)) } else { // If no error then just close the pipe. pipeWriter.Close() @@ -251,26 +319,31 @@ func (p *v2Puller) pullV2Tag(tag, taggedName string) (verified bool, err error) }() for i := len(manifest.FSLayers) - 1; i >= 0; i-- { - img, err := image.NewImgJSON([]byte(manifest.History[i].V1Compatibility)) - if err != nil { - logrus.Debugf("error getting image v1 json: %v", err) - return false, err - } + + img := imgs[i] downloads[i].img = img downloads[i].digest = manifest.FSLayers[i].BlobSum - p.graph.Retain(p.sessionID, img.ID) - layerIDs = append(layerIDs, img.ID) + p.graph.Retain(p.sessionID, img.id) + layerIDs = append(layerIDs, img.id) + + p.graph.imageMutex.Lock(img.id) // Check if exists - if p.graph.Exists(img.ID) { - logrus.Debugf("Image already exists: %s", img.ID) + if p.graph.Exists(img.id) { + if err := p.validateImageInGraph(img.id, imgs, i); err != nil { + p.graph.imageMutex.Unlock(img.id) + return false, fmt.Errorf("image validation failed: %v", err) + } + logrus.Debugf("Image already exists: %s", img.id) + p.graph.imageMutex.Unlock(img.id) continue } + p.graph.imageMutex.Unlock(img.id) - out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Pulling fs layer", nil)) + out.Write(p.sf.FormatProgress(stringid.TruncateID(img.id), "Pulling fs layer", nil)) - downloads[i].err = make(chan error) + downloads[i].err = make(chan error, 1) downloads[i].out = pipeWriter go p.download(&downloads[i]) } @@ -289,32 +362,54 @@ func (p *v2Puller) pullV2Tag(tag, taggedName string) (verified bool, err error) defer d.tmpFile.Close() d.tmpFile.Seek(0, 0) if d.tmpFile != nil { + err := func() error { + reader := progressreader.New(progressreader.Config{ + In: d.tmpFile, + Out: out, + Formatter: p.sf, + Size: int(d.size), + NewLines: false, + ID: stringid.TruncateID(d.img.id), + Action: "Extracting", + }) - reader := progressreader.New(progressreader.Config{ - In: d.tmpFile, - Out: out, - Formatter: p.sf, - Size: int(d.size), - NewLines: false, - ID: stringid.TruncateID(d.img.ID), - Action: "Extracting", - }) + p.graph.imageMutex.Lock(d.img.id) + defer p.graph.imageMutex.Unlock(d.img.id) - err = p.graph.Register(d.img, reader) + // Must recheck the data on disk if any exists. + // This protects against races where something + // else is written to the graph under this ID + // after attemptIDReuse. + if p.graph.Exists(d.img.id) { + if err := p.validateImageInGraph(d.img.id, imgs, i); err != nil { + return fmt.Errorf("image validation failed: %v", err) + } + } + + if err := p.graph.register(d.img, reader); err != nil { + return err + } + + if err := p.graph.setLayerDigest(d.img.id, d.digest); err != nil { + return err + } + + if err := p.graph.setV1CompatibilityConfig(d.img.id, d.img.v1Compatibility); err != nil { + return err + } + + return nil + }() if err != nil { return false, err } - if err := p.graph.SetDigest(d.img.ID, d.digest); err != nil { - return false, err - } - // FIXME: Pool release here for parallel tag pull (ensures any downloads block until fully extracted) } - out.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.ID), "Pull complete", nil)) + out.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.id), "Pull complete", nil)) tagUpdated = true } else { - out.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.ID), "Already exists", nil)) + out.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.id), "Already exists", nil)) } } @@ -347,12 +442,12 @@ func (p *v2Puller) pullV2Tag(tag, taggedName string) (verified bool, err error) // use the digest whether we pull by it or not. Unfortunately, the tag // store treats the digest as a separate tag, meaning there may be an // untagged digest image that would seem to be dangling by a user. - if err = p.SetDigest(p.repoInfo.LocalName, tag, downloads[0].img.ID); err != nil { + if err = p.SetDigest(p.repoInfo.LocalName, tag, downloads[0].img.id); err != nil { return false, err } } else { // only set the repository/tag -> image ID mapping when pulling by tag (i.e. not by digest) - if err = p.Tag(p.repoInfo.LocalName, tag, downloads[0].img.ID, true); err != nil { + if err = p.Tag(p.repoInfo.LocalName, tag, downloads[0].img.id, true); err != nil { return false, err } } @@ -443,3 +538,218 @@ func (p *v2Puller) validateManifest(m *manifest.SignedManifest, tag string) (ver } return verified, nil } + +// fixManifestLayers removes repeated layers from the manifest and checks the +// correctness of the parent chain. +func fixManifestLayers(m *manifest.Manifest) error { + images := make([]*image.Image, len(m.FSLayers)) + for i := range m.FSLayers { + img, err := image.NewImgJSON([]byte(m.History[i].V1Compatibility)) + if err != nil { + return err + } + images[i] = img + if err := image.ValidateID(img.ID); err != nil { + return err + } + } + + if images[len(images)-1].Parent != "" { + return errors.New("Invalid parent ID in the base layer of the image.") + } + + // check general duplicates to error instead of a deadlock + idmap := make(map[string]struct{}) + + var lastID string + for _, img := range images { + // skip IDs that appear after each other, we handle those later + if _, exists := idmap[img.ID]; img.ID != lastID && exists { + return fmt.Errorf("ID %+v appears multiple times in manifest.", img.ID) + } + lastID = img.ID + idmap[lastID] = struct{}{} + } + + // backwards loop so that we keep the remaining indexes after removing items + for i := len(images) - 2; i >= 0; i-- { + if images[i].ID == images[i+1].ID { // repeated ID. remove and continue + m.FSLayers = append(m.FSLayers[:i], m.FSLayers[i+1:]...) + m.History = append(m.History[:i], m.History[i+1:]...) + } else if images[i].Parent != images[i+1].ID { + return fmt.Errorf("Invalid parent ID. Expected %v, got %v.", images[i+1].ID, images[i].Parent) + } + } + + return nil +} + +// getImageInfos returns an imageinfo struct for every image in the manifest. +// These objects contain both calculated strongIDs and compatibilityIDs found +// in v1Compatibility object. +func (p *v2Puller) getImageInfos(m manifest.Manifest) ([]contentAddressableDescriptor, error) { + imgs := make([]contentAddressableDescriptor, len(m.FSLayers)) + + var parent digest.Digest + for i := len(imgs) - 1; i >= 0; i-- { + var err error + imgs[i], err = newContentAddressableImage([]byte(m.History[i].V1Compatibility), m.FSLayers[i].BlobSum, parent) + if err != nil { + return nil, err + } + parent = imgs[i].strongID + } + + p.attemptIDReuse(imgs) + + return imgs, nil +} + +var idReuseLock sync.Mutex + +// attemptIDReuse does a best attempt to match verified compatibilityIDs +// already in the graph with the computed strongIDs so we can keep using them. +// This process will never fail but may just return the strongIDs if none of +// the compatibilityIDs exists or can be verified. If the strongIDs themselves +// fail verification, we deterministically generate alternate IDs to use until +// we find one that's available or already exists with the correct data. +func (p *v2Puller) attemptIDReuse(imgs []contentAddressableDescriptor) { + // This function needs to be protected with a global lock, because it + // locks multiple IDs at once, and there's no good way to make sure + // the locking happens a deterministic order. + idReuseLock.Lock() + defer idReuseLock.Unlock() + + idMap := make(map[string]struct{}) + for _, img := range imgs { + idMap[img.id] = struct{}{} + idMap[img.compatibilityID] = struct{}{} + + if p.graph.Exists(img.compatibilityID) { + if _, err := p.graph.GenerateV1CompatibilityChain(img.compatibilityID); err != nil { + logrus.Debugf("Migration v1Compatibility generation error: %v", err) + return + } + } + } + for id := range idMap { + p.graph.imageMutex.Lock(id) + defer p.graph.imageMutex.Unlock(id) + } + + // continueReuse controls whether the function will try to find + // existing layers on disk under the old v1 IDs, to avoid repulling + // them. The hashes are checked to ensure these layers are okay to + // use. continueReuse starts out as true, but is set to false if + // the code encounters something that doesn't match the expected hash. + continueReuse := true + + for i := len(imgs) - 1; i >= 0; i-- { + if p.graph.Exists(imgs[i].id) { + // Found an image in the graph under the strongID. Validate the + // image before using it. + if err := p.validateImageInGraph(imgs[i].id, imgs, i); err != nil { + continueReuse = false + logrus.Debugf("not using existing strongID: %v", err) + + // The strong ID existed in the graph but didn't + // validate successfully. We can't use the strong ID + // because it didn't validate successfully. Treat the + // graph like a hash table with probing... compute + // SHA256(id) until we find an ID that either doesn't + // already exist in the graph, or has existing content + // that validates successfully. + for { + if err := p.tryNextID(imgs, i, idMap); err != nil { + logrus.Debug(err.Error()) + } else { + break + } + } + } + continue + } + + if continueReuse { + compatibilityID := imgs[i].compatibilityID + if err := p.validateImageInGraph(compatibilityID, imgs, i); err != nil { + logrus.Debugf("stopping ID reuse: %v", err) + continueReuse = false + } else { + // The compatibility ID exists in the graph and was + // validated. Use it. + imgs[i].id = compatibilityID + } + } + } + + // fix up the parents of the images + for i := 0; i < len(imgs); i++ { + if i == len(imgs)-1 { // Base layer + imgs[i].parent = "" + } else { + imgs[i].parent = imgs[i+1].id + } + } +} + +// validateImageInGraph checks that an image in the graph has the expected +// strongID. id is the entry in the graph to check, imgs is the slice of +// images being processed (for access to the parent), and i is the index +// into this slice which the graph entry should be checked against. +func (p *v2Puller) validateImageInGraph(id string, imgs []contentAddressableDescriptor, i int) error { + img, err := p.graph.Get(id) + if err != nil { + return fmt.Errorf("missing: %v", err) + } + layerID, err := p.graph.getLayerDigest(id) + if err != nil { + return fmt.Errorf("digest: %v", err) + } + var parentID digest.Digest + if i != len(imgs)-1 { + if img.Parent != imgs[i+1].id { // comparing that graph points to validated ID + return fmt.Errorf("parent: %v %v", img.Parent, imgs[i+1].id) + } else { + parentID = imgs[i+1].strongID + } + } else if img.Parent != "" { + return fmt.Errorf("unexpected parent: %v", img.Parent) + } + + v1Config, err := p.graph.getV1CompatibilityConfig(img.ID) + if err != nil { + return fmt.Errorf("v1Compatibility: %v %v", img.ID, err) + } + + json, err := image.MakeImageConfig(v1Config, layerID, parentID) + if err != nil { + return fmt.Errorf("make config: %v", err) + } + + if dgst, err := image.StrongID(json); err == nil && dgst == imgs[i].strongID { + logrus.Debugf("Validated %v as %v", dgst, id) + } else { + return fmt.Errorf("digest mismatch: %v %v, error: %v", dgst, imgs[i].strongID, err) + } + + // All clear + return nil +} + +func (p *v2Puller) tryNextID(imgs []contentAddressableDescriptor, i int, idMap map[string]struct{}) error { + nextID, _ := digest.FromBytes([]byte(imgs[i].id)) + imgs[i].id = nextID.Hex() + + if _, exists := idMap[imgs[i].id]; !exists { + p.graph.imageMutex.Lock(imgs[i].id) + defer p.graph.imageMutex.Unlock(imgs[i].id) + } + + if p.graph.Exists(imgs[i].id) { + if err := p.validateImageInGraph(imgs[i].id, imgs, i); err != nil { + return fmt.Errorf("not using existing strongID permutation %s: %v", imgs[i].id, err) + } + } + return nil +} diff --git a/graph/pull_v2_test.go b/graph/pull_v2_test.go new file mode 100644 index 0000000000..a1777b81b8 --- /dev/null +++ b/graph/pull_v2_test.go @@ -0,0 +1,100 @@ +package graph + +import ( + "reflect" + "strings" + "testing" + + "github.com/docker/distribution/digest" + "github.com/docker/distribution/manifest" +) + +// TestFixManifestLayers checks that fixManifestLayers removes a duplicate +// layer, and that it makes no changes to the manifest when called a second +// time, after the duplicate is removed. +func TestFixManifestLayers(t *testing.T) { + duplicateLayerManifest := manifest.Manifest{ + FSLayers: []manifest.FSLayer{ + {BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")}, + {BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")}, + {BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")}, + }, + History: []manifest.History{ + {V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"}, + {V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"}, + {V1Compatibility: "{\"id\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:07.568027497Z\",\"container\":\"fe9e5a5264a843c9292d17b736c92dd19bdb49986a8782d7389964ddaff887cc\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"cd /go/src/github.com/tonistiigi/dnsdock \\u0026\\u0026 go get -v github.com/tools/godep \\u0026\\u0026 godep restore \\u0026\\u0026 go install -ldflags \\\"-X main.version `git describe --tags HEAD``if [[ -n $(command git status --porcelain --untracked-files=no 2\\u003e/dev/null) ]]; then echo \\\"-dirty\\\"; fi`\\\" ./...\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/bash\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":118430532}\n"}, + }, + } + + duplicateLayerManifestExpectedOutput := manifest.Manifest{ + FSLayers: []manifest.FSLayer{ + {BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")}, + {BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")}, + }, + History: []manifest.History{ + {V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"}, + {V1Compatibility: "{\"id\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:07.568027497Z\",\"container\":\"fe9e5a5264a843c9292d17b736c92dd19bdb49986a8782d7389964ddaff887cc\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"cd /go/src/github.com/tonistiigi/dnsdock \\u0026\\u0026 go get -v github.com/tools/godep \\u0026\\u0026 godep restore \\u0026\\u0026 go install -ldflags \\\"-X main.version `git describe --tags HEAD``if [[ -n $(command git status --porcelain --untracked-files=no 2\\u003e/dev/null) ]]; then echo \\\"-dirty\\\"; fi`\\\" ./...\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/bash\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":118430532}\n"}, + }, + } + + if err := fixManifestLayers(&duplicateLayerManifest); err != nil { + t.Fatalf("unexpected error from fixManifestLayers: %v", err) + } + + if !reflect.DeepEqual(duplicateLayerManifest, duplicateLayerManifestExpectedOutput) { + t.Fatal("incorrect output from fixManifestLayers on duplicate layer manifest") + } + + // Run fixManifestLayers again and confirm that it doesn't change the + // manifest (which no longer has duplicate layers). + if err := fixManifestLayers(&duplicateLayerManifest); err != nil { + t.Fatalf("unexpected error from fixManifestLayers: %v", err) + } + + if !reflect.DeepEqual(duplicateLayerManifest, duplicateLayerManifestExpectedOutput) { + t.Fatal("incorrect output from fixManifestLayers on duplicate layer manifest (second pass)") + } +} + +// TestFixManifestLayersBaseLayerParent makes sure that fixManifestLayers fails +// if the base layer configuration specifies a parent. +func TestFixManifestLayersBaseLayerParent(t *testing.T) { + duplicateLayerManifest := manifest.Manifest{ + FSLayers: []manifest.FSLayer{ + {BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")}, + {BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")}, + {BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")}, + }, + History: []manifest.History{ + {V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"}, + {V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"}, + {V1Compatibility: "{\"id\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"parent\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"created\":\"2015-08-19T16:49:07.568027497Z\",\"container\":\"fe9e5a5264a843c9292d17b736c92dd19bdb49986a8782d7389964ddaff887cc\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"cd /go/src/github.com/tonistiigi/dnsdock \\u0026\\u0026 go get -v github.com/tools/godep \\u0026\\u0026 godep restore \\u0026\\u0026 go install -ldflags \\\"-X main.version `git describe --tags HEAD``if [[ -n $(command git status --porcelain --untracked-files=no 2\\u003e/dev/null) ]]; then echo \\\"-dirty\\\"; fi`\\\" ./...\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/bash\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":118430532}\n"}, + }, + } + + if err := fixManifestLayers(&duplicateLayerManifest); err == nil || !strings.Contains(err.Error(), "Invalid parent ID in the base layer of the image.") { + t.Fatalf("expected an invalid parent ID error from fixManifestLayers") + } +} + +// TestFixManifestLayersBadParent makes sure that fixManifestLayers fails +// if an image configuration specifies a parent that doesn't directly follow +// that (deduplicated) image in the image history. +func TestFixManifestLayersBadParent(t *testing.T) { + duplicateLayerManifest := manifest.Manifest{ + FSLayers: []manifest.FSLayer{ + {BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")}, + {BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")}, + {BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")}, + }, + History: []manifest.History{ + {V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ac3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"}, + {V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ac3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"}, + {V1Compatibility: "{\"id\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:07.568027497Z\",\"container\":\"fe9e5a5264a843c9292d17b736c92dd19bdb49986a8782d7389964ddaff887cc\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"cd /go/src/github.com/tonistiigi/dnsdock \\u0026\\u0026 go get -v github.com/tools/godep \\u0026\\u0026 godep restore \\u0026\\u0026 go install -ldflags \\\"-X main.version `git describe --tags HEAD``if [[ -n $(command git status --porcelain --untracked-files=no 2\\u003e/dev/null) ]]; then echo \\\"-dirty\\\"; fi`\\\" ./...\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/bash\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":118430532}\n"}, + }, + } + + if err := fixManifestLayers(&duplicateLayerManifest); err == nil || !strings.Contains(err.Error(), "Invalid parent ID.") { + t.Fatalf("expected an invalid parent ID error from fixManifestLayers") + } +} diff --git a/graph/push_v1.go b/graph/push_v1.go index d473040f88..71d4c8e8db 100644 --- a/graph/push_v1.go +++ b/graph/push_v1.go @@ -8,6 +8,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/distribution/registry/client/transport" + "github.com/docker/docker/image" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/progressreader" "github.com/docker/docker/pkg/streamformatter" @@ -127,7 +128,7 @@ func (s *TagStore) createImageIndex(images []string, tags map[string][]string) [ continue } // If the image does not have a tag it still needs to be sent to the - // registry with an empty tag so that it is accociated with the repository + // registry with an empty tag so that it is associated with the repository imageIndex = append(imageIndex, ®istry.ImgData{ ID: id, Tag: "", @@ -137,9 +138,10 @@ func (s *TagStore) createImageIndex(images []string, tags map[string][]string) [ } type imagePushData struct { - id string - endpoint string - tokens []string + id string + compatibilityID string + endpoint string + tokens []string } // lookupImageOnEndpoint checks the specified endpoint to see if an image exists @@ -147,7 +149,7 @@ type imagePushData struct { func (p *v1Pusher) lookupImageOnEndpoint(wg *sync.WaitGroup, images chan imagePushData, imagesToPush chan string) { defer wg.Done() for image := range images { - if err := p.session.LookupRemoteImage(image.id, image.endpoint); err != nil { + if err := p.session.LookupRemoteImage(image.compatibilityID, image.endpoint); err != nil { logrus.Errorf("Error in LookupRemoteImage: %s", err) imagesToPush <- image.id continue @@ -181,10 +183,15 @@ func (p *v1Pusher) pushImageToEndpoint(endpoint string, imageIDs []string, tags pushes <- shouldPush }() for _, id := range imageIDs { + compatibilityID, err := p.getV1ID(id) + if err != nil { + return err + } imageData <- imagePushData{ - id: id, - endpoint: endpoint, - tokens: repo.Tokens, + id: id, + compatibilityID: compatibilityID, + endpoint: endpoint, + tokens: repo.Tokens, } } // close the channel to notify the workers that there will be no more images to check. @@ -204,7 +211,11 @@ func (p *v1Pusher) pushImageToEndpoint(endpoint string, imageIDs []string, tags } for _, tag := range tags[id] { p.out.Write(p.sf.FormatStatus("", "Pushing tag for rev [%s] on {%s}", stringid.TruncateID(id), endpoint+"repositories/"+p.repoInfo.RemoteName+"/tags/"+tag)) - if err := p.session.PushRegistryTag(p.repoInfo.RemoteName, id, tag, endpoint); err != nil { + compatibilityID, err := p.getV1ID(id) + if err != nil { + return err + } + if err := p.session.PushRegistryTag(p.repoInfo.RemoteName, compatibilityID, tag, endpoint); err != nil { return err } } @@ -214,7 +225,6 @@ func (p *v1Pusher) pushImageToEndpoint(endpoint string, imageIDs []string, tags // pushRepository pushes layers that do not already exist on the registry. func (p *v1Pusher) pushRepository(tag string) error { - logrus.Debugf("Local repo: %s", p.localRepo) p.out = ioutils.NewWriteFlusher(p.config.OutStream) imgList, tags, err := p.getImageList(tag) @@ -227,6 +237,12 @@ func (p *v1Pusher) pushRepository(tag string) error { logrus.Debugf("Preparing to push %s with the following images and tags", p.localRepo) for _, data := range imageIndex { logrus.Debugf("Pushing ID: %s with Tag: %s", data.ID, data.Tag) + + // convert IDs to compatibilityIDs, imageIndex only used in registry calls + data.ID, err = p.getV1ID(data.ID) + if err != nil { + return err + } } if _, err := p.poolAdd("push", p.repoInfo.LocalName); err != nil { @@ -256,20 +272,27 @@ func (p *v1Pusher) pushRepository(tag string) error { } func (p *v1Pusher) pushImage(imgID, ep string, token []string) (checksum string, err error) { - jsonRaw, err := p.graph.RawJSON(imgID) + jsonRaw, err := p.getV1Config(imgID) if err != nil { return "", fmt.Errorf("Cannot retrieve the path for {%s}: %s", imgID, err) } p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Pushing", nil)) + compatibilityID, err := p.getV1ID(imgID) + if err != nil { + return "", err + } + + // General rule is to use ID for graph accesses and compatibilityID for + // calls to session.registry() imgData := ®istry.ImgData{ - ID: imgID, + ID: compatibilityID, } // Send the json if err := p.session.PushImageJSONRegistry(imgData, jsonRaw, ep); err != nil { if err == registry.ErrAlreadyExists { - p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgData.ID), "Image already pushed, skipping", nil)) + p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Image already pushed, skipping", nil)) return "", nil } return "", err @@ -282,7 +305,7 @@ func (p *v1Pusher) pushImage(imgID, ep string, token []string) (checksum string, defer os.RemoveAll(layerData.Name()) // Send the layer - logrus.Debugf("rendered layer for %s of [%d] size", imgData.ID, layerData.Size) + logrus.Debugf("rendered layer for %s of [%d] size", imgID, layerData.Size) checksum, checksumPayload, err := p.session.PushImageLayerRegistry(imgData.ID, progressreader.New(progressreader.Config{ @@ -291,7 +314,7 @@ func (p *v1Pusher) pushImage(imgID, ep string, token []string) (checksum string, Formatter: p.sf, Size: int(layerData.Size), NewLines: false, - ID: stringid.TruncateID(imgData.ID), + ID: stringid.TruncateID(imgID), Action: "Pushing", }), ep, jsonRaw) if err != nil { @@ -304,6 +327,30 @@ func (p *v1Pusher) pushImage(imgID, ep string, token []string) (checksum string, return "", err } - p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgData.ID), "Image successfully pushed", nil)) + p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Image successfully pushed", nil)) return imgData.Checksum, nil } + +// getV1ID returns the compatibilityID for the ID in the graph. compatibilityID +// is read from from the v1Compatibility config file in the disk. +func (p *v1Pusher) getV1ID(id string) (string, error) { + jsonData, err := p.getV1Config(id) + if err != nil { + return "", err + } + img, err := image.NewImgJSON(jsonData) + if err != nil { + return "", err + } + return img.ID, nil +} + +// getV1Config returns v1Compatibility config for the image in the graph. If +// there is no v1Compatibility file on disk for the image +func (p *v1Pusher) getV1Config(id string) ([]byte, error) { + jsonData, err := p.graph.GenerateV1CompatibilityChain(id) + if err != nil { + return nil, err + } + return jsonData, nil +} diff --git a/graph/push_v2.go b/graph/push_v2.go index 92d63ca4a6..3f23299171 100644 --- a/graph/push_v2.go +++ b/graph/push_v2.go @@ -129,13 +129,8 @@ func (p *v2Pusher) pushV2Tag(tag string) error { } } - jsonData, err := p.graph.RawJSON(layer.ID) - if err != nil { - return fmt.Errorf("cannot retrieve the path for %s: %s", layer.ID, err) - } - var exists bool - dgst, err := p.graph.GetDigest(layer.ID) + dgst, err := p.graph.GetLayerDigest(layer.ID) switch err { case nil: _, err := p.repo.Blobs(context.Background()).Stat(context.Background(), dgst) @@ -162,13 +157,19 @@ func (p *v2Pusher) pushV2Tag(tag string) error { return err } else if pushDigest != dgst { // Cache new checksum - if err := p.graph.SetDigest(layer.ID, pushDigest); err != nil { + if err := p.graph.SetLayerDigest(layer.ID, pushDigest); err != nil { return err } dgst = pushDigest } } + // read v1Compatibility config, generate new if needed + jsonData, err := p.graph.GenerateV1CompatibilityChain(layer.ID) + if err != nil { + return err + } + m.FSLayers = append(m.FSLayers, manifest.FSLayer{BlobSum: dgst}) m.History = append(m.History, manifest.History{V1Compatibility: string(jsonData)}) diff --git a/graph/service.go b/graph/service.go index a7b9b49359..0066bb1a15 100644 --- a/graph/service.go +++ b/graph/service.go @@ -10,20 +10,6 @@ import ( "github.com/docker/docker/api/types" ) -func (s *TagStore) LookupRaw(name string) ([]byte, error) { - image, err := s.LookupImage(name) - if err != nil || image == nil { - return nil, fmt.Errorf("No such image %s", name) - } - - imageInspectRaw, err := s.graph.RawJSON(image.ID) - if err != nil { - return nil, err - } - - return imageInspectRaw, nil -} - // Lookup return an image encoded in JSON func (s *TagStore) Lookup(name string) (*types.ImageInspect, error) { image, err := s.LookupImage(name) diff --git a/graph/tags_unit_test.go b/graph/tags_unit_test.go index 8dffb2a6bc..32e9246036 100644 --- a/graph/tags_unit_test.go +++ b/graph/tags_unit_test.go @@ -81,7 +81,7 @@ func mkTestTagStore(root string, t *testing.T) *TagStore { t.Fatal(err) } img := &image.Image{ID: testOfficialImageID} - if err := graph.Register(img, officialArchive); err != nil { + if err := graph.Register(v1ImageDescriptor{img}, officialArchive); err != nil { t.Fatal(err) } if err := store.Tag(testOfficialImageName, "", testOfficialImageID, false); err != nil { @@ -92,7 +92,7 @@ func mkTestTagStore(root string, t *testing.T) *TagStore { t.Fatal(err) } img = &image.Image{ID: testPrivateImageID} - if err := graph.Register(img, privateArchive); err != nil { + if err := graph.Register(v1ImageDescriptor{img}, privateArchive); err != nil { t.Fatal(err) } if err := store.Tag(testPrivateImageName, "", testPrivateImageID, false); err != nil { diff --git a/image/fixtures/post1.9/expected_computed_id b/image/fixtures/post1.9/expected_computed_id new file mode 100644 index 0000000000..cba6d81f4e --- /dev/null +++ b/image/fixtures/post1.9/expected_computed_id @@ -0,0 +1 @@ +sha256:f2722a8ec6926e02fa9f2674072cbc2a25cf0f449f27350f613cd843b02c9105 diff --git a/image/fixtures/post1.9/expected_config b/image/fixtures/post1.9/expected_config new file mode 100644 index 0000000000..ae27bdd429 --- /dev/null +++ b/image/fixtures/post1.9/expected_config @@ -0,0 +1 @@ +{"architecture":"amd64","config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":null,"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"fb1f7270da9519308361b99dc8e0d30f12c24dfd28537c2337ece995ac853a16","container_config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":["/bin/sh","-c","#(nop) ADD file:11998b2a4d664a75cd0c3f4e4cb1837434e0f997ba157a0ac1d3c68a07aa2f4f in /"],"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2015-09-08T21:30:30.807853054Z","docker_version":"1.9.0-dev","layer_id":"sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a","os":"linux","parent_id":"sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02"} diff --git a/image/fixtures/post1.9/layer_id b/image/fixtures/post1.9/layer_id new file mode 100644 index 0000000000..ded2db28e7 --- /dev/null +++ b/image/fixtures/post1.9/layer_id @@ -0,0 +1 @@ +sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a diff --git a/image/fixtures/post1.9/parent_id b/image/fixtures/post1.9/parent_id new file mode 100644 index 0000000000..7d524f80c2 --- /dev/null +++ b/image/fixtures/post1.9/parent_id @@ -0,0 +1 @@ +sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02 diff --git a/image/fixtures/post1.9/v1compatibility b/image/fixtures/post1.9/v1compatibility new file mode 100644 index 0000000000..d6697c2b68 --- /dev/null +++ b/image/fixtures/post1.9/v1compatibility @@ -0,0 +1 @@ +{"id":"8dfb96b5d09e6cf6f376d81f1e2770ee5ede309f9bd9e079688c9782649ab326","parent":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","created":"2015-09-08T21:30:30.807853054Z","container":"fb1f7270da9519308361b99dc8e0d30f12c24dfd28537c2337ece995ac853a16","container_config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":["/bin/sh","-c","#(nop) ADD file:11998b2a4d664a75cd0c3f4e4cb1837434e0f997ba157a0ac1d3c68a07aa2f4f in /"],"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"docker_version":"1.9.0-dev","config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":null,"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"architecture":"amd64","os":"linux"} diff --git a/image/fixtures/pre1.9/expected_computed_id b/image/fixtures/pre1.9/expected_computed_id new file mode 100644 index 0000000000..c27b0b6a20 --- /dev/null +++ b/image/fixtures/pre1.9/expected_computed_id @@ -0,0 +1 @@ +sha256:fd6ebfedda8ea140a9380767e15bd32c6e899303cfe34bc4580c931f2f816f89 diff --git a/image/fixtures/pre1.9/expected_config b/image/fixtures/pre1.9/expected_config new file mode 100644 index 0000000000..83fc30487a --- /dev/null +++ b/image/fixtures/pre1.9/expected_config @@ -0,0 +1 @@ +{"architecture":"amd64","config":{"Hostname":"03797203757d","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":null,"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"OnBuild":[],"Labels":{}},"container":"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253","container_config":{"Hostname":"03797203757d","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":["/bin/sh","-c","#(nop) ENTRYPOINT [\"/go/bin/dnsdock\"]"],"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"OnBuild":[],"Labels":{}},"created":"2015-08-19T16:49:11.368300679Z","docker_version":"1.6.2","layer_id":"sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a","os":"linux","parent_id":"sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02"} diff --git a/image/fixtures/pre1.9/layer_id b/image/fixtures/pre1.9/layer_id new file mode 100644 index 0000000000..ded2db28e7 --- /dev/null +++ b/image/fixtures/pre1.9/layer_id @@ -0,0 +1 @@ +sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a diff --git a/image/fixtures/pre1.9/parent_id b/image/fixtures/pre1.9/parent_id new file mode 100644 index 0000000000..7d524f80c2 --- /dev/null +++ b/image/fixtures/pre1.9/parent_id @@ -0,0 +1 @@ +sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02 diff --git a/image/fixtures/pre1.9/v1compatibility b/image/fixtures/pre1.9/v1compatibility new file mode 100644 index 0000000000..af96e82506 --- /dev/null +++ b/image/fixtures/pre1.9/v1compatibility @@ -0,0 +1 @@ +{"id":"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9","parent":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","created":"2015-08-19T16:49:11.368300679Z","container":"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253","container_config":{"Hostname":"03797203757d","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":["/bin/sh","-c","#(nop) ENTRYPOINT [\"/go/bin/dnsdock\"]"],"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"NetworkDisabled":false,"MacAddress":"","OnBuild":[],"Labels":{}},"docker_version":"1.6.2","config":{"Hostname":"03797203757d","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":null,"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"NetworkDisabled":false,"MacAddress":"","OnBuild":[],"Labels":{}},"architecture":"amd64","os":"linux","Size":0} diff --git a/image/image.go b/image/image.go index 218f18f2d7..3f72b0d07f 100644 --- a/image/image.go +++ b/image/image.go @@ -6,24 +6,59 @@ import ( "regexp" "time" + "github.com/Sirupsen/logrus" + "github.com/docker/distribution/digest" + "github.com/docker/docker/pkg/version" "github.com/docker/docker/runconfig" ) var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`) +// noFallbackMinVersion is the minimum version for which v1compatibility +// information will not be marshaled through the Image struct to remove +// blank fields. +var noFallbackMinVersion = version.Version("1.8.3") + +// ImageDescriptor provides the information necessary to register an image in +// the graph. +type ImageDescriptor interface { + ID() string + Parent() string + MarshalConfig() ([]byte, error) +} + +// Image stores the image configuration. +// All fields in this struct must be marked `omitempty` to keep getting +// predictable hashes from the old `v1Compatibility` configuration. type Image struct { - ID string `json:"id"` - Parent string `json:"parent,omitempty"` - Comment string `json:"comment,omitempty"` - Created time.Time `json:"created"` - Container string `json:"container,omitempty"` - ContainerConfig runconfig.Config `json:"container_config,omitempty"` - DockerVersion string `json:"docker_version,omitempty"` - Author string `json:"author,omitempty"` - Config *runconfig.Config `json:"config,omitempty"` - Architecture string `json:"architecture,omitempty"` - OS string `json:"os,omitempty"` - Size int64 + // ID a unique 64 character identifier of the image + ID string `json:"id,omitempty"` + // Parent id of the image + Parent string `json:"parent,omitempty"` + // Comment user added comment + Comment string `json:"comment,omitempty"` + // Created timestamp when image was created + Created time.Time `json:"created"` + // Container is the id of the container used to commit + Container string `json:"container,omitempty"` + // ContainerConfig is the configuration of the container that is committed into the image + ContainerConfig runconfig.Config `json:"container_config,omitempty"` + // DockerVersion specifies version on which image is built + DockerVersion string `json:"docker_version,omitempty"` + // Author of the image + Author string `json:"author,omitempty"` + // Config is the configuration of the container received from the client + Config *runconfig.Config `json:"config,omitempty"` + // Architecture is the hardware that the image is build and runs on + Architecture string `json:"architecture,omitempty"` + // OS is the operating system used to build and run the image + OS string `json:"os,omitempty"` + // Size is the total size of the image including all layers it is composed of + Size int64 `json:",omitempty"` // capitalized for backwards compatibility + // ParentID specifies the strong, content address of the parent configuration. + ParentID digest.Digest `json:"parent_id,omitempty"` + // LayerID provides the content address of the associated layer. + LayerID digest.Digest `json:"layer_id,omitempty"` } // Build an Image object from raw json data @@ -44,3 +79,70 @@ func ValidateID(id string) error { } return nil } + +// MakeImageConfig returns immutable configuration JSON for image based on the +// v1Compatibility object, layer digest and parent StrongID. SHA256() of this +// config is the new image ID (strongID). +func MakeImageConfig(v1Compatibility []byte, layerID, parentID digest.Digest) ([]byte, error) { + + // Detect images created after 1.8.3 + img, err := NewImgJSON(v1Compatibility) + if err != nil { + return nil, err + } + useFallback := version.Version(img.DockerVersion).LessThan(noFallbackMinVersion) + + if useFallback { + // Fallback for pre-1.8.3. Calculate base config based on Image struct + // so that fields with default values added by Docker will use same ID + logrus.Debugf("Using fallback hash for %v", layerID) + + v1Compatibility, err = json.Marshal(img) + if err != nil { + return nil, err + } + } + + var c map[string]*json.RawMessage + if err := json.Unmarshal(v1Compatibility, &c); err != nil { + return nil, err + } + + if err := layerID.Validate(); err != nil { + return nil, fmt.Errorf("invalid layerID: %v", err) + } + + c["layer_id"] = rawJSON(layerID) + + if parentID != "" { + if err := parentID.Validate(); err != nil { + return nil, fmt.Errorf("invalid parentID %v", err) + } + c["parent_id"] = rawJSON(parentID) + } + + delete(c, "id") + delete(c, "parent") + delete(c, "Size") // Size is calculated from data on disk and is inconsitent + + return json.Marshal(c) +} + +// StrongID returns image ID for the config JSON. +func StrongID(configJSON []byte) (digest.Digest, error) { + digester := digest.Canonical.New() + if _, err := digester.Hash().Write(configJSON); err != nil { + return "", err + } + dgst := digester.Digest() + logrus.Debugf("H(%v) = %v", string(configJSON), dgst) + return dgst, nil +} + +func rawJSON(value interface{}) *json.RawMessage { + jsonval, err := json.Marshal(value) + if err != nil { + return nil + } + return (*json.RawMessage)(&jsonval) +} diff --git a/image/image_test.go b/image/image_test.go new file mode 100644 index 0000000000..77d92c4490 --- /dev/null +++ b/image/image_test.go @@ -0,0 +1,55 @@ +package image + +import ( + "bytes" + "io/ioutil" + "testing" + + "github.com/docker/distribution/digest" +) + +var fixtures = []string{ + "fixtures/pre1.9", + "fixtures/post1.9", +} + +func loadFixtureFile(t *testing.T, path string) []byte { + fileData, err := ioutil.ReadFile(path) + if err != nil { + t.Fatalf("error opening %s: %v", path, err) + } + + return bytes.TrimSpace(fileData) +} + +// TestMakeImageConfig makes sure that MakeImageConfig returns the expected +// canonical JSON for a reference Image. +func TestMakeImageConfig(t *testing.T) { + for _, fixture := range fixtures { + v1Compatibility := loadFixtureFile(t, fixture+"/v1compatibility") + expectedConfig := loadFixtureFile(t, fixture+"/expected_config") + layerID := digest.Digest(loadFixtureFile(t, fixture+"/layer_id")) + parentID := digest.Digest(loadFixtureFile(t, fixture+"/parent_id")) + + json, err := MakeImageConfig(v1Compatibility, layerID, parentID) + if err != nil { + t.Fatalf("MakeImageConfig on %s returned error: %v", fixture, err) + } + if !bytes.Equal(json, expectedConfig) { + t.Fatalf("did not get expected JSON for %s\nexpected: %s\ngot: %s", fixture, expectedConfig, json) + } + } +} + +// TestGetStrongID makes sure that GetConfigJSON returns the expected +// hash for a reference Image. +func TestGetStrongID(t *testing.T) { + for _, fixture := range fixtures { + expectedConfig := loadFixtureFile(t, fixture+"/expected_config") + expectedComputedID := digest.Digest(loadFixtureFile(t, fixture+"/expected_computed_id")) + + if id, err := StrongID(expectedConfig); err != nil || id != expectedComputedID { + t.Fatalf("did not get expected ID for %s\nexpected: %s\ngot: %s\nerror: %v", fixture, expectedComputedID, id, err) + } + } +} diff --git a/integration-cli/docker_cli_pull_test.go b/integration-cli/docker_cli_pull_test.go index 67965ab721..8bfca8db1e 100644 --- a/integration-cli/docker_cli_pull_test.go +++ b/integration-cli/docker_cli_pull_test.go @@ -1,13 +1,16 @@ package main import ( + "encoding/json" "fmt" + "io/ioutil" + "os" "os/exec" + "path/filepath" "strings" "time" - "io/ioutil" - + "github.com/docker/distribution/digest" "github.com/go-check/check" ) @@ -406,3 +409,254 @@ func (s *DockerTrustSuite) TestPullClientDisconnect(c *check.C) { } } + +type idAndParent struct { + ID string + Parent string +} + +func inspectImage(c *check.C, imageRef string) idAndParent { + out, _ := dockerCmd(c, "inspect", imageRef) + var inspectOutput []idAndParent + err := json.Unmarshal([]byte(out), &inspectOutput) + if err != nil { + c.Fatal(err) + } + + return inspectOutput[0] +} + +func imageID(c *check.C, imageRef string) string { + return inspectImage(c, imageRef).ID +} + +func imageParent(c *check.C, imageRef string) string { + return inspectImage(c, imageRef).Parent +} + +// TestPullMigration verifies that pulling an image based on layers +// that already exists locally will reuse those existing layers. +func (s *DockerRegistrySuite) TestPullMigration(c *check.C) { + repoName := privateRegistryURL + "/dockercli/migration" + + baseImage := repoName + ":base" + _, err := buildImage(baseImage, fmt.Sprintf(` + FROM scratch + ENV IMAGE base + CMD echo %s + `, baseImage), true) + if err != nil { + c.Fatal(err) + } + + baseIDBeforePush := imageID(c, baseImage) + baseParentBeforePush := imageParent(c, baseImage) + + derivedImage := repoName + ":derived" + _, err = buildImage(derivedImage, fmt.Sprintf(` + FROM %s + CMD echo %s + `, baseImage, derivedImage), true) + if err != nil { + c.Fatal(err) + } + + derivedIDBeforePush := imageID(c, derivedImage) + + dockerCmd(c, "push", derivedImage) + + // Remove derived image from the local store + dockerCmd(c, "rmi", derivedImage) + + // Repull + dockerCmd(c, "pull", derivedImage) + + // Check that the parent of this pulled image is the original base + // image + derivedIDAfterPull1 := imageID(c, derivedImage) + derivedParentAfterPull1 := imageParent(c, derivedImage) + + if derivedIDAfterPull1 == derivedIDBeforePush { + c.Fatal("image's ID should have changed on after deleting and pulling") + } + + if derivedParentAfterPull1 != baseIDBeforePush { + c.Fatalf("pulled image's parent ID (%s) does not match base image's ID (%s)", derivedParentAfterPull1, baseIDBeforePush) + } + + // Confirm that repushing and repulling does not change the computed ID + dockerCmd(c, "push", derivedImage) + dockerCmd(c, "rmi", derivedImage) + dockerCmd(c, "pull", derivedImage) + + derivedIDAfterPull2 := imageID(c, derivedImage) + derivedParentAfterPull2 := imageParent(c, derivedImage) + + if derivedIDAfterPull2 != derivedIDAfterPull1 { + c.Fatal("image's ID unexpectedly changed after a repush/repull") + } + + if derivedParentAfterPull2 != baseIDBeforePush { + c.Fatalf("pulled image's parent ID (%s) does not match base image's ID (%s)", derivedParentAfterPull2, baseIDBeforePush) + } + + // Remove everything, repull, and make sure everything uses computed IDs + dockerCmd(c, "rmi", baseImage, derivedImage) + dockerCmd(c, "pull", derivedImage) + + derivedIDAfterPull3 := imageID(c, derivedImage) + derivedParentAfterPull3 := imageParent(c, derivedImage) + derivedGrandparentAfterPull3 := imageParent(c, derivedParentAfterPull3) + + if derivedIDAfterPull3 != derivedIDAfterPull1 { + c.Fatal("image's ID unexpectedly changed after a second repull") + } + + if derivedParentAfterPull3 == baseIDBeforePush { + c.Fatalf("pulled image's parent ID (%s) should not match base image's original ID (%s)", derivedParentAfterPull3, derivedIDBeforePush) + } + + if derivedGrandparentAfterPull3 == baseParentBeforePush { + c.Fatal("base image's parent ID should have been rewritten on pull") + } +} + +// TestPullMigrationRun verifies that pulling an image based on layers +// that already exists locally will result in an image that runs properly. +func (s *DockerRegistrySuite) TestPullMigrationRun(c *check.C) { + type idAndParent struct { + ID string + Parent string + } + + derivedImage := privateRegistryURL + "/dockercli/migration-run" + baseImage := "busybox" + + _, err := buildImage(derivedImage, fmt.Sprintf(` + FROM %s + RUN dd if=/dev/zero of=/file bs=1024 count=1024 + CMD echo %s + `, baseImage, derivedImage), true) + if err != nil { + c.Fatal(err) + } + + baseIDBeforePush := imageID(c, baseImage) + derivedIDBeforePush := imageID(c, derivedImage) + + dockerCmd(c, "push", derivedImage) + + // Remove derived image from the local store + dockerCmd(c, "rmi", derivedImage) + + // Repull + dockerCmd(c, "pull", derivedImage) + + // Check that this pulled image is based on the original base image + derivedIDAfterPull1 := imageID(c, derivedImage) + derivedParentAfterPull1 := imageParent(c, imageParent(c, derivedImage)) + + if derivedIDAfterPull1 == derivedIDBeforePush { + c.Fatal("image's ID should have changed on after deleting and pulling") + } + + if derivedParentAfterPull1 != baseIDBeforePush { + c.Fatalf("pulled image's parent ID (%s) does not match base image's ID (%s)", derivedParentAfterPull1, baseIDBeforePush) + } + + // Make sure the image runs correctly + out, _ := dockerCmd(c, "run", "--rm", derivedImage) + if strings.TrimSpace(out) != derivedImage { + c.Fatalf("expected %s; got %s", derivedImage, out) + } + + // Confirm that repushing and repulling does not change the computed ID + dockerCmd(c, "push", derivedImage) + dockerCmd(c, "rmi", derivedImage) + dockerCmd(c, "pull", derivedImage) + + derivedIDAfterPull2 := imageID(c, derivedImage) + derivedParentAfterPull2 := imageParent(c, imageParent(c, derivedImage)) + + if derivedIDAfterPull2 != derivedIDAfterPull1 { + c.Fatal("image's ID unexpectedly changed after a repush/repull") + } + + if derivedParentAfterPull2 != baseIDBeforePush { + c.Fatalf("pulled image's parent ID (%s) does not match base image's ID (%s)", derivedParentAfterPull2, baseIDBeforePush) + } + + // Make sure the image still runs + out, _ = dockerCmd(c, "run", "--rm", derivedImage) + if strings.TrimSpace(out) != derivedImage { + c.Fatalf("expected %s; got %s", derivedImage, out) + } +} + +// TestPullConflict provides coverage of the situation where a computed +// strongID conflicts with some unverifiable data in the graph. +func (s *DockerRegistrySuite) TestPullConflict(c *check.C) { + repoName := privateRegistryURL + "/dockercli/conflict" + + _, err := buildImage(repoName, ` + FROM scratch + ENV IMAGE conflict + CMD echo conflict + `, true) + if err != nil { + c.Fatal(err) + } + + dockerCmd(c, "push", repoName) + + // Pull to make it content-addressable + dockerCmd(c, "rmi", repoName) + dockerCmd(c, "pull", repoName) + + IDBeforeLoad := imageID(c, repoName) + + // Load/save to turn this into an unverified image with the same ID + tmpDir, err := ioutil.TempDir("", "conflict-save-output") + if err != nil { + c.Errorf("failed to create temporary directory: %s", err) + } + defer os.RemoveAll(tmpDir) + + tarFile := filepath.Join(tmpDir, "repo.tar") + + dockerCmd(c, "save", "-o", tarFile, repoName) + dockerCmd(c, "rmi", repoName) + dockerCmd(c, "load", "-i", tarFile) + + // Check that the the ID is the same after save/load. + IDAfterLoad := imageID(c, repoName) + + if IDAfterLoad != IDBeforeLoad { + c.Fatal("image's ID should be the same after save/load") + } + + // Repull + dockerCmd(c, "pull", repoName) + + // Check that the ID is now different because of the conflict. + IDAfterPull1 := imageID(c, repoName) + + // Expect the new ID to be SHA256(oldID) + expectedIDDigest, err := digest.FromBytes([]byte(IDBeforeLoad)) + if err != nil { + c.Fatalf("digest error: %v", err) + } + expectedID := expectedIDDigest.Hex() + if IDAfterPull1 != expectedID { + c.Fatalf("image's ID should have changed on pull to %s (got %s)", expectedID, IDAfterPull1) + } + + // A second pull should use the new ID again. + dockerCmd(c, "pull", repoName) + + IDAfterPull2 := imageID(c, repoName) + + if IDAfterPull2 != IDAfterPull1 { + c.Fatal("image's ID unexpectedly changed after a repull") + } +} diff --git a/runconfig/config.go b/runconfig/config.go index 2196828d10..3ab9bac238 100644 --- a/runconfig/config.go +++ b/runconfig/config.go @@ -114,29 +114,31 @@ func NewCommand(parts ...string) *Command { // Note: the Config structure should hold only portable information about the container. // Here, "portable" means "independent from the host we are running on". // Non-portable information *should* appear in HostConfig. +// All fields added to this struct must be marked `omitempty` to keep getting +// predictable hashes from the old `v1Compatibility` configuration. type Config struct { - Hostname string - Domainname string - User string - AttachStdin bool - AttachStdout bool - AttachStderr bool - ExposedPorts map[nat.Port]struct{} - PublishService string - Tty bool // Attach standard streams to a tty, including stdin if it is not closed. - OpenStdin bool // Open stdin - StdinOnce bool // If true, close stdin after the 1 attached client disconnects. - Env []string - Cmd *Command - Image string // Name of the image as it was passed by the operator (eg. could be symbolic) - Volumes map[string]struct{} - VolumeDriver string - WorkingDir string - Entrypoint *Entrypoint - NetworkDisabled bool - MacAddress string - OnBuild []string - Labels map[string]string + Hostname string // Hostname + Domainname string // Domainname + User string // User that will run the command(s) inside the container + AttachStdin bool // Attach the standard input, makes possible user interaction + AttachStdout bool // Attach the standard output + AttachStderr bool // Attach the standard error + ExposedPorts map[nat.Port]struct{} `json:",omitempty"` // List of exposed ports + PublishService string `json:",omitempty"` // Name of the network service exposed by the container + Tty bool // Attach standard streams to a tty, including stdin if it is not closed. + OpenStdin bool // Open stdin + StdinOnce bool // If true, close stdin after the 1 attached client disconnects. + Env []string // List of environment variable to set in the container + Cmd *Command // Command to run when starting the container + Image string // Name of the image as it was passed by the operator (eg. could be symbolic) + Volumes map[string]struct{} // List of volumes (mounts) used for the container + VolumeDriver string `json:",omitempty"` // Name of the volume driver used to mount volumes + WorkingDir string // Current directory (PWD) in the command will be launched + Entrypoint *Entrypoint // Entrypoint to run when starting the container + NetworkDisabled bool `json:",omitempty"` // Is network disabled + MacAddress string `json:",omitempty"` // Mac Address of the container + OnBuild []string // ONBUILD metadata that were defined on the image Dockerfile + Labels map[string]string // List of labels set to this container } type ContainerConfigWrapper struct {