diff --git a/daemon/commit.go b/daemon/commit.go index 8865d832f3..6af494da74 100644 --- a/daemon/commit.go +++ b/daemon/commit.go @@ -1,9 +1,7 @@ package daemon // import "github.com/docker/docker/daemon" import ( - "encoding/json" "fmt" - "io" "runtime" "strings" "time" @@ -12,10 +10,6 @@ import ( containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/builder/dockerfile" "github.com/docker/docker/errdefs" - "github.com/docker/docker/image" - "github.com/docker/docker/layer" - "github.com/docker/docker/pkg/ioutils" - "github.com/docker/docker/pkg/system" "github.com/pkg/errors" ) @@ -190,115 +184,3 @@ func (daemon *Daemon) CreateImageFromContainer(name string, c *backend.CreateIma containerActions.WithValues("commit").UpdateSince(start) return id.String(), nil } - -func (daemon *Daemon) commitImage(c backend.CommitConfig) (image.ID, error) { - layerStore, ok := daemon.layerStores[c.ContainerOS] - if !ok { - return "", system.ErrNotSupportedOperatingSystem - } - rwTar, err := exportContainerRw(layerStore, c.ContainerID, c.ContainerMountLabel) - if err != nil { - return "", err - } - defer func() { - if rwTar != nil { - rwTar.Close() - } - }() - - var parent *image.Image - if c.ParentImageID == "" { - parent = new(image.Image) - parent.RootFS = image.NewRootFS() - } else { - parent, err = daemon.imageStore.Get(image.ID(c.ParentImageID)) - if err != nil { - return "", err - } - } - - l, err := layerStore.Register(rwTar, parent.RootFS.ChainID()) - if err != nil { - return "", err - } - defer layer.ReleaseAndLog(layerStore, l) - - cc := image.ChildConfig{ - ContainerID: c.ContainerID, - Author: c.Author, - Comment: c.Comment, - ContainerConfig: c.ContainerConfig, - Config: c.Config, - DiffID: l.DiffID(), - } - config, err := json.Marshal(image.NewChildImage(parent, cc, c.ContainerOS)) - if err != nil { - return "", err - } - - id, err := daemon.imageStore.Create(config) - if err != nil { - return "", err - } - - if c.ParentImageID != "" { - if err := daemon.imageStore.SetParent(id, image.ID(c.ParentImageID)); err != nil { - return "", err - } - } - return id, nil -} - -func exportContainerRw(layerStore layer.Store, id, mountLabel string) (arch io.ReadCloser, err error) { - rwlayer, err := layerStore.GetRWLayer(id) - if err != nil { - return nil, err - } - defer func() { - if err != nil { - layerStore.ReleaseRWLayer(rwlayer) - } - }() - - // TODO: this mount call is not necessary as we assume that TarStream() should - // mount the layer if needed. But the Diff() function for windows requests that - // the layer should be mounted when calling it. So we reserve this mount call - // until windows driver can implement Diff() interface correctly. - _, err = rwlayer.Mount(mountLabel) - if err != nil { - return nil, err - } - - archive, err := rwlayer.TarStream() - if err != nil { - rwlayer.Unmount() - return nil, err - } - return ioutils.NewReadCloserWrapper(archive, func() error { - archive.Close() - err = rwlayer.Unmount() - layerStore.ReleaseRWLayer(rwlayer) - return err - }), - nil -} - -// CommitBuildStep is used by the builder to create an image for each step in -// the build. -// -// This method is different from CreateImageFromContainer: -// * it doesn't attempt to validate container state -// * it doesn't send a commit action to metrics -// * it doesn't log a container commit event -// -// This is a temporary shim. Should be removed when builder stops using commit. -func (daemon *Daemon) CommitBuildStep(c backend.CommitConfig) (image.ID, error) { - container, err := daemon.GetContainer(c.ContainerID) - if err != nil { - return "", err - } - c.ContainerMountLabel = container.MountLabel - c.ContainerOS = container.OS - c.ParentImageID = string(container.ImageID) - return daemon.commitImage(c) -} diff --git a/daemon/events.go b/daemon/events.go index 2e594c0269..cf1634a198 100644 --- a/daemon/events.go +++ b/daemon/events.go @@ -44,30 +44,6 @@ func (daemon *Daemon) LogContainerEventWithAttributes(container *container.Conta daemon.EventsService.Log(action, events.ContainerEventType, actor) } -// LogImageEvent generates an event related to an image with only the default attributes. -func (daemon *Daemon) LogImageEvent(imageID, refName, action string) { - daemon.LogImageEventWithAttributes(imageID, refName, action, map[string]string{}) -} - -// LogImageEventWithAttributes generates an event related to an image with specific given attributes. -func (daemon *Daemon) LogImageEventWithAttributes(imageID, refName, action string, attributes map[string]string) { - img, err := daemon.GetImage(imageID) - if err == nil && img.Config != nil { - // image has not been removed yet. - // it could be missing if the event is `delete`. - copyAttributes(attributes, img.Config.Labels) - } - if refName != "" { - attributes["name"] = refName - } - actor := events.Actor{ - ID: imageID, - Attributes: attributes, - } - - daemon.EventsService.Log(action, events.ImageEventType, actor) -} - // LogPluginEvent generates an event related to a plugin with only the default attributes. func (daemon *Daemon) LogPluginEvent(pluginID, refName, action string) { daemon.LogPluginEventWithAttributes(pluginID, refName, action, map[string]string{}) diff --git a/daemon/image_commit.go b/daemon/image_commit.go new file mode 100644 index 0000000000..48f890b80d --- /dev/null +++ b/daemon/image_commit.go @@ -0,0 +1,124 @@ +package daemon // import "github.com/docker/docker/daemon" + +import ( + "encoding/json" + "io" + + "github.com/docker/docker/api/types/backend" + "github.com/docker/docker/image" + "github.com/docker/docker/layer" + "github.com/docker/docker/pkg/ioutils" + "github.com/docker/docker/pkg/system" +) + +func (daemon *Daemon) commitImage(c backend.CommitConfig) (image.ID, error) { + layerStore, ok := daemon.layerStores[c.ContainerOS] + if !ok { + return "", system.ErrNotSupportedOperatingSystem + } + rwTar, err := exportContainerRw(layerStore, c.ContainerID, c.ContainerMountLabel) + if err != nil { + return "", err + } + defer func() { + if rwTar != nil { + rwTar.Close() + } + }() + + var parent *image.Image + if c.ParentImageID == "" { + parent = new(image.Image) + parent.RootFS = image.NewRootFS() + } else { + parent, err = daemon.imageStore.Get(image.ID(c.ParentImageID)) + if err != nil { + return "", err + } + } + + l, err := layerStore.Register(rwTar, parent.RootFS.ChainID()) + if err != nil { + return "", err + } + defer layer.ReleaseAndLog(layerStore, l) + + cc := image.ChildConfig{ + ContainerID: c.ContainerID, + Author: c.Author, + Comment: c.Comment, + ContainerConfig: c.ContainerConfig, + Config: c.Config, + DiffID: l.DiffID(), + } + config, err := json.Marshal(image.NewChildImage(parent, cc, c.ContainerOS)) + if err != nil { + return "", err + } + + id, err := daemon.imageStore.Create(config) + if err != nil { + return "", err + } + + if c.ParentImageID != "" { + if err := daemon.imageStore.SetParent(id, image.ID(c.ParentImageID)); err != nil { + return "", err + } + } + return id, nil +} + +func exportContainerRw(layerStore layer.Store, id, mountLabel string) (arch io.ReadCloser, err error) { + rwlayer, err := layerStore.GetRWLayer(id) + if err != nil { + return nil, err + } + defer func() { + if err != nil { + layerStore.ReleaseRWLayer(rwlayer) + } + }() + + // TODO: this mount call is not necessary as we assume that TarStream() should + // mount the layer if needed. But the Diff() function for windows requests that + // the layer should be mounted when calling it. So we reserve this mount call + // until windows driver can implement Diff() interface correctly. + _, err = rwlayer.Mount(mountLabel) + if err != nil { + return nil, err + } + + archive, err := rwlayer.TarStream() + if err != nil { + rwlayer.Unmount() + return nil, err + } + return ioutils.NewReadCloserWrapper(archive, func() error { + archive.Close() + err = rwlayer.Unmount() + layerStore.ReleaseRWLayer(rwlayer) + return err + }), + nil +} + +// CommitBuildStep is used by the builder to create an image for each step in +// the build. +// +// This method is different from CreateImageFromContainer: +// * it doesn't attempt to validate container state +// * it doesn't send a commit action to metrics +// * it doesn't log a container commit event +// +// This is a temporary shim. Should be removed when builder stops using commit. +func (daemon *Daemon) CommitBuildStep(c backend.CommitConfig) (image.ID, error) { + container, err := daemon.GetContainer(c.ContainerID) + if err != nil { + return "", err + } + c.ContainerMountLabel = container.MountLabel + c.ContainerOS = container.OS + c.ParentImageID = string(container.ImageID) + return daemon.commitImage(c) +} diff --git a/daemon/image_events.go b/daemon/image_events.go new file mode 100644 index 0000000000..0d364f7b58 --- /dev/null +++ b/daemon/image_events.go @@ -0,0 +1,29 @@ +package daemon // import "github.com/docker/docker/daemon" + +import ( + "github.com/docker/docker/api/types/events" +) + +// LogImageEvent generates an event related to an image with only the default attributes. +func (daemon *Daemon) LogImageEvent(imageID, refName, action string) { + daemon.LogImageEventWithAttributes(imageID, refName, action, map[string]string{}) +} + +// LogImageEventWithAttributes generates an event related to an image with specific given attributes. +func (daemon *Daemon) LogImageEventWithAttributes(imageID, refName, action string, attributes map[string]string) { + img, err := daemon.GetImage(imageID) + if err == nil && img.Config != nil { + // image has not been removed yet. + // it could be missing if the event is `delete`. + copyAttributes(attributes, img.Config.Labels) + } + if refName != "" { + attributes["name"] = refName + } + actor := events.Actor{ + ID: imageID, + Attributes: attributes, + } + + daemon.EventsService.Log(action, events.ImageEventType, actor) +} diff --git a/daemon/image_prune.go b/daemon/image_prune.go new file mode 100644 index 0000000000..193c3aa0ac --- /dev/null +++ b/daemon/image_prune.go @@ -0,0 +1,168 @@ +package daemon // import "github.com/docker/docker/daemon" + +import ( + "sync/atomic" + + "github.com/docker/distribution/reference" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/errdefs" + "github.com/docker/docker/image" + "github.com/docker/docker/layer" + digest "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" + "golang.org/x/net/context" +) + +var imagesAcceptedFilters = map[string]bool{ + "dangling": true, + "label": true, + "label!": true, + "until": true, +} + +// ImagesPrune removes unused images +func (daemon *Daemon) ImagesPrune(ctx context.Context, pruneFilters filters.Args) (*types.ImagesPruneReport, error) { + if !atomic.CompareAndSwapInt32(&daemon.pruneRunning, 0, 1) { + return nil, errPruneRunning + } + defer atomic.StoreInt32(&daemon.pruneRunning, 0) + + // make sure that only accepted filters have been received + err := pruneFilters.Validate(imagesAcceptedFilters) + if err != nil { + return nil, err + } + + rep := &types.ImagesPruneReport{} + + danglingOnly := true + if pruneFilters.Contains("dangling") { + if pruneFilters.ExactMatch("dangling", "false") || pruneFilters.ExactMatch("dangling", "0") { + danglingOnly = false + } else if !pruneFilters.ExactMatch("dangling", "true") && !pruneFilters.ExactMatch("dangling", "1") { + return nil, invalidFilter{"dangling", pruneFilters.Get("dangling")} + } + } + + until, err := getUntilFromPruneFilters(pruneFilters) + if err != nil { + return nil, err + } + + var allImages map[image.ID]*image.Image + if danglingOnly { + allImages = daemon.imageStore.Heads() + } else { + allImages = daemon.imageStore.Map() + } + + // Filter intermediary images and get their unique size + allLayers := make(map[layer.ChainID]layer.Layer) + for _, ls := range daemon.layerStores { + for k, v := range ls.Map() { + allLayers[k] = v + } + } + topImages := map[image.ID]*image.Image{} + for id, img := range allImages { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + dgst := digest.Digest(id) + if len(daemon.referenceStore.References(dgst)) == 0 && len(daemon.imageStore.Children(id)) != 0 { + continue + } + if !until.IsZero() && img.Created.After(until) { + continue + } + if img.Config != nil && !matchLabels(pruneFilters, img.Config.Labels) { + continue + } + topImages[id] = img + } + } + + canceled := false +deleteImagesLoop: + for id := range topImages { + select { + case <-ctx.Done(): + // we still want to calculate freed size and return the data + canceled = true + break deleteImagesLoop + default: + } + + deletedImages := []types.ImageDeleteResponseItem{} + refs := daemon.referenceStore.References(id.Digest()) + if len(refs) > 0 { + shouldDelete := !danglingOnly + if !shouldDelete { + hasTag := false + for _, ref := range refs { + if _, ok := ref.(reference.NamedTagged); ok { + hasTag = true + break + } + } + + // Only delete if it's untagged (i.e. repo:) + shouldDelete = !hasTag + } + + if shouldDelete { + for _, ref := range refs { + imgDel, err := daemon.ImageDelete(ref.String(), false, true) + if imageDeleteFailed(ref.String(), err) { + continue + } + deletedImages = append(deletedImages, imgDel...) + } + } + } else { + hex := id.Digest().Hex() + imgDel, err := daemon.ImageDelete(hex, false, true) + if imageDeleteFailed(hex, err) { + continue + } + deletedImages = append(deletedImages, imgDel...) + } + + rep.ImagesDeleted = append(rep.ImagesDeleted, deletedImages...) + } + + // Compute how much space was freed + for _, d := range rep.ImagesDeleted { + if d.Deleted != "" { + chid := layer.ChainID(d.Deleted) + if l, ok := allLayers[chid]; ok { + diffSize, err := l.DiffSize() + if err != nil { + logrus.Warnf("failed to get layer %s size: %v", chid, err) + continue + } + rep.SpaceReclaimed += uint64(diffSize) + } + } + } + + if canceled { + logrus.Debugf("ImagesPrune operation cancelled: %#v", *rep) + } + + return rep, nil +} + +func imageDeleteFailed(ref string, err error) bool { + switch { + case err == nil: + return false + case errdefs.IsConflict(err): + return true + default: + logrus.Warnf("failed to prune image %s: %v", ref, err) + return true + } +} diff --git a/daemon/image_windows.go b/daemon/image_windows.go new file mode 100644 index 0000000000..d661400a38 --- /dev/null +++ b/daemon/image_windows.go @@ -0,0 +1,35 @@ +package daemon // import "github.com/docker/docker/daemon" + +import ( + "github.com/docker/docker/image" + "github.com/docker/docker/layer" + "github.com/docker/docker/pkg/system" + "github.com/pkg/errors" +) + +// GetLayerFolders returns the layer folders from an image RootFS +func (daemon *Daemon) GetLayerFolders(img *image.Image, rwLayer layer.RWLayer) ([]string, error) { + folders := []string{} + max := len(img.RootFS.DiffIDs) + for index := 1; index <= max; index++ { + // FIXME: why does this mutate the RootFS? + img.RootFS.DiffIDs = img.RootFS.DiffIDs[:index] + if !system.IsOSSupported(img.OperatingSystem()) { + return nil, errors.Wrapf(system.ErrNotSupportedOperatingSystem, "cannot get layerpath for ImageID %s", img.RootFS.ChainID()) + } + layerPath, err := layer.GetLayerPath(daemon.layerStores[img.OperatingSystem()], img.RootFS.ChainID()) + if err != nil { + return nil, errors.Wrapf(err, "failed to get layer path from graphdriver %s for ImageID %s", daemon.layerStores[img.OperatingSystem()], img.RootFS.ChainID()) + } + // Reverse order, expecting parent first + folders = append([]string{layerPath}, folders...) + } + if rwLayer == nil { + return nil, errors.New("RWLayer is unexpectedly nil") + } + m, err := rwLayer.Metadata() + if err != nil { + return nil, errors.Wrap(err, "failed to get layer metadata") + } + return append(folders, m["dir"]), nil +} diff --git a/daemon/oci_windows.go b/daemon/oci_windows.go index 64c651c4af..e4b536f4e4 100644 --- a/daemon/oci_windows.go +++ b/daemon/oci_windows.go @@ -1,7 +1,6 @@ package daemon // import "github.com/docker/docker/daemon" import ( - "errors" "fmt" "io/ioutil" "path/filepath" @@ -10,11 +9,11 @@ import ( containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/container" - "github.com/docker/docker/layer" "github.com/docker/docker/oci" "github.com/docker/docker/pkg/sysinfo" "github.com/docker/docker/pkg/system" "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" "golang.org/x/sys/windows" "golang.org/x/sys/windows/registry" ) @@ -139,29 +138,10 @@ func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) { } } s.Process.User.Username = c.Config.User - - // Get the layer path for each layer. - max := len(img.RootFS.DiffIDs) - for i := 1; i <= max; i++ { - img.RootFS.DiffIDs = img.RootFS.DiffIDs[:i] - if !system.IsOSSupported(img.OperatingSystem()) { - return nil, fmt.Errorf("cannot get layerpath for ImageID %s: %s ", img.RootFS.ChainID(), system.ErrNotSupportedOperatingSystem) - } - layerPath, err := layer.GetLayerPath(daemon.layerStores[img.OperatingSystem()], img.RootFS.ChainID()) - if err != nil { - return nil, fmt.Errorf("failed to get layer path from graphdriver %s for ImageID %s - %s", daemon.layerStores[img.OperatingSystem()], img.RootFS.ChainID(), err) - } - // Reverse order, expecting parent most first - s.Windows.LayerFolders = append([]string{layerPath}, s.Windows.LayerFolders...) - } - if c.RWLayer == nil { - return nil, errors.New("RWLayer of container " + c.ID + " is unexpectedly nil") - } - m, err := c.RWLayer.Metadata() + s.Windows.LayerFolders, err = daemon.GetLayerFolders(img, c.RWLayer) if err != nil { - return nil, fmt.Errorf("failed to get layer metadata - %s", err) + return nil, errors.Wrapf(err, "container %s", c.ID) } - s.Windows.LayerFolders = append(s.Windows.LayerFolders, m["dir"]) dnsSearch := daemon.getDNSSearchSettings(c) diff --git a/daemon/prune.go b/daemon/prune.go index eee9e106bd..f16e448ba8 100644 --- a/daemon/prune.go +++ b/daemon/prune.go @@ -6,18 +6,13 @@ import ( "sync/atomic" "time" - "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" timetypes "github.com/docker/docker/api/types/time" - "github.com/docker/docker/errdefs" - "github.com/docker/docker/image" - "github.com/docker/docker/layer" "github.com/docker/docker/pkg/directory" "github.com/docker/docker/runconfig" "github.com/docker/docker/volume" "github.com/docker/libnetwork" - digest "github.com/opencontainers/go-digest" "github.com/sirupsen/logrus" "golang.org/x/net/context" ) @@ -36,12 +31,7 @@ var ( "label": true, "label!": true, } - imagesAcceptedFilters = map[string]bool{ - "dangling": true, - "label": true, - "label!": true, - "until": true, - } + networksAcceptedFilters = map[string]bool{ "label": true, "label!": true, @@ -159,152 +149,6 @@ func (daemon *Daemon) VolumesPrune(ctx context.Context, pruneFilters filters.Arg return rep, err } -// ImagesPrune removes unused images -func (daemon *Daemon) ImagesPrune(ctx context.Context, pruneFilters filters.Args) (*types.ImagesPruneReport, error) { - if !atomic.CompareAndSwapInt32(&daemon.pruneRunning, 0, 1) { - return nil, errPruneRunning - } - defer atomic.StoreInt32(&daemon.pruneRunning, 0) - - // make sure that only accepted filters have been received - err := pruneFilters.Validate(imagesAcceptedFilters) - if err != nil { - return nil, err - } - - rep := &types.ImagesPruneReport{} - - danglingOnly := true - if pruneFilters.Contains("dangling") { - if pruneFilters.ExactMatch("dangling", "false") || pruneFilters.ExactMatch("dangling", "0") { - danglingOnly = false - } else if !pruneFilters.ExactMatch("dangling", "true") && !pruneFilters.ExactMatch("dangling", "1") { - return nil, invalidFilter{"dangling", pruneFilters.Get("dangling")} - } - } - - until, err := getUntilFromPruneFilters(pruneFilters) - if err != nil { - return nil, err - } - - var allImages map[image.ID]*image.Image - if danglingOnly { - allImages = daemon.imageStore.Heads() - } else { - allImages = daemon.imageStore.Map() - } - - // Filter intermediary images and get their unique size - allLayers := make(map[layer.ChainID]layer.Layer) - for _, ls := range daemon.layerStores { - for k, v := range ls.Map() { - allLayers[k] = v - } - } - topImages := map[image.ID]*image.Image{} - for id, img := range allImages { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - dgst := digest.Digest(id) - if len(daemon.referenceStore.References(dgst)) == 0 && len(daemon.imageStore.Children(id)) != 0 { - continue - } - if !until.IsZero() && img.Created.After(until) { - continue - } - if img.Config != nil && !matchLabels(pruneFilters, img.Config.Labels) { - continue - } - topImages[id] = img - } - } - - canceled := false -deleteImagesLoop: - for id := range topImages { - select { - case <-ctx.Done(): - // we still want to calculate freed size and return the data - canceled = true - break deleteImagesLoop - default: - } - - deletedImages := []types.ImageDeleteResponseItem{} - refs := daemon.referenceStore.References(id.Digest()) - if len(refs) > 0 { - shouldDelete := !danglingOnly - if !shouldDelete { - hasTag := false - for _, ref := range refs { - if _, ok := ref.(reference.NamedTagged); ok { - hasTag = true - break - } - } - - // Only delete if it's untagged (i.e. repo:) - shouldDelete = !hasTag - } - - if shouldDelete { - for _, ref := range refs { - imgDel, err := daemon.ImageDelete(ref.String(), false, true) - if imageDeleteFailed(ref.String(), err) { - continue - } - deletedImages = append(deletedImages, imgDel...) - } - } - } else { - hex := id.Digest().Hex() - imgDel, err := daemon.ImageDelete(hex, false, true) - if imageDeleteFailed(hex, err) { - continue - } - deletedImages = append(deletedImages, imgDel...) - } - - rep.ImagesDeleted = append(rep.ImagesDeleted, deletedImages...) - } - - // Compute how much space was freed - for _, d := range rep.ImagesDeleted { - if d.Deleted != "" { - chid := layer.ChainID(d.Deleted) - if l, ok := allLayers[chid]; ok { - diffSize, err := l.DiffSize() - if err != nil { - logrus.Warnf("failed to get layer %s size: %v", chid, err) - continue - } - rep.SpaceReclaimed += uint64(diffSize) - } - } - } - - if canceled { - logrus.Debugf("ImagesPrune operation cancelled: %#v", *rep) - } - - return rep, nil -} - -func imageDeleteFailed(ref string, err error) bool { - switch { - case err == nil: - return false - case errdefs.IsConflict(err): - return true - default: - logrus.Warnf("failed to prune image %s: %v", ref, err) - return true - } -} - // localNetworksPrune removes unused local networks func (daemon *Daemon) localNetworksPrune(ctx context.Context, pruneFilters filters.Args) *types.NetworksPruneReport { rep := &types.NetworksPruneReport{}