diff --git a/daemon/cluster/executor/backend.go b/daemon/cluster/executor/backend.go index 81ba79e91e..3a5ff479bd 100644 --- a/daemon/cluster/executor/backend.go +++ b/daemon/cluster/executor/backend.go @@ -62,6 +62,7 @@ type Backend interface { GetAttachmentStore() *networkSettings.AttachmentStore } +// ImageBackend is used by an executor to perform image operations type ImageBackend interface { PullImage(ctx context.Context, image, tag, platform string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error GetRepository(context.Context, reference.Named, *types.AuthConfig) (distribution.Repository, bool, error) diff --git a/daemon/container_operations_windows.go b/daemon/container_operations_windows.go index 0559b8ac3e..a4541b0634 100644 --- a/daemon/container_operations_windows.go +++ b/daemon/container_operations_windows.go @@ -64,12 +64,6 @@ func (daemon *Daemon) setupConfigDir(c *container.Container) (setupErr error) { return nil } -// getSize returns real size & virtual size -func (daemon *Daemon) getSize(containerID string) (int64, int64) { - // TODO Windows - return 0, 0 -} - func (daemon *Daemon) setupIpcDirs(container *container.Container) error { return nil } diff --git a/daemon/create.go b/daemon/create.go index 386d346f73..e57452b188 100644 --- a/daemon/create.go +++ b/daemon/create.go @@ -157,7 +157,7 @@ func (daemon *Daemon) create(params types.ContainerCreateConfig, managed bool) ( } // Set RWLayer for container after mount labels have been set - rwLayer, err := daemon.imageService.GetRWLayer(container, setupInitLayer(daemon.idMappings)) + rwLayer, err := daemon.imageService.CreateLayer(container, setupInitLayer(daemon.idMappings)) if err != nil { return nil, errdefs.System(err) } diff --git a/daemon/daemon.go b/daemon/daemon.go index 4a41e504ca..8c36f042ea 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -21,21 +21,21 @@ import ( "github.com/docker/docker/api/types" containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/swarm" + "github.com/docker/docker/builder" "github.com/docker/docker/container" "github.com/docker/docker/daemon/config" "github.com/docker/docker/daemon/discovery" "github.com/docker/docker/daemon/events" "github.com/docker/docker/daemon/exec" + "github.com/docker/docker/daemon/images" "github.com/docker/docker/daemon/logger" "github.com/docker/docker/daemon/network" "github.com/docker/docker/errdefs" "github.com/sirupsen/logrus" // register graph drivers - "github.com/docker/docker/builder" _ "github.com/docker/docker/daemon/graphdriver/register" "github.com/docker/docker/daemon/stats" dmetadata "github.com/docker/docker/distribution/metadata" - "github.com/docker/docker/distribution/xfer" "github.com/docker/docker/dockerversion" "github.com/docker/docker/image" "github.com/docker/docker/layer" @@ -75,23 +75,21 @@ type Daemon struct { containers container.Store containersReplica container.ViewDB execCommands *exec.Store - - imageService *imageService - - idIndex *truncindex.TruncIndex - configStore *config.Config - statsCollector *stats.Collector - defaultLogConfig containertypes.LogConfig - RegistryService registry.Service - EventsService *events.Events - netController libnetwork.NetworkController - volumes *store.VolumeStore - discoveryWatcher discovery.Reloader - root string - seccompEnabled bool - apparmorEnabled bool - shutdown bool - idMappings *idtools.IDMappings + imageService *images.ImageService + idIndex *truncindex.TruncIndex + configStore *config.Config + statsCollector *stats.Collector + defaultLogConfig containertypes.LogConfig + RegistryService registry.Service + EventsService *events.Events + netController libnetwork.NetworkController + volumes *store.VolumeStore + discoveryWatcher discovery.Reloader + root string + seccompEnabled bool + apparmorEnabled bool + shutdown bool + idMappings *idtools.IDMappings // TODO: move graphDrivers field to an InfoService graphDrivers map[string]string // By operating system @@ -158,7 +156,7 @@ func (daemon *Daemon) restore() error { // Ignore the container if it does not support the current driver being used by the graph currentDriverForContainerOS := daemon.graphDrivers[container.OS] if (container.Driver == "" && currentDriverForContainerOS == "aufs") || container.Driver == currentDriverForContainerOS { - rwlayer, err := daemon.imageService.GetRWLayerByID(container.ID, container.OS) + rwlayer, err := daemon.imageService.GetLayerByID(container.ID, container.OS) if err != nil { logrus.Errorf("Failed to load container mount %v: %v", id, err) continue @@ -808,8 +806,6 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe return nil, err } - eventsService := events.New() - // We have a single tag/reference store for the daemon globally. However, it's // stored under the graphdriver. On host platforms which only support a single // container OS, but multiple selectable graphdrivers, this means depending on which @@ -863,7 +859,7 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe d.idIndex = truncindex.NewTruncIndex([]string{}) d.statsCollector = d.newStatsCollector(1 * time.Second) - d.EventsService = eventsService + d.EventsService = events.New() d.volumes = volStore d.root = config.Root d.idMappings = idMappings @@ -872,19 +868,21 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe d.linkIndex = newLinkIndex() - logrus.Debugf("Max Concurrent Downloads: %d", *config.MaxConcurrentDownloads) - logrus.Debugf("Max Concurrent Uploads: %d", *config.MaxConcurrentUploads) - d.imageService = &imageService{ - trustKey: trustKey, - uploadManager: xfer.NewLayerUploadManager(*config.MaxConcurrentUploads), - downloadManager: xfer.NewLayerDownloadManager(layerStores, *config.MaxConcurrentDownloads), - registryService: registryService, - referenceStore: rs, - distributionMetadataStore: distributionMetadataStore, - imageStore: imageStore, - eventsService: eventsService, - containers: d.containers, - } + // TODO: imageStore, distributionMetadataStore, and ReferenceStore are only + // used above to run migration. They could be initialized in ImageService + // if migration is called from daemon/images. layerStore might move as well. + d.imageService = images.NewImageService(images.ImageServiceConfig{ + ContainerStore: d.containers, + DistributionMetadataStore: distributionMetadataStore, + EventsService: d.EventsService, + ImageStore: imageStore, + LayerStores: layerStores, + MaxConcurrentDownloads: *config.MaxConcurrentDownloads, + MaxConcurrentUploads: *config.MaxConcurrentUploads, + ReferenceStore: rs, + RegistryService: registryService, + TrustKey: trustKey, + }) go d.execCommandGC() @@ -1007,7 +1005,7 @@ func (daemon *Daemon) Shutdown() error { logrus.Errorf("Stop container error: %v", err) return } - if mountid, err := daemon.imageService.GetContainerMountID(c.ID, c.OS); err == nil { + if mountid, err := daemon.imageService.GetLayerMountID(c.ID, c.OS); err == nil { daemon.cleanupMountsByID(mountid) } logrus.Debugf("container stopped %s", c.ID) @@ -1020,7 +1018,9 @@ func (daemon *Daemon) Shutdown() error { } } - daemon.imageService.Cleanup() + if daemon.imageService != nil { + daemon.imageService.Cleanup() + } // If we are part of a cluster, clean up cluster's stuff if daemon.clusterProvider != nil { @@ -1320,14 +1320,15 @@ func (daemon *Daemon) IDMappings() *idtools.IDMappings { return daemon.idMappings } -func (daemon *Daemon) ImageService() *imageService { +// ImageService returns the Daemon's ImageService +func (daemon *Daemon) ImageService() *images.ImageService { return daemon.imageService } -// TODO: tmp hack to merge interfaces +// BuilderBackend returns the backend used by builder func (daemon *Daemon) BuilderBackend() builder.Backend { return struct { *Daemon - *imageService + *images.ImageService }{daemon, daemon.imageService} } diff --git a/daemon/delete.go b/daemon/delete.go index 749afeb812..1eff18968f 100644 --- a/daemon/delete.go +++ b/daemon/delete.go @@ -120,7 +120,7 @@ func (daemon *Daemon) cleanupContainer(container *container.Container, forceRemo // When container creation fails and `RWLayer` has not been created yet, we // do not call `ReleaseRWLayer` if container.RWLayer != nil { - err := daemon.imageService.ReleaseContainerLayer(container.RWLayer, container.OS) + err := daemon.imageService.ReleaseLayer(container.RWLayer, container.OS) if err != nil { err = errors.Wrapf(err, "container %s", container.ID) container.SetRemovalError(err) diff --git a/daemon/export.go b/daemon/export.go index 27181c9176..52c23a3c28 100644 --- a/daemon/export.go +++ b/daemon/export.go @@ -51,13 +51,13 @@ func (daemon *Daemon) containerExport(container *container.Container) (arch io.R if !system.IsOSSupported(container.OS) { return nil, fmt.Errorf("cannot export %s: %s ", container.ID, system.ErrNotSupportedOperatingSystem) } - rwlayer, err := daemon.imageService.GetRWLayerByID(container.ID, container.OS) + rwlayer, err := daemon.imageService.GetLayerByID(container.ID, container.OS) if err != nil { return nil, err } defer func() { if err != nil { - daemon.imageService.ReleaseContainerLayer(rwlayer, container.OS) + daemon.imageService.ReleaseLayer(rwlayer, container.OS) } }() @@ -78,7 +78,7 @@ func (daemon *Daemon) containerExport(container *container.Container) (arch io.R arch = ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() rwlayer.Unmount() - daemon.imageService.ReleaseContainerLayer(rwlayer, container.OS) + daemon.imageService.ReleaseLayer(rwlayer, container.OS) return err }) daemon.LogContainerEvent(container, "export") diff --git a/daemon/image.go b/daemon/image.go deleted file mode 100644 index 366dff44fc..0000000000 --- a/daemon/image.go +++ /dev/null @@ -1,266 +0,0 @@ -package daemon // import "github.com/docker/docker/daemon" - -import ( - "context" - "fmt" - "os" - - "github.com/docker/distribution/reference" - "github.com/docker/docker/api/types/events" - "github.com/docker/docker/container" - daemonevents "github.com/docker/docker/daemon/events" - "github.com/docker/docker/distribution/metadata" - "github.com/docker/docker/distribution/xfer" - "github.com/docker/docker/errdefs" - "github.com/docker/docker/image" - "github.com/docker/docker/layer" - dockerreference "github.com/docker/docker/reference" - "github.com/docker/docker/registry" - "github.com/docker/libtrust" - "github.com/opencontainers/go-digest" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" -) - -// errImageDoesNotExist is error returned when no image can be found for a reference. -type errImageDoesNotExist struct { - ref reference.Reference -} - -func (e errImageDoesNotExist) Error() string { - ref := e.ref - if named, ok := ref.(reference.Named); ok { - ref = reference.TagNameOnly(named) - } - return fmt.Sprintf("No such image: %s", reference.FamiliarString(ref)) -} - -func (e errImageDoesNotExist) NotFound() {} - -// GetImageIDAndOS returns an image ID and operating system corresponding to the image referred to by -// refOrID. -// called from list.go foldFilter() -func (i imageService) GetImageIDAndOS(refOrID string) (image.ID, string, error) { - ref, err := reference.ParseAnyReference(refOrID) - if err != nil { - return "", "", errdefs.InvalidParameter(err) - } - namedRef, ok := ref.(reference.Named) - if !ok { - digested, ok := ref.(reference.Digested) - if !ok { - return "", "", errImageDoesNotExist{ref} - } - id := image.IDFromDigest(digested.Digest()) - if img, err := i.imageStore.Get(id); err == nil { - return id, img.OperatingSystem(), nil - } - return "", "", errImageDoesNotExist{ref} - } - - if digest, err := i.referenceStore.Get(namedRef); err == nil { - // Search the image stores to get the operating system, defaulting to host OS. - id := image.IDFromDigest(digest) - if img, err := i.imageStore.Get(id); err == nil { - return id, img.OperatingSystem(), nil - } - } - - // Search based on ID - if id, err := i.imageStore.Search(refOrID); err == nil { - img, err := i.imageStore.Get(id) - if err != nil { - return "", "", errImageDoesNotExist{ref} - } - return id, img.OperatingSystem(), nil - } - - return "", "", errImageDoesNotExist{ref} -} - -// GetImage returns an image corresponding to the image referred to by refOrID. -func (i *imageService) GetImage(refOrID string) (*image.Image, error) { - imgID, _, err := i.GetImageIDAndOS(refOrID) - if err != nil { - return nil, err - } - return i.imageStore.Get(imgID) -} - -type containerStore interface { - // used by image delete - First(container.StoreFilter) *container.Container - // used by image prune, and image list - List() []*container.Container - // TODO: remove, only used for CommitBuildStep - Get(string) *container.Container -} - -type imageService struct { - eventsService *daemonevents.Events - containers containerStore - downloadManager *xfer.LayerDownloadManager - uploadManager *xfer.LayerUploadManager - - // TODO: should accept a trust service instead of a key - trustKey libtrust.PrivateKey - - registryService registry.Service - referenceStore dockerreference.Store - distributionMetadataStore metadata.Store - imageStore image.Store - layerStores map[string]layer.Store // By operating system - - pruneRunning int32 -} - -// called from info.go -func (i *imageService) CountImages() int { - return len(i.imageStore.Map()) -} - -// called from list.go to filter containers -func (i *imageService) Children(id image.ID) []image.ID { - return i.imageStore.Children(id) -} - -// TODO: accept an opt struct instead of container? -// called from create.go -func (i *imageService) GetRWLayer(container *container.Container, initFunc layer.MountInit) (layer.RWLayer, error) { - var layerID layer.ChainID - if container.ImageID != "" { - img, err := i.imageStore.Get(container.ImageID) - if err != nil { - return nil, err - } - layerID = img.RootFS.ChainID() - } - - rwLayerOpts := &layer.CreateRWLayerOpts{ - MountLabel: container.MountLabel, - InitFunc: initFunc, - StorageOpt: container.HostConfig.StorageOpt, - } - - // Indexing by OS is safe here as validation of OS has already been performed in create() (the only - // caller), and guaranteed non-nil - return i.layerStores[container.OS].CreateRWLayer(container.ID, layerID, rwLayerOpts) -} - -// called from daemon.go Daemon.restore(), and Daemon.containerExport() -func (i *imageService) GetRWLayerByID(cid string, os string) (layer.RWLayer, error) { - return i.layerStores[os].GetRWLayer(cid) -} - -// called from info.go -func (i *imageService) GraphDriverStatuses() map[string][][2]string { - result := make(map[string][][2]string) - for os, store := range i.layerStores { - result[os] = store.DriverStatus() - } - return result -} - -// called from daemon.go Daemon.Shutdown(), and Daemon.Cleanup() (cleanup is actually continerCleanup) -func (i *imageService) GetContainerMountID(cid string, os string) (string, error) { - return i.layerStores[os].GetMountID(cid) -} - -// called from daemon.go Daemon.Shutdown() -func (i *imageService) Cleanup() { - for os, ls := range i.layerStores { - if ls != nil { - if err := ls.Cleanup(); err != nil { - logrus.Errorf("Error during layer Store.Cleanup(): %v %s", err, os) - } - } - } -} - -// moved from Daemon.GraphDriverName, multiple calls -func (i *imageService) GraphDriverForOS(os string) string { - return i.layerStores[os].DriverName() -} - -// called from delete.go Daemon.cleanupContainer(), and Daemon.containerExport() -func (i *imageService) ReleaseContainerLayer(rwlayer layer.RWLayer, containerOS string) error { - metadata, err := i.layerStores[containerOS].ReleaseRWLayer(rwlayer) - layer.LogReleaseMetadata(metadata) - if err != nil && err != layer.ErrMountDoesNotExist && !os.IsNotExist(errors.Cause(err)) { - return errors.Wrapf(err, "driver %q failed to remove root filesystem", - i.layerStores[containerOS].DriverName()) - } - return nil -} - -// called from disk_usage.go -func (i *imageService) LayerDiskUsage(ctx context.Context) (int64, error) { - var allLayersSize int64 - layerRefs := i.getLayerRefs() - for _, ls := range i.layerStores { - allLayers := ls.Map() - for _, l := range allLayers { - select { - case <-ctx.Done(): - return allLayersSize, ctx.Err() - default: - size, err := l.DiffSize() - if err == nil { - if _, ok := layerRefs[l.ChainID()]; ok { - allLayersSize += size - } else { - logrus.Warnf("found leaked image layer %v", l.ChainID()) - } - } else { - logrus.Warnf("failed to get diff size for layer %v", l.ChainID()) - } - } - } - } - return allLayersSize, nil -} - -func (i *imageService) getLayerRefs() map[layer.ChainID]int { - tmpImages := i.imageStore.Map() - layerRefs := map[layer.ChainID]int{} - for id, img := range tmpImages { - dgst := digest.Digest(id) - if len(i.referenceStore.References(dgst)) == 0 && len(i.imageStore.Children(id)) != 0 { - continue - } - - rootFS := *img.RootFS - rootFS.DiffIDs = nil - for _, id := range img.RootFS.DiffIDs { - rootFS.Append(id) - chid := rootFS.ChainID() - layerRefs[chid]++ - } - } - - return layerRefs -} - -// LogImageEvent generates an event related to an image with only the default attributes. -func (i *imageService) LogImageEvent(imageID, refName, action string) { - i.LogImageEventWithAttributes(imageID, refName, action, map[string]string{}) -} - -// LogImageEventWithAttributes generates an event related to an image with specific given attributes. -func (i *imageService) LogImageEventWithAttributes(imageID, refName, action string, attributes map[string]string) { - img, err := i.GetImage(imageID) - if err == nil && img.Config != nil { - // image has not been removed yet. - // it could be missing if the event is `delete`. - copyAttributes(attributes, img.Config.Labels) - } - if refName != "" { - attributes["name"] = refName - } - actor := events.Actor{ - ID: imageID, - Attributes: attributes, - } - - i.eventsService.Log(action, events.ImageEventType, actor) -} diff --git a/daemon/image_events.go b/daemon/image_events.go deleted file mode 100644 index 0d364f7b58..0000000000 --- a/daemon/image_events.go +++ /dev/null @@ -1,29 +0,0 @@ -package daemon // import "github.com/docker/docker/daemon" - -import ( - "github.com/docker/docker/api/types/events" -) - -// LogImageEvent generates an event related to an image with only the default attributes. -func (daemon *Daemon) LogImageEvent(imageID, refName, action string) { - daemon.LogImageEventWithAttributes(imageID, refName, action, map[string]string{}) -} - -// LogImageEventWithAttributes generates an event related to an image with specific given attributes. -func (daemon *Daemon) LogImageEventWithAttributes(imageID, refName, action string, attributes map[string]string) { - img, err := daemon.GetImage(imageID) - if err == nil && img.Config != nil { - // image has not been removed yet. - // it could be missing if the event is `delete`. - copyAttributes(attributes, img.Config.Labels) - } - if refName != "" { - attributes["name"] = refName - } - actor := events.Actor{ - ID: imageID, - Attributes: attributes, - } - - daemon.EventsService.Log(action, events.ImageEventType, actor) -} diff --git a/daemon/cache.go b/daemon/images/cache.go similarity index 79% rename from daemon/cache.go rename to daemon/images/cache.go index 13ddfbe41f..3b433106e8 100644 --- a/daemon/cache.go +++ b/daemon/images/cache.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "github.com/docker/docker/builder" @@ -7,7 +7,7 @@ import ( ) // MakeImageCache creates a stateful image cache. -func (i *imageService) MakeImageCache(sourceRefs []string) builder.ImageCache { +func (i *ImageService) MakeImageCache(sourceRefs []string) builder.ImageCache { if len(sourceRefs) == 0 { return cache.NewLocal(i.imageStore) } diff --git a/daemon/images/image.go b/daemon/images/image.go new file mode 100644 index 0000000000..7dda553138 --- /dev/null +++ b/daemon/images/image.go @@ -0,0 +1,75 @@ +package images // import "github.com/docker/docker/daemon/images" + +import ( + "fmt" + + "github.com/docker/distribution/reference" + "github.com/docker/docker/errdefs" + "github.com/docker/docker/image" +) + +// ErrImageDoesNotExist is error returned when no image can be found for a reference. +type ErrImageDoesNotExist struct { + ref reference.Reference +} + +func (e ErrImageDoesNotExist) Error() string { + ref := e.ref + if named, ok := ref.(reference.Named); ok { + ref = reference.TagNameOnly(named) + } + return fmt.Sprintf("No such image: %s", reference.FamiliarString(ref)) +} + +// NotFound implements the NotFound interface +func (e ErrImageDoesNotExist) NotFound() {} + +// GetImageIDAndOS returns an image ID and operating system corresponding to the image referred to by +// refOrID. +// called from list.go foldFilter() +func (i ImageService) GetImageIDAndOS(refOrID string) (image.ID, string, error) { + ref, err := reference.ParseAnyReference(refOrID) + if err != nil { + return "", "", errdefs.InvalidParameter(err) + } + namedRef, ok := ref.(reference.Named) + if !ok { + digested, ok := ref.(reference.Digested) + if !ok { + return "", "", ErrImageDoesNotExist{ref} + } + id := image.IDFromDigest(digested.Digest()) + if img, err := i.imageStore.Get(id); err == nil { + return id, img.OperatingSystem(), nil + } + return "", "", ErrImageDoesNotExist{ref} + } + + if digest, err := i.referenceStore.Get(namedRef); err == nil { + // Search the image stores to get the operating system, defaulting to host OS. + id := image.IDFromDigest(digest) + if img, err := i.imageStore.Get(id); err == nil { + return id, img.OperatingSystem(), nil + } + } + + // Search based on ID + if id, err := i.imageStore.Search(refOrID); err == nil { + img, err := i.imageStore.Get(id) + if err != nil { + return "", "", ErrImageDoesNotExist{ref} + } + return id, img.OperatingSystem(), nil + } + + return "", "", ErrImageDoesNotExist{ref} +} + +// GetImage returns an image corresponding to the image referred to by refOrID. +func (i *ImageService) GetImage(refOrID string) (*image.Image, error) { + imgID, _, err := i.GetImageIDAndOS(refOrID) + if err != nil { + return nil, err + } + return i.imageStore.Get(imgID) +} diff --git a/daemon/image_builder.go b/daemon/images/image_builder.go similarity index 95% rename from daemon/image_builder.go rename to daemon/images/image_builder.go index 50a5212884..b88e1bc0fa 100644 --- a/daemon/image_builder.go +++ b/daemon/images/image_builder.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "io" @@ -8,7 +8,6 @@ import ( "github.com/docker/docker/api/types/backend" "github.com/docker/docker/builder" "github.com/docker/docker/image" - "github.com/docker/docker/image/cache" "github.com/docker/docker/layer" "github.com/docker/docker/pkg/containerfs" "github.com/docker/docker/pkg/stringid" @@ -138,7 +137,7 @@ func newROLayerForImage(img *image.Image, layerStore layer.Store) (builder.ROLay } // TODO: could this use the regular daemon PullImage ? -func (i *imageService) pullForBuilder(ctx context.Context, name string, authConfigs map[string]types.AuthConfig, output io.Writer, os string) (*image.Image, error) { +func (i *ImageService) pullForBuilder(ctx context.Context, name string, authConfigs map[string]types.AuthConfig, output io.Writer, os string) (*image.Image, error) { ref, err := reference.ParseNormalizedNamed(name) if err != nil { return nil, err @@ -166,7 +165,7 @@ func (i *imageService) pullForBuilder(ctx context.Context, name string, authConf // GetImageAndReleasableLayer returns an image and releaseable layer for a reference or ID. // Every call to GetImageAndReleasableLayer MUST call releasableLayer.Release() to prevent // leaking of layers. -func (i *imageService) GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ROLayer, error) { +func (i *ImageService) GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ROLayer, error) { if refOrID == "" { if !system.IsOSSupported(opts.OS) { return nil, nil, system.ErrNotSupportedOperatingSystem @@ -204,7 +203,7 @@ func (i *imageService) GetImageAndReleasableLayer(ctx context.Context, refOrID s // CreateImage creates a new image by adding a config and ID to the image store. // This is similar to LoadImage() except that it receives JSON encoded bytes of // an image instead of a tar archive. -func (i *imageService) CreateImage(config []byte, parent string) (builder.Image, error) { +func (i *ImageService) CreateImage(config []byte, parent string) (builder.Image, error) { id, err := i.imageStore.Create(config) if err != nil { return nil, errors.Wrapf(err, "failed to create image") diff --git a/daemon/image_commit.go b/daemon/images/image_commit.go similarity index 94% rename from daemon/image_commit.go rename to daemon/images/image_commit.go index 6188b52577..4caba9f27b 100644 --- a/daemon/image_commit.go +++ b/daemon/images/image_commit.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "encoding/json" @@ -13,7 +13,7 @@ import ( ) // CommitImage creates a new image from a commit config -func (i *imageService) CommitImage(c backend.CommitConfig) (image.ID, error) { +func (i *ImageService) CommitImage(c backend.CommitConfig) (image.ID, error) { layerStore, ok := i.layerStores[c.ContainerOS] if !ok { return "", system.ErrNotSupportedOperatingSystem @@ -114,7 +114,7 @@ func exportContainerRw(layerStore layer.Store, id, mountLabel string) (arch io.R // * it doesn't log a container commit event // // This is a temporary shim. Should be removed when builder stops using commit. -func (i *imageService) CommitBuildStep(c backend.CommitConfig) (image.ID, error) { +func (i *ImageService) CommitBuildStep(c backend.CommitConfig) (image.ID, error) { container := i.containers.Get(c.ContainerID) if container == nil { // TODO: use typed error diff --git a/daemon/image_delete.go b/daemon/images/image_delete.go similarity index 96% rename from daemon/image_delete.go rename to daemon/images/image_delete.go index 2615f05ca7..60f60afaa7 100644 --- a/daemon/image_delete.go +++ b/daemon/images/image_delete.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "fmt" @@ -60,7 +60,7 @@ const ( // meaning any delete conflicts will cause the image to not be deleted and the // conflict will not be reported. // -func (i *imageService) ImageDelete(imageRef string, force, prune bool) ([]types.ImageDeleteResponseItem, error) { +func (i *ImageService) ImageDelete(imageRef string, force, prune bool) ([]types.ImageDeleteResponseItem, error) { start := time.Now() records := []types.ImageDeleteResponseItem{} @@ -229,7 +229,7 @@ func isImageIDPrefix(imageID, possiblePrefix string) bool { // repositoryRef must not be an image ID but a repository name followed by an // optional tag or digest reference. If tag or digest is omitted, the default // tag is used. Returns the resolved image reference and an error. -func (i *imageService) removeImageRef(ref reference.Named) (reference.Named, error) { +func (i *ImageService) removeImageRef(ref reference.Named) (reference.Named, error) { ref = reference.TagNameOnly(ref) // Ignore the boolean value returned, as far as we're concerned, this @@ -245,7 +245,7 @@ func (i *imageService) removeImageRef(ref reference.Named) (reference.Named, err // on the first encountered error. Removed references are logged to this // daemon's event service. An "Untagged" types.ImageDeleteResponseItem is added to the // given list of records. -func (i *imageService) removeAllReferencesToImageID(imgID image.ID, records *[]types.ImageDeleteResponseItem) error { +func (i *ImageService) removeAllReferencesToImageID(imgID image.ID, records *[]types.ImageDeleteResponseItem) error { imageRefs := i.referenceStore.References(imgID.Digest()) for _, imageRef := range imageRefs { @@ -296,7 +296,7 @@ func (idc *imageDeleteConflict) Conflict() {} // conflict is encountered, it will be returned immediately without deleting // the image. If quiet is true, any encountered conflicts will be ignored and // the function will return nil immediately without deleting the image. -func (i *imageService) imageDeleteHelper(imgID image.ID, records *[]types.ImageDeleteResponseItem, force, prune, quiet bool) error { +func (i *ImageService) imageDeleteHelper(imgID image.ID, records *[]types.ImageDeleteResponseItem, force, prune, quiet bool) error { // First, determine if this image has any conflicts. Ignore soft conflicts // if force is true. c := conflictHard @@ -355,7 +355,7 @@ func (i *imageService) imageDeleteHelper(imgID image.ID, records *[]types.ImageD // using the image. A soft conflict is any tags/digest referencing the given // image or any stopped container using the image. If ignoreSoftConflicts is // true, this function will not check for soft conflict conditions. -func (i *imageService) checkImageDeleteConflict(imgID image.ID, mask conflictType) *imageDeleteConflict { +func (i *ImageService) checkImageDeleteConflict(imgID image.ID, mask conflictType) *imageDeleteConflict { // Check if the image has any descendant images. if mask&conflictDependentChild != 0 && len(i.imageStore.Children(imgID)) > 0 { return &imageDeleteConflict{ @@ -408,6 +408,6 @@ func (i *imageService) checkImageDeleteConflict(imgID image.ID, mask conflictTyp // imageIsDangling returns whether the given image is "dangling" which means // that there are no repository references to the given image and it has no // child images. -func (i *imageService) imageIsDangling(imgID image.ID) bool { +func (i *ImageService) imageIsDangling(imgID image.ID) bool { return !(len(i.referenceStore.References(imgID.Digest())) > 0 || len(i.imageStore.Children(imgID)) > 0) } diff --git a/daemon/images/image_events.go b/daemon/images/image_events.go new file mode 100644 index 0000000000..d0b3064d70 --- /dev/null +++ b/daemon/images/image_events.go @@ -0,0 +1,39 @@ +package images // import "github.com/docker/docker/daemon/images" + +import ( + "github.com/docker/docker/api/types/events" +) + +// LogImageEvent generates an event related to an image with only the default attributes. +func (i *ImageService) LogImageEvent(imageID, refName, action string) { + i.LogImageEventWithAttributes(imageID, refName, action, map[string]string{}) +} + +// LogImageEventWithAttributes generates an event related to an image with specific given attributes. +func (i *ImageService) LogImageEventWithAttributes(imageID, refName, action string, attributes map[string]string) { + img, err := i.GetImage(imageID) + if err == nil && img.Config != nil { + // image has not been removed yet. + // it could be missing if the event is `delete`. + copyAttributes(attributes, img.Config.Labels) + } + if refName != "" { + attributes["name"] = refName + } + actor := events.Actor{ + ID: imageID, + Attributes: attributes, + } + + i.eventsService.Log(action, events.ImageEventType, actor) +} + +// copyAttributes guarantees that labels are not mutated by event triggers. +func copyAttributes(attributes, labels map[string]string) { + if labels == nil { + return + } + for k, v := range labels { + attributes[k] = v + } +} diff --git a/daemon/image_exporter.go b/daemon/images/image_exporter.go similarity index 82% rename from daemon/image_exporter.go rename to daemon/images/image_exporter.go index 575565fc25..58105dcb71 100644 --- a/daemon/image_exporter.go +++ b/daemon/images/image_exporter.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "io" @@ -11,7 +11,7 @@ import ( // stream. All images with the given tag and all versions containing // the same tag are exported. names is the set of tags to export, and // outStream is the writer which the images are written to. -func (i *imageService) ExportImage(names []string, outStream io.Writer) error { +func (i *ImageService) ExportImage(names []string, outStream io.Writer) error { imageExporter := tarexport.NewTarExporter(i.imageStore, i.layerStores, i.referenceStore, i) return imageExporter.Save(names, outStream) } @@ -19,7 +19,7 @@ func (i *imageService) ExportImage(names []string, outStream io.Writer) error { // LoadImage uploads a set of images into the repository. This is the // complement of ImageExport. The input stream is an uncompressed tar // ball containing images and metadata. -func (i *imageService) LoadImage(inTar io.ReadCloser, outStream io.Writer, quiet bool) error { +func (i *ImageService) LoadImage(inTar io.ReadCloser, outStream io.Writer, quiet bool) error { imageExporter := tarexport.NewTarExporter(i.imageStore, i.layerStores, i.referenceStore, i) return imageExporter.Load(inTar, outStream, quiet) } diff --git a/daemon/image_history.go b/daemon/images/image_history.go similarity index 93% rename from daemon/image_history.go rename to daemon/images/image_history.go index 621e8cec05..2b92292631 100644 --- a/daemon/image_history.go +++ b/daemon/images/image_history.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "fmt" @@ -11,7 +11,7 @@ import ( // ImageHistory returns a slice of ImageHistory structures for the specified image // name by walking the image lineage. -func (i *imageService) ImageHistory(name string) ([]*image.HistoryResponseItem, error) { +func (i *ImageService) ImageHistory(name string) ([]*image.HistoryResponseItem, error) { start := time.Now() img, err := i.GetImage(name) if err != nil { diff --git a/daemon/image_import.go b/daemon/images/image_import.go similarity index 96% rename from daemon/image_import.go rename to daemon/images/image_import.go index f9bf0b119b..8d54e0704f 100644 --- a/daemon/image_import.go +++ b/daemon/images/image_import.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "encoding/json" @@ -27,7 +27,7 @@ import ( // inConfig (if src is "-"), or from a URI specified in src. Progress output is // written to outStream. Repository and tag names can optionally be given in // the repo and tag arguments, respectively. -func (i *imageService) ImportImage(src string, repository, os string, tag string, msg string, inConfig io.ReadCloser, outStream io.Writer, changes []string) error { +func (i *ImageService) ImportImage(src string, repository, os string, tag string, msg string, inConfig io.ReadCloser, outStream io.Writer, changes []string) error { var ( rc io.ReadCloser resp *http.Response diff --git a/daemon/image_inspect.go b/daemon/images/image_inspect.go similarity index 92% rename from daemon/image_inspect.go rename to daemon/images/image_inspect.go index c00ceb343c..16c4c9b2dc 100644 --- a/daemon/image_inspect.go +++ b/daemon/images/image_inspect.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "time" @@ -13,7 +13,7 @@ import ( // LookupImage looks up an image by name and returns it as an ImageInspect // structure. -func (i *imageService) LookupImage(name string) (*types.ImageInspect, error) { +func (i *ImageService) LookupImage(name string) (*types.ImageInspect, error) { img, err := i.GetImage(name) if err != nil { return nil, errors.Wrapf(err, "no such image: %s", name) @@ -86,7 +86,7 @@ func (i *imageService) LookupImage(name string) (*types.ImageInspect, error) { }, } - imageInspect.GraphDriver.Name = i.GraphDriverForOS(img.OperatingSystem()) + imageInspect.GraphDriver.Name = i.layerStores[img.OperatingSystem()].DriverName() imageInspect.GraphDriver.Data = layerMetadata return imageInspect, nil diff --git a/daemon/image_prune.go b/daemon/images/image_prune.go similarity index 73% rename from daemon/image_prune.go rename to daemon/images/image_prune.go index 92399a38be..9427536759 100644 --- a/daemon/image_prune.go +++ b/daemon/images/image_prune.go @@ -1,11 +1,14 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( + "fmt" "sync/atomic" + "time" "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" + timetypes "github.com/docker/docker/api/types/time" "github.com/docker/docker/errdefs" "github.com/docker/docker/image" "github.com/docker/docker/layer" @@ -21,8 +24,12 @@ var imagesAcceptedFilters = map[string]bool{ "until": true, } +// errPruneRunning is returned when a prune request is received while +// one is in progress +var errPruneRunning = fmt.Errorf("a prune operation is already running") + // ImagesPrune removes unused images -func (i *imageService) ImagesPrune(ctx context.Context, pruneFilters filters.Args) (*types.ImagesPruneReport, error) { +func (i *ImageService) ImagesPrune(ctx context.Context, pruneFilters filters.Args) (*types.ImagesPruneReport, error) { if !atomic.CompareAndSwapInt32(&i.pruneRunning, 0, 1) { return nil, errPruneRunning } @@ -96,7 +103,7 @@ deleteImagesLoop: } deletedImages := []types.ImageDeleteResponseItem{} - refs := i.referenceStore.References(dgst) + refs := i.referenceStore.References(id.Digest()) if len(refs) > 0 { shouldDelete := !danglingOnly if !shouldDelete { @@ -166,3 +173,38 @@ func imageDeleteFailed(ref string, err error) bool { return true } } + +func matchLabels(pruneFilters filters.Args, labels map[string]string) bool { + if !pruneFilters.MatchKVList("label", labels) { + return false + } + // By default MatchKVList will return true if field (like 'label!') does not exist + // So we have to add additional Contains("label!") check + if pruneFilters.Contains("label!") { + if pruneFilters.MatchKVList("label!", labels) { + return false + } + } + return true +} + +func getUntilFromPruneFilters(pruneFilters filters.Args) (time.Time, error) { + until := time.Time{} + if !pruneFilters.Contains("until") { + return until, nil + } + untilFilters := pruneFilters.Get("until") + if len(untilFilters) > 1 { + return until, fmt.Errorf("more than one until filter specified") + } + ts, err := timetypes.GetTimestamp(untilFilters[0], time.Now()) + if err != nil { + return until, err + } + seconds, nanoseconds, err := timetypes.ParseTimestamps(ts, 0) + if err != nil { + return until, err + } + until = time.Unix(seconds, nanoseconds) + return until, nil +} diff --git a/daemon/image_pull.go b/daemon/images/image_pull.go similarity index 93% rename from daemon/image_pull.go rename to daemon/images/image_pull.go index f6606cb38c..41056af678 100644 --- a/daemon/image_pull.go +++ b/daemon/images/image_pull.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "io" @@ -19,7 +19,7 @@ import ( // PullImage initiates a pull operation. image is the repository name to pull, and // tag may be either empty, or indicate a specific tag to pull. -func (i *imageService) PullImage(ctx context.Context, image, tag, os string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { +func (i *ImageService) PullImage(ctx context.Context, image, tag, os string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { // Special case: "pull -a" may send an image name with a // trailing :. This is ugly, but let's not break API // compatibility. @@ -47,7 +47,7 @@ func (i *imageService) PullImage(ctx context.Context, image, tag, os string, met return i.pullImageWithReference(ctx, ref, os, metaHeaders, authConfig, outStream) } -func (i *imageService) pullImageWithReference(ctx context.Context, ref reference.Named, os string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { +func (i *ImageService) pullImageWithReference(ctx context.Context, ref reference.Named, os string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { // Include a buffer so that slow client connections don't affect // transfer performance. progressChan := make(chan progress.Progress, 100) @@ -89,7 +89,7 @@ func (i *imageService) pullImageWithReference(ctx context.Context, ref reference } // GetRepository returns a repository from the registry. -func (i *imageService) GetRepository(ctx context.Context, ref reference.Named, authConfig *types.AuthConfig) (dist.Repository, bool, error) { +func (i *ImageService) GetRepository(ctx context.Context, ref reference.Named, authConfig *types.AuthConfig) (dist.Repository, bool, error) { // get repository info repoInfo, err := i.registryService.ResolveRepository(ref) if err != nil { diff --git a/daemon/image_push.go b/daemon/images/image_push.go similarity index 93% rename from daemon/image_push.go rename to daemon/images/image_push.go index 23c2d1e2eb..af606a07b5 100644 --- a/daemon/image_push.go +++ b/daemon/images/image_push.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "io" @@ -13,7 +13,7 @@ import ( ) // PushImage initiates a push operation on the repository named localName. -func (i *imageService) PushImage(ctx context.Context, image, tag string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { +func (i *ImageService) PushImage(ctx context.Context, image, tag string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { ref, err := reference.ParseNormalizedNamed(image) if err != nil { return err diff --git a/daemon/image_search.go b/daemon/images/image_search.go similarity index 95% rename from daemon/image_search.go rename to daemon/images/image_search.go index 101bd2e306..a901571439 100644 --- a/daemon/image_search.go +++ b/daemon/images/image_search.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "strconv" @@ -22,7 +22,7 @@ var acceptedSearchFilterTags = map[string]bool{ // // TODO: this could be implemented in a registry service instead of the image // service. -func (i *imageService) SearchRegistryForImages(ctx context.Context, filtersArgs string, term string, limit int, +func (i *ImageService) SearchRegistryForImages(ctx context.Context, filtersArgs string, term string, limit int, authConfig *types.AuthConfig, headers map[string][]string) (*registrytypes.SearchResults, error) { diff --git a/daemon/search_test.go b/daemon/images/image_search_test.go similarity index 97% rename from daemon/search_test.go rename to daemon/images/image_search_test.go index 0237b103b8..74795c5865 100644 --- a/daemon/search_test.go +++ b/daemon/images/image_search_test.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "errors" @@ -76,8 +76,8 @@ func TestSearchRegistryForImagesErrors(t *testing.T) { }, } for index, e := range errorCases { - daemon := &Daemon{ - RegistryService: &FakeService{ + daemon := &ImageService{ + registryService: &FakeService{ shouldReturnError: e.shouldReturnError, }, } @@ -322,8 +322,8 @@ func TestSearchRegistryForImages(t *testing.T) { }, } for index, s := range successCases { - daemon := &Daemon{ - RegistryService: &FakeService{ + daemon := &ImageService{ + registryService: &FakeService{ term: term, results: s.registryResults, }, diff --git a/daemon/image_tag.go b/daemon/images/image_tag.go similarity index 84% rename from daemon/image_tag.go rename to daemon/images/image_tag.go index 6e929437f5..6456ff81f9 100644 --- a/daemon/image_tag.go +++ b/daemon/images/image_tag.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "github.com/docker/distribution/reference" @@ -7,7 +7,7 @@ import ( // TagImage creates the tag specified by newTag, pointing to the image named // imageName (alternatively, imageName can also be an image ID). -func (i *imageService) TagImage(imageName, repository, tag string) (string, error) { +func (i *ImageService) TagImage(imageName, repository, tag string) (string, error) { imageID, _, err := i.GetImageIDAndOS(imageName) if err != nil { return "", err @@ -28,7 +28,7 @@ func (i *imageService) TagImage(imageName, repository, tag string) (string, erro } // TagImageWithReference adds the given reference to the image ID provided. -func (i *imageService) TagImageWithReference(imageID image.ID, newTag reference.Named) error { +func (i *ImageService) TagImageWithReference(imageID image.ID, newTag reference.Named) error { if err := i.referenceStore.AddTag(newTag, imageID.Digest(), true); err != nil { return err } diff --git a/daemon/image_getsize_unix.go b/daemon/images/image_unix.go similarity index 83% rename from daemon/image_getsize_unix.go rename to daemon/images/image_unix.go index faa1edd843..3f577271a2 100644 --- a/daemon/image_getsize_unix.go +++ b/daemon/images/image_unix.go @@ -1,6 +1,6 @@ // +build linux freebsd -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "runtime" @@ -8,8 +8,8 @@ import ( "github.com/sirupsen/logrus" ) -// getSize returns the real size & virtual size of the container. -func (i *imageService) GetContainerLayerSize(containerID string) (int64, int64) { +// GetContainerLayerSize returns the real size & virtual size of the container. +func (i *ImageService) GetContainerLayerSize(containerID string) (int64, int64) { var ( sizeRw, sizeRootfs int64 err error diff --git a/daemon/image_windows.go b/daemon/images/image_windows.go similarity index 67% rename from daemon/image_windows.go rename to daemon/images/image_windows.go index d661400a38..6f4be49736 100644 --- a/daemon/image_windows.go +++ b/daemon/images/image_windows.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images import ( "github.com/docker/docker/image" @@ -7,8 +7,14 @@ import ( "github.com/pkg/errors" ) +// GetContainerLayerSize returns real size & virtual size +func (i *ImageService) GetContainerLayerSize(containerID string) (int64, int64) { + // TODO Windows + return 0, 0 +} + // GetLayerFolders returns the layer folders from an image RootFS -func (daemon *Daemon) GetLayerFolders(img *image.Image, rwLayer layer.RWLayer) ([]string, error) { +func (i *ImageService) GetLayerFolders(img *image.Image, rwLayer layer.RWLayer) ([]string, error) { folders := []string{} max := len(img.RootFS.DiffIDs) for index := 1; index <= max; index++ { @@ -17,9 +23,9 @@ func (daemon *Daemon) GetLayerFolders(img *image.Image, rwLayer layer.RWLayer) ( if !system.IsOSSupported(img.OperatingSystem()) { return nil, errors.Wrapf(system.ErrNotSupportedOperatingSystem, "cannot get layerpath for ImageID %s", img.RootFS.ChainID()) } - layerPath, err := layer.GetLayerPath(daemon.layerStores[img.OperatingSystem()], img.RootFS.ChainID()) + layerPath, err := layer.GetLayerPath(i.layerStores[img.OperatingSystem()], img.RootFS.ChainID()) if err != nil { - return nil, errors.Wrapf(err, "failed to get layer path from graphdriver %s for ImageID %s", daemon.layerStores[img.OperatingSystem()], img.RootFS.ChainID()) + return nil, errors.Wrapf(err, "failed to get layer path from graphdriver %s for ImageID %s", i.layerStores[img.OperatingSystem()], img.RootFS.ChainID()) } // Reverse order, expecting parent first folders = append([]string{layerPath}, folders...) diff --git a/daemon/images.go b/daemon/images/images.go similarity index 97% rename from daemon/images.go rename to daemon/images/images.go index 903583bbcb..46056f15b5 100644 --- a/daemon/images.go +++ b/daemon/images/images.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "encoding/json" @@ -34,7 +34,7 @@ func (r byCreated) Swap(i, j int) { r[i], r[j] = r[j], r[i] } func (r byCreated) Less(i, j int) bool { return r[i].Created < r[j].Created } // Map returns a map of all images in the ImageStore -func (i *imageService) Map() map[image.ID]*image.Image { +func (i *ImageService) Map() map[image.ID]*image.Image { return i.imageStore.Map() } @@ -43,7 +43,7 @@ func (i *imageService) Map() map[image.ID]*image.Image { // filter is a shell glob string applied to repository names. The argument // named all controls whether all images in the graph are filtered, or just // the heads. -func (i *imageService) Images(imageFilters filters.Args, all bool, withExtraAttrs bool) ([]*types.ImageSummary, error) { +func (i *ImageService) Images(imageFilters filters.Args, all bool, withExtraAttrs bool) ([]*types.ImageSummary, error) { var ( allImages map[image.ID]*image.Image err error @@ -249,7 +249,7 @@ func (i *imageService) Images(imageFilters filters.Args, all bool, withExtraAttr // This new image contains only the layers from it's parent + 1 extra layer which contains the diff of all the layers in between. // The existing image(s) is not destroyed. // If no parent is specified, a new image with the diff of all the specified image's layers merged into a new layer that has no parents. -func (i *imageService) SquashImage(id, parent string) (string, error) { +func (i *ImageService) SquashImage(id, parent string) (string, error) { var ( img *image.Image diff --git a/daemon/images/locals.go b/daemon/images/locals.go new file mode 100644 index 0000000000..a57ea2da60 --- /dev/null +++ b/daemon/images/locals.go @@ -0,0 +1,32 @@ +package images // import "github.com/docker/docker/daemon/images" + +import ( + "fmt" + + metrics "github.com/docker/go-metrics" +) + +type invalidFilter struct { + filter string + value interface{} +} + +func (e invalidFilter) Error() string { + msg := "Invalid filter '" + e.filter + if e.value != nil { + msg += fmt.Sprintf("=%s", e.value) + } + return msg + "'" +} + +func (e invalidFilter) InvalidParameter() {} + +var imageActions metrics.LabeledTimer + +func init() { + ns := metrics.NewNamespace("engine", "daemon", nil) + imageActions = ns.NewLabeledTimer("image_actions", "The number of seconds it takes to process each image action", "action") + // TODO: is it OK to register a namespace with the same name? Or does this + // need to be exported from somewhere? + metrics.Register(ns) +} diff --git a/daemon/images/service.go b/daemon/images/service.go new file mode 100644 index 0000000000..70a8bf4455 --- /dev/null +++ b/daemon/images/service.go @@ -0,0 +1,229 @@ +package images // import "github.com/docker/docker/daemon/images" + +import ( + "context" + "os" + + "github.com/docker/docker/container" + daemonevents "github.com/docker/docker/daemon/events" + "github.com/docker/docker/distribution/metadata" + "github.com/docker/docker/distribution/xfer" + "github.com/docker/docker/image" + "github.com/docker/docker/layer" + dockerreference "github.com/docker/docker/reference" + "github.com/docker/docker/registry" + "github.com/docker/libtrust" + "github.com/opencontainers/go-digest" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +type containerStore interface { + // used by image delete + First(container.StoreFilter) *container.Container + // used by image prune, and image list + List() []*container.Container + // TODO: remove, only used for CommitBuildStep + Get(string) *container.Container +} + +// ImageServiceConfig is the configuration used to create a new ImageService +type ImageServiceConfig struct { + ContainerStore containerStore + DistributionMetadataStore metadata.Store + EventsService *daemonevents.Events + ImageStore image.Store + LayerStores map[string]layer.Store + MaxConcurrentDownloads int + MaxConcurrentUploads int + ReferenceStore dockerreference.Store + RegistryService registry.Service + TrustKey libtrust.PrivateKey +} + +// NewImageService returns a new ImageService from a configuration +func NewImageService(config ImageServiceConfig) *ImageService { + logrus.Debugf("Max Concurrent Downloads: %d", config.MaxConcurrentDownloads) + logrus.Debugf("Max Concurrent Uploads: %d", config.MaxConcurrentUploads) + return &ImageService{ + containers: config.ContainerStore, + distributionMetadataStore: config.DistributionMetadataStore, + downloadManager: xfer.NewLayerDownloadManager(config.LayerStores, config.MaxConcurrentDownloads), + eventsService: config.EventsService, + imageStore: config.ImageStore, + layerStores: config.LayerStores, + referenceStore: config.ReferenceStore, + registryService: config.RegistryService, + trustKey: config.TrustKey, + uploadManager: xfer.NewLayerUploadManager(config.MaxConcurrentUploads), + } +} + +// ImageService provides a backend for image management +type ImageService struct { + containers containerStore + distributionMetadataStore metadata.Store + downloadManager *xfer.LayerDownloadManager + eventsService *daemonevents.Events + imageStore image.Store + layerStores map[string]layer.Store // By operating system + pruneRunning int32 + referenceStore dockerreference.Store + registryService registry.Service + trustKey libtrust.PrivateKey + uploadManager *xfer.LayerUploadManager +} + +// CountImages returns the number of images stored by ImageService +// called from info.go +func (i *ImageService) CountImages() int { + return len(i.imageStore.Map()) +} + +// Children returns the children image.IDs for a parent image. +// called from list.go to filter containers +// TODO: refactor to expose an ancestry for image.ID? +func (i *ImageService) Children(id image.ID) []image.ID { + return i.imageStore.Children(id) +} + +// CreateLayer creates a filesystem layer for a container. +// called from create.go +// TODO: accept an opt struct instead of container? +func (i *ImageService) CreateLayer(container *container.Container, initFunc layer.MountInit) (layer.RWLayer, error) { + var layerID layer.ChainID + if container.ImageID != "" { + img, err := i.imageStore.Get(container.ImageID) + if err != nil { + return nil, err + } + layerID = img.RootFS.ChainID() + } + + rwLayerOpts := &layer.CreateRWLayerOpts{ + MountLabel: container.MountLabel, + InitFunc: initFunc, + StorageOpt: container.HostConfig.StorageOpt, + } + + // Indexing by OS is safe here as validation of OS has already been performed in create() (the only + // caller), and guaranteed non-nil + return i.layerStores[container.OS].CreateRWLayer(container.ID, layerID, rwLayerOpts) +} + +// GetLayerByID returns a layer by ID and operating system +// called from daemon.go Daemon.restore(), and Daemon.containerExport() +func (i *ImageService) GetLayerByID(cid string, os string) (layer.RWLayer, error) { + return i.layerStores[os].GetRWLayer(cid) +} + +// LayerStoreStatus returns the status for each layer store +// called from info.go +func (i *ImageService) LayerStoreStatus() map[string][][2]string { + result := make(map[string][][2]string) + for os, store := range i.layerStores { + result[os] = store.DriverStatus() + } + return result +} + +// GetLayerMountID returns the mount ID for a layer +// called from daemon.go Daemon.Shutdown(), and Daemon.Cleanup() (cleanup is actually continerCleanup) +// TODO: needs to be refactored to Unmount (see callers), or removed and replaced +// with GetLayerByID +func (i *ImageService) GetLayerMountID(cid string, os string) (string, error) { + return i.layerStores[os].GetMountID(cid) +} + +// Cleanup resources before the process is shutdown. +// called from daemon.go Daemon.Shutdown() +func (i *ImageService) Cleanup() { + for os, ls := range i.layerStores { + if ls != nil { + if err := ls.Cleanup(); err != nil { + logrus.Errorf("Error during layer Store.Cleanup(): %v %s", err, os) + } + } + } +} + +// GraphDriverForOS returns the name of the graph drvier +// moved from Daemon.GraphDriverName, used by: +// - newContainer +// - to report an error in Daemon.Mount(container) +func (i *ImageService) GraphDriverForOS(os string) string { + return i.layerStores[os].DriverName() +} + +// ReleaseLayer releases a layer allowing it to be removed +// called from delete.go Daemon.cleanupContainer(), and Daemon.containerExport() +func (i *ImageService) ReleaseLayer(rwlayer layer.RWLayer, containerOS string) error { + metadata, err := i.layerStores[containerOS].ReleaseRWLayer(rwlayer) + layer.LogReleaseMetadata(metadata) + if err != nil && err != layer.ErrMountDoesNotExist && !os.IsNotExist(errors.Cause(err)) { + return errors.Wrapf(err, "driver %q failed to remove root filesystem", + i.layerStores[containerOS].DriverName()) + } + return nil +} + +// LayerDiskUsage returns the number of bytes used by layer stores +// called from disk_usage.go +func (i *ImageService) LayerDiskUsage(ctx context.Context) (int64, error) { + var allLayersSize int64 + layerRefs := i.getLayerRefs() + for _, ls := range i.layerStores { + allLayers := ls.Map() + for _, l := range allLayers { + select { + case <-ctx.Done(): + return allLayersSize, ctx.Err() + default: + size, err := l.DiffSize() + if err == nil { + if _, ok := layerRefs[l.ChainID()]; ok { + allLayersSize += size + } else { + logrus.Warnf("found leaked image layer %v", l.ChainID()) + } + } else { + logrus.Warnf("failed to get diff size for layer %v", l.ChainID()) + } + } + } + } + return allLayersSize, nil +} + +func (i *ImageService) getLayerRefs() map[layer.ChainID]int { + tmpImages := i.imageStore.Map() + layerRefs := map[layer.ChainID]int{} + for id, img := range tmpImages { + dgst := digest.Digest(id) + if len(i.referenceStore.References(dgst)) == 0 && len(i.imageStore.Children(id)) != 0 { + continue + } + + rootFS := *img.RootFS + rootFS.DiffIDs = nil + for _, id := range img.RootFS.DiffIDs { + rootFS.Append(id) + chid := rootFS.ChainID() + layerRefs[chid]++ + } + } + + return layerRefs +} + +// UpdateConfig values +// +// called from reload.go +func (i *ImageService) UpdateConfig(maxDownloads, maxUploads *int) { + if i.downloadManager != nil && maxDownloads != nil { + i.downloadManager.SetConcurrency(*maxDownloads) + } + if i.uploadManager != nil && maxUploads != nil { + i.uploadManager.SetConcurrency(*maxUploads) + } +} diff --git a/daemon/info.go b/daemon/info.go index 24940be16a..6f5ae69896 100644 --- a/daemon/info.go +++ b/daemon/info.go @@ -80,7 +80,7 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) { var ds [][2]string drivers := "" - statuses := daemon.imageService.GraphDriverStatuses() + statuses := daemon.imageService.LayerStoreStatus() for os, gd := range daemon.graphDrivers { ds = append(ds, statuses[os]...) drivers += gd diff --git a/daemon/list.go b/daemon/list.go index 254cb7253a..ddc6fe70be 100644 --- a/daemon/list.go +++ b/daemon/list.go @@ -9,6 +9,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/container" + "github.com/docker/docker/daemon/images" "github.com/docker/docker/errdefs" "github.com/docker/docker/image" "github.com/docker/docker/volume" @@ -592,7 +593,7 @@ func (daemon *Daemon) refreshImage(s *container.Snapshot, ctx *listContext) (*ty image := s.Image // keep the original ref if still valid (hasn't changed) if image != s.ImageID { id, _, err := daemon.imageService.GetImageIDAndOS(image) - if _, isDNE := err.(errImageDoesNotExist); err != nil && !isDNE { + if _, isDNE := err.(images.ErrImageDoesNotExist); err != nil && !isDNE { return nil, err } if err != nil || id.String() != s.ImageID { diff --git a/daemon/metrics.go b/daemon/metrics.go index b283526d9a..02a36603c9 100644 --- a/daemon/metrics.go +++ b/daemon/metrics.go @@ -14,7 +14,6 @@ const metricsPluginType = "MetricsCollector" var ( containerActions metrics.LabeledTimer - imageActions metrics.LabeledTimer networkActions metrics.LabeledTimer engineInfo metrics.LabeledGauge engineCpus metrics.Gauge @@ -52,7 +51,6 @@ func init() { engineMemory = ns.NewGauge("engine_memory", "The number of bytes of memory that the host system of the engine has", metrics.Bytes) healthChecksCounter = ns.NewCounter("health_checks", "The total number of health checks") healthChecksFailedCounter = ns.NewCounter("health_checks_failed", "The total number of failed health checks") - imageActions = ns.NewLabeledTimer("image_actions", "The number of seconds it takes to process each image action", "action") stateCtr = newStateCounter(ns.NewDesc("container_states", "The count of containers in various states", metrics.Unit("containers"), "state")) ns.Add(stateCtr) diff --git a/daemon/oci_windows.go b/daemon/oci_windows.go index 93d4ae2297..98243bcf8a 100644 --- a/daemon/oci_windows.go +++ b/daemon/oci_windows.go @@ -138,7 +138,7 @@ func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) { } } s.Process.User.Username = c.Config.User - s.Windows.LayerFolders, err = daemon.GetLayerFolders(img, c.RWLayer) + s.Windows.LayerFolders, err = daemon.imageService.GetLayerFolders(img, c.RWLayer) if err != nil { return nil, errors.Wrapf(err, "container %s", c.ID) } diff --git a/daemon/reload.go b/daemon/reload.go index 5d5cba985d..210864ff87 100644 --- a/daemon/reload.go +++ b/daemon/reload.go @@ -90,12 +90,6 @@ func (daemon *Daemon) reloadMaxConcurrentDownloadsAndUploads(conf *config.Config daemon.configStore.MaxConcurrentDownloads = &maxConcurrentDownloads } logrus.Debugf("Reset Max Concurrent Downloads: %d", *daemon.configStore.MaxConcurrentDownloads) - if daemon.imageService.downloadManager != nil { - daemon.imageService.downloadManager.SetConcurrency(*daemon.configStore.MaxConcurrentDownloads) - } - - // prepare reload event attributes with updatable configurations - attributes["max-concurrent-downloads"] = fmt.Sprintf("%d", *daemon.configStore.MaxConcurrentDownloads) // If no value is set for max-concurrent-upload we assume it is the default value // We always "reset" as the cost is lightweight and easy to maintain. @@ -106,10 +100,10 @@ func (daemon *Daemon) reloadMaxConcurrentDownloadsAndUploads(conf *config.Config daemon.configStore.MaxConcurrentUploads = &maxConcurrentUploads } logrus.Debugf("Reset Max Concurrent Uploads: %d", *daemon.configStore.MaxConcurrentUploads) - if daemon.imageService.uploadManager != nil { - daemon.imageService.uploadManager.SetConcurrency(*daemon.configStore.MaxConcurrentUploads) - } + daemon.imageService.UpdateConfig(conf.MaxConcurrentDownloads, conf.MaxConcurrentUploads) + // prepare reload event attributes with updatable configurations + attributes["max-concurrent-downloads"] = fmt.Sprintf("%d", *daemon.configStore.MaxConcurrentDownloads) // prepare reload event attributes with updatable configurations attributes["max-concurrent-uploads"] = fmt.Sprintf("%d", *daemon.configStore.MaxConcurrentUploads) } diff --git a/daemon/reload_test.go b/daemon/reload_test.go index 3e43b6d818..a2500b2bb2 100644 --- a/daemon/reload_test.go +++ b/daemon/reload_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/docker/docker/daemon/config" + "github.com/docker/docker/daemon/images" "github.com/docker/docker/pkg/discovery" _ "github.com/docker/docker/pkg/discovery/memory" "github.com/docker/docker/registry" @@ -21,7 +22,7 @@ func TestDaemonReloadLabels(t *testing.T) { Labels: []string{"foo:bar"}, }, }, - imageService: &imageService{}, + imageService: images.NewImageService(images.ImageServiceConfig{}), } valuesSets := make(map[string]interface{}) @@ -46,7 +47,7 @@ func TestDaemonReloadLabels(t *testing.T) { func TestDaemonReloadAllowNondistributableArtifacts(t *testing.T) { daemon := &Daemon{ configStore: &config.Config{}, - imageService: &imageService{}, + imageService: images.NewImageService(images.ImageServiceConfig{}), } var err error @@ -101,7 +102,7 @@ func TestDaemonReloadAllowNondistributableArtifacts(t *testing.T) { func TestDaemonReloadMirrors(t *testing.T) { daemon := &Daemon{ - imageService: &imageService{}, + imageService: images.NewImageService(images.ImageServiceConfig{}), } var err error daemon.RegistryService, err = registry.NewService(registry.ServiceOptions{ @@ -200,7 +201,7 @@ func TestDaemonReloadMirrors(t *testing.T) { func TestDaemonReloadInsecureRegistries(t *testing.T) { daemon := &Daemon{ - imageService: &imageService{}, + imageService: images.NewImageService(images.ImageServiceConfig{}), } var err error // initialize daemon with existing insecure registries: "127.0.0.0/8", "10.10.1.11:5000", "10.10.1.22:5000" @@ -292,7 +293,7 @@ func TestDaemonReloadInsecureRegistries(t *testing.T) { func TestDaemonReloadNotAffectOthers(t *testing.T) { daemon := &Daemon{ - imageService: &imageService{}, + imageService: images.NewImageService(images.ImageServiceConfig{}), } daemon.configStore = &config.Config{ CommonConfig: config.CommonConfig{ @@ -326,7 +327,7 @@ func TestDaemonReloadNotAffectOthers(t *testing.T) { func TestDaemonDiscoveryReload(t *testing.T) { daemon := &Daemon{ - imageService: &imageService{}, + imageService: images.NewImageService(images.ImageServiceConfig{}), } daemon.configStore = &config.Config{ CommonConfig: config.CommonConfig{ @@ -405,7 +406,7 @@ func TestDaemonDiscoveryReload(t *testing.T) { func TestDaemonDiscoveryReloadFromEmptyDiscovery(t *testing.T) { daemon := &Daemon{ - imageService: &imageService{}, + imageService: images.NewImageService(images.ImageServiceConfig{}), } daemon.configStore = &config.Config{} @@ -452,7 +453,7 @@ func TestDaemonDiscoveryReloadFromEmptyDiscovery(t *testing.T) { func TestDaemonDiscoveryReloadOnlyClusterAdvertise(t *testing.T) { daemon := &Daemon{ - imageService: &imageService{}, + imageService: images.NewImageService(images.ImageServiceConfig{}), } daemon.configStore = &config.Config{ CommonConfig: config.CommonConfig{ @@ -498,7 +499,7 @@ func TestDaemonDiscoveryReloadOnlyClusterAdvertise(t *testing.T) { func TestDaemonReloadNetworkDiagnosticPort(t *testing.T) { daemon := &Daemon{ - imageService: &imageService{}, + imageService: images.NewImageService(images.ImageServiceConfig{}), } daemon.configStore = &config.Config{} diff --git a/daemon/start.go b/daemon/start.go index ad61747a2f..c00bd9ceb2 100644 --- a/daemon/start.go +++ b/daemon/start.go @@ -223,7 +223,7 @@ func (daemon *Daemon) Cleanup(container *container.Container) { if err := daemon.conditionalUnmountOnCleanup(container); err != nil { // FIXME: remove once reference counting for graphdrivers has been refactored // Ensure that all the mounts are gone - if mountid, err := daemon.imageService.GetContainerMountID(container.ID, container.OS); err == nil { + if mountid, err := daemon.imageService.GetLayerMountID(container.ID, container.OS); err == nil { daemon.cleanupMountsByID(mountid) } }