Merge pull request #36373 from dnephin/prepare-for-image-service

Move daemon image code in preparation for ImageService
This commit is contained in:
Sebastiaan van Stijn 2018-02-22 17:32:43 +01:00 committed by GitHub
commit 867a10aade
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 360 additions and 322 deletions

View file

@ -1,9 +1,7 @@
package daemon // import "github.com/docker/docker/daemon"
import (
"encoding/json"
"fmt"
"io"
"runtime"
"strings"
"time"
@ -12,10 +10,6 @@ import (
containertypes "github.com/docker/docker/api/types/container"
"github.com/docker/docker/builder/dockerfile"
"github.com/docker/docker/errdefs"
"github.com/docker/docker/image"
"github.com/docker/docker/layer"
"github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/pkg/system"
"github.com/pkg/errors"
)
@ -190,115 +184,3 @@ func (daemon *Daemon) CreateImageFromContainer(name string, c *backend.CreateIma
containerActions.WithValues("commit").UpdateSince(start)
return id.String(), nil
}
func (daemon *Daemon) commitImage(c backend.CommitConfig) (image.ID, error) {
layerStore, ok := daemon.layerStores[c.ContainerOS]
if !ok {
return "", system.ErrNotSupportedOperatingSystem
}
rwTar, err := exportContainerRw(layerStore, c.ContainerID, c.ContainerMountLabel)
if err != nil {
return "", err
}
defer func() {
if rwTar != nil {
rwTar.Close()
}
}()
var parent *image.Image
if c.ParentImageID == "" {
parent = new(image.Image)
parent.RootFS = image.NewRootFS()
} else {
parent, err = daemon.imageStore.Get(image.ID(c.ParentImageID))
if err != nil {
return "", err
}
}
l, err := layerStore.Register(rwTar, parent.RootFS.ChainID())
if err != nil {
return "", err
}
defer layer.ReleaseAndLog(layerStore, l)
cc := image.ChildConfig{
ContainerID: c.ContainerID,
Author: c.Author,
Comment: c.Comment,
ContainerConfig: c.ContainerConfig,
Config: c.Config,
DiffID: l.DiffID(),
}
config, err := json.Marshal(image.NewChildImage(parent, cc, c.ContainerOS))
if err != nil {
return "", err
}
id, err := daemon.imageStore.Create(config)
if err != nil {
return "", err
}
if c.ParentImageID != "" {
if err := daemon.imageStore.SetParent(id, image.ID(c.ParentImageID)); err != nil {
return "", err
}
}
return id, nil
}
func exportContainerRw(layerStore layer.Store, id, mountLabel string) (arch io.ReadCloser, err error) {
rwlayer, err := layerStore.GetRWLayer(id)
if err != nil {
return nil, err
}
defer func() {
if err != nil {
layerStore.ReleaseRWLayer(rwlayer)
}
}()
// TODO: this mount call is not necessary as we assume that TarStream() should
// mount the layer if needed. But the Diff() function for windows requests that
// the layer should be mounted when calling it. So we reserve this mount call
// until windows driver can implement Diff() interface correctly.
_, err = rwlayer.Mount(mountLabel)
if err != nil {
return nil, err
}
archive, err := rwlayer.TarStream()
if err != nil {
rwlayer.Unmount()
return nil, err
}
return ioutils.NewReadCloserWrapper(archive, func() error {
archive.Close()
err = rwlayer.Unmount()
layerStore.ReleaseRWLayer(rwlayer)
return err
}),
nil
}
// CommitBuildStep is used by the builder to create an image for each step in
// the build.
//
// This method is different from CreateImageFromContainer:
// * it doesn't attempt to validate container state
// * it doesn't send a commit action to metrics
// * it doesn't log a container commit event
//
// This is a temporary shim. Should be removed when builder stops using commit.
func (daemon *Daemon) CommitBuildStep(c backend.CommitConfig) (image.ID, error) {
container, err := daemon.GetContainer(c.ContainerID)
if err != nil {
return "", err
}
c.ContainerMountLabel = container.MountLabel
c.ContainerOS = container.OS
c.ParentImageID = string(container.ImageID)
return daemon.commitImage(c)
}

View file

@ -44,30 +44,6 @@ func (daemon *Daemon) LogContainerEventWithAttributes(container *container.Conta
daemon.EventsService.Log(action, events.ContainerEventType, actor)
}
// LogImageEvent generates an event related to an image with only the default attributes.
func (daemon *Daemon) LogImageEvent(imageID, refName, action string) {
daemon.LogImageEventWithAttributes(imageID, refName, action, map[string]string{})
}
// LogImageEventWithAttributes generates an event related to an image with specific given attributes.
func (daemon *Daemon) LogImageEventWithAttributes(imageID, refName, action string, attributes map[string]string) {
img, err := daemon.GetImage(imageID)
if err == nil && img.Config != nil {
// image has not been removed yet.
// it could be missing if the event is `delete`.
copyAttributes(attributes, img.Config.Labels)
}
if refName != "" {
attributes["name"] = refName
}
actor := events.Actor{
ID: imageID,
Attributes: attributes,
}
daemon.EventsService.Log(action, events.ImageEventType, actor)
}
// LogPluginEvent generates an event related to a plugin with only the default attributes.
func (daemon *Daemon) LogPluginEvent(pluginID, refName, action string) {
daemon.LogPluginEventWithAttributes(pluginID, refName, action, map[string]string{})

124
daemon/image_commit.go Normal file
View file

@ -0,0 +1,124 @@
package daemon // import "github.com/docker/docker/daemon"
import (
"encoding/json"
"io"
"github.com/docker/docker/api/types/backend"
"github.com/docker/docker/image"
"github.com/docker/docker/layer"
"github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/pkg/system"
)
func (daemon *Daemon) commitImage(c backend.CommitConfig) (image.ID, error) {
layerStore, ok := daemon.layerStores[c.ContainerOS]
if !ok {
return "", system.ErrNotSupportedOperatingSystem
}
rwTar, err := exportContainerRw(layerStore, c.ContainerID, c.ContainerMountLabel)
if err != nil {
return "", err
}
defer func() {
if rwTar != nil {
rwTar.Close()
}
}()
var parent *image.Image
if c.ParentImageID == "" {
parent = new(image.Image)
parent.RootFS = image.NewRootFS()
} else {
parent, err = daemon.imageStore.Get(image.ID(c.ParentImageID))
if err != nil {
return "", err
}
}
l, err := layerStore.Register(rwTar, parent.RootFS.ChainID())
if err != nil {
return "", err
}
defer layer.ReleaseAndLog(layerStore, l)
cc := image.ChildConfig{
ContainerID: c.ContainerID,
Author: c.Author,
Comment: c.Comment,
ContainerConfig: c.ContainerConfig,
Config: c.Config,
DiffID: l.DiffID(),
}
config, err := json.Marshal(image.NewChildImage(parent, cc, c.ContainerOS))
if err != nil {
return "", err
}
id, err := daemon.imageStore.Create(config)
if err != nil {
return "", err
}
if c.ParentImageID != "" {
if err := daemon.imageStore.SetParent(id, image.ID(c.ParentImageID)); err != nil {
return "", err
}
}
return id, nil
}
func exportContainerRw(layerStore layer.Store, id, mountLabel string) (arch io.ReadCloser, err error) {
rwlayer, err := layerStore.GetRWLayer(id)
if err != nil {
return nil, err
}
defer func() {
if err != nil {
layerStore.ReleaseRWLayer(rwlayer)
}
}()
// TODO: this mount call is not necessary as we assume that TarStream() should
// mount the layer if needed. But the Diff() function for windows requests that
// the layer should be mounted when calling it. So we reserve this mount call
// until windows driver can implement Diff() interface correctly.
_, err = rwlayer.Mount(mountLabel)
if err != nil {
return nil, err
}
archive, err := rwlayer.TarStream()
if err != nil {
rwlayer.Unmount()
return nil, err
}
return ioutils.NewReadCloserWrapper(archive, func() error {
archive.Close()
err = rwlayer.Unmount()
layerStore.ReleaseRWLayer(rwlayer)
return err
}),
nil
}
// CommitBuildStep is used by the builder to create an image for each step in
// the build.
//
// This method is different from CreateImageFromContainer:
// * it doesn't attempt to validate container state
// * it doesn't send a commit action to metrics
// * it doesn't log a container commit event
//
// This is a temporary shim. Should be removed when builder stops using commit.
func (daemon *Daemon) CommitBuildStep(c backend.CommitConfig) (image.ID, error) {
container, err := daemon.GetContainer(c.ContainerID)
if err != nil {
return "", err
}
c.ContainerMountLabel = container.MountLabel
c.ContainerOS = container.OS
c.ParentImageID = string(container.ImageID)
return daemon.commitImage(c)
}

29
daemon/image_events.go Normal file
View file

@ -0,0 +1,29 @@
package daemon // import "github.com/docker/docker/daemon"
import (
"github.com/docker/docker/api/types/events"
)
// LogImageEvent generates an event related to an image with only the default attributes.
func (daemon *Daemon) LogImageEvent(imageID, refName, action string) {
daemon.LogImageEventWithAttributes(imageID, refName, action, map[string]string{})
}
// LogImageEventWithAttributes generates an event related to an image with specific given attributes.
func (daemon *Daemon) LogImageEventWithAttributes(imageID, refName, action string, attributes map[string]string) {
img, err := daemon.GetImage(imageID)
if err == nil && img.Config != nil {
// image has not been removed yet.
// it could be missing if the event is `delete`.
copyAttributes(attributes, img.Config.Labels)
}
if refName != "" {
attributes["name"] = refName
}
actor := events.Actor{
ID: imageID,
Attributes: attributes,
}
daemon.EventsService.Log(action, events.ImageEventType, actor)
}

168
daemon/image_prune.go Normal file
View file

@ -0,0 +1,168 @@
package daemon // import "github.com/docker/docker/daemon"
import (
"sync/atomic"
"github.com/docker/distribution/reference"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/errdefs"
"github.com/docker/docker/image"
"github.com/docker/docker/layer"
digest "github.com/opencontainers/go-digest"
"github.com/sirupsen/logrus"
"golang.org/x/net/context"
)
var imagesAcceptedFilters = map[string]bool{
"dangling": true,
"label": true,
"label!": true,
"until": true,
}
// ImagesPrune removes unused images
func (daemon *Daemon) ImagesPrune(ctx context.Context, pruneFilters filters.Args) (*types.ImagesPruneReport, error) {
if !atomic.CompareAndSwapInt32(&daemon.pruneRunning, 0, 1) {
return nil, errPruneRunning
}
defer atomic.StoreInt32(&daemon.pruneRunning, 0)
// make sure that only accepted filters have been received
err := pruneFilters.Validate(imagesAcceptedFilters)
if err != nil {
return nil, err
}
rep := &types.ImagesPruneReport{}
danglingOnly := true
if pruneFilters.Contains("dangling") {
if pruneFilters.ExactMatch("dangling", "false") || pruneFilters.ExactMatch("dangling", "0") {
danglingOnly = false
} else if !pruneFilters.ExactMatch("dangling", "true") && !pruneFilters.ExactMatch("dangling", "1") {
return nil, invalidFilter{"dangling", pruneFilters.Get("dangling")}
}
}
until, err := getUntilFromPruneFilters(pruneFilters)
if err != nil {
return nil, err
}
var allImages map[image.ID]*image.Image
if danglingOnly {
allImages = daemon.imageStore.Heads()
} else {
allImages = daemon.imageStore.Map()
}
// Filter intermediary images and get their unique size
allLayers := make(map[layer.ChainID]layer.Layer)
for _, ls := range daemon.layerStores {
for k, v := range ls.Map() {
allLayers[k] = v
}
}
topImages := map[image.ID]*image.Image{}
for id, img := range allImages {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
dgst := digest.Digest(id)
if len(daemon.referenceStore.References(dgst)) == 0 && len(daemon.imageStore.Children(id)) != 0 {
continue
}
if !until.IsZero() && img.Created.After(until) {
continue
}
if img.Config != nil && !matchLabels(pruneFilters, img.Config.Labels) {
continue
}
topImages[id] = img
}
}
canceled := false
deleteImagesLoop:
for id := range topImages {
select {
case <-ctx.Done():
// we still want to calculate freed size and return the data
canceled = true
break deleteImagesLoop
default:
}
deletedImages := []types.ImageDeleteResponseItem{}
refs := daemon.referenceStore.References(id.Digest())
if len(refs) > 0 {
shouldDelete := !danglingOnly
if !shouldDelete {
hasTag := false
for _, ref := range refs {
if _, ok := ref.(reference.NamedTagged); ok {
hasTag = true
break
}
}
// Only delete if it's untagged (i.e. repo:<none>)
shouldDelete = !hasTag
}
if shouldDelete {
for _, ref := range refs {
imgDel, err := daemon.ImageDelete(ref.String(), false, true)
if imageDeleteFailed(ref.String(), err) {
continue
}
deletedImages = append(deletedImages, imgDel...)
}
}
} else {
hex := id.Digest().Hex()
imgDel, err := daemon.ImageDelete(hex, false, true)
if imageDeleteFailed(hex, err) {
continue
}
deletedImages = append(deletedImages, imgDel...)
}
rep.ImagesDeleted = append(rep.ImagesDeleted, deletedImages...)
}
// Compute how much space was freed
for _, d := range rep.ImagesDeleted {
if d.Deleted != "" {
chid := layer.ChainID(d.Deleted)
if l, ok := allLayers[chid]; ok {
diffSize, err := l.DiffSize()
if err != nil {
logrus.Warnf("failed to get layer %s size: %v", chid, err)
continue
}
rep.SpaceReclaimed += uint64(diffSize)
}
}
}
if canceled {
logrus.Debugf("ImagesPrune operation cancelled: %#v", *rep)
}
return rep, nil
}
func imageDeleteFailed(ref string, err error) bool {
switch {
case err == nil:
return false
case errdefs.IsConflict(err):
return true
default:
logrus.Warnf("failed to prune image %s: %v", ref, err)
return true
}
}

35
daemon/image_windows.go Normal file
View file

@ -0,0 +1,35 @@
package daemon // import "github.com/docker/docker/daemon"
import (
"github.com/docker/docker/image"
"github.com/docker/docker/layer"
"github.com/docker/docker/pkg/system"
"github.com/pkg/errors"
)
// GetLayerFolders returns the layer folders from an image RootFS
func (daemon *Daemon) GetLayerFolders(img *image.Image, rwLayer layer.RWLayer) ([]string, error) {
folders := []string{}
max := len(img.RootFS.DiffIDs)
for index := 1; index <= max; index++ {
// FIXME: why does this mutate the RootFS?
img.RootFS.DiffIDs = img.RootFS.DiffIDs[:index]
if !system.IsOSSupported(img.OperatingSystem()) {
return nil, errors.Wrapf(system.ErrNotSupportedOperatingSystem, "cannot get layerpath for ImageID %s", img.RootFS.ChainID())
}
layerPath, err := layer.GetLayerPath(daemon.layerStores[img.OperatingSystem()], img.RootFS.ChainID())
if err != nil {
return nil, errors.Wrapf(err, "failed to get layer path from graphdriver %s for ImageID %s", daemon.layerStores[img.OperatingSystem()], img.RootFS.ChainID())
}
// Reverse order, expecting parent first
folders = append([]string{layerPath}, folders...)
}
if rwLayer == nil {
return nil, errors.New("RWLayer is unexpectedly nil")
}
m, err := rwLayer.Metadata()
if err != nil {
return nil, errors.Wrap(err, "failed to get layer metadata")
}
return append(folders, m["dir"]), nil
}

View file

@ -1,7 +1,6 @@
package daemon // import "github.com/docker/docker/daemon"
import (
"errors"
"fmt"
"io/ioutil"
"path/filepath"
@ -10,11 +9,11 @@ import (
containertypes "github.com/docker/docker/api/types/container"
"github.com/docker/docker/container"
"github.com/docker/docker/layer"
"github.com/docker/docker/oci"
"github.com/docker/docker/pkg/sysinfo"
"github.com/docker/docker/pkg/system"
"github.com/opencontainers/runtime-spec/specs-go"
"github.com/pkg/errors"
"golang.org/x/sys/windows"
"golang.org/x/sys/windows/registry"
)
@ -139,29 +138,10 @@ func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) {
}
}
s.Process.User.Username = c.Config.User
// Get the layer path for each layer.
max := len(img.RootFS.DiffIDs)
for i := 1; i <= max; i++ {
img.RootFS.DiffIDs = img.RootFS.DiffIDs[:i]
if !system.IsOSSupported(img.OperatingSystem()) {
return nil, fmt.Errorf("cannot get layerpath for ImageID %s: %s ", img.RootFS.ChainID(), system.ErrNotSupportedOperatingSystem)
}
layerPath, err := layer.GetLayerPath(daemon.layerStores[img.OperatingSystem()], img.RootFS.ChainID())
if err != nil {
return nil, fmt.Errorf("failed to get layer path from graphdriver %s for ImageID %s - %s", daemon.layerStores[img.OperatingSystem()], img.RootFS.ChainID(), err)
}
// Reverse order, expecting parent most first
s.Windows.LayerFolders = append([]string{layerPath}, s.Windows.LayerFolders...)
}
if c.RWLayer == nil {
return nil, errors.New("RWLayer of container " + c.ID + " is unexpectedly nil")
}
m, err := c.RWLayer.Metadata()
s.Windows.LayerFolders, err = daemon.GetLayerFolders(img, c.RWLayer)
if err != nil {
return nil, fmt.Errorf("failed to get layer metadata - %s", err)
return nil, errors.Wrapf(err, "container %s", c.ID)
}
s.Windows.LayerFolders = append(s.Windows.LayerFolders, m["dir"])
dnsSearch := daemon.getDNSSearchSettings(c)

View file

@ -6,18 +6,13 @@ import (
"sync/atomic"
"time"
"github.com/docker/distribution/reference"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
timetypes "github.com/docker/docker/api/types/time"
"github.com/docker/docker/errdefs"
"github.com/docker/docker/image"
"github.com/docker/docker/layer"
"github.com/docker/docker/pkg/directory"
"github.com/docker/docker/runconfig"
"github.com/docker/docker/volume"
"github.com/docker/libnetwork"
digest "github.com/opencontainers/go-digest"
"github.com/sirupsen/logrus"
"golang.org/x/net/context"
)
@ -36,12 +31,7 @@ var (
"label": true,
"label!": true,
}
imagesAcceptedFilters = map[string]bool{
"dangling": true,
"label": true,
"label!": true,
"until": true,
}
networksAcceptedFilters = map[string]bool{
"label": true,
"label!": true,
@ -159,152 +149,6 @@ func (daemon *Daemon) VolumesPrune(ctx context.Context, pruneFilters filters.Arg
return rep, err
}
// ImagesPrune removes unused images
func (daemon *Daemon) ImagesPrune(ctx context.Context, pruneFilters filters.Args) (*types.ImagesPruneReport, error) {
if !atomic.CompareAndSwapInt32(&daemon.pruneRunning, 0, 1) {
return nil, errPruneRunning
}
defer atomic.StoreInt32(&daemon.pruneRunning, 0)
// make sure that only accepted filters have been received
err := pruneFilters.Validate(imagesAcceptedFilters)
if err != nil {
return nil, err
}
rep := &types.ImagesPruneReport{}
danglingOnly := true
if pruneFilters.Contains("dangling") {
if pruneFilters.ExactMatch("dangling", "false") || pruneFilters.ExactMatch("dangling", "0") {
danglingOnly = false
} else if !pruneFilters.ExactMatch("dangling", "true") && !pruneFilters.ExactMatch("dangling", "1") {
return nil, invalidFilter{"dangling", pruneFilters.Get("dangling")}
}
}
until, err := getUntilFromPruneFilters(pruneFilters)
if err != nil {
return nil, err
}
var allImages map[image.ID]*image.Image
if danglingOnly {
allImages = daemon.imageStore.Heads()
} else {
allImages = daemon.imageStore.Map()
}
// Filter intermediary images and get their unique size
allLayers := make(map[layer.ChainID]layer.Layer)
for _, ls := range daemon.layerStores {
for k, v := range ls.Map() {
allLayers[k] = v
}
}
topImages := map[image.ID]*image.Image{}
for id, img := range allImages {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
dgst := digest.Digest(id)
if len(daemon.referenceStore.References(dgst)) == 0 && len(daemon.imageStore.Children(id)) != 0 {
continue
}
if !until.IsZero() && img.Created.After(until) {
continue
}
if img.Config != nil && !matchLabels(pruneFilters, img.Config.Labels) {
continue
}
topImages[id] = img
}
}
canceled := false
deleteImagesLoop:
for id := range topImages {
select {
case <-ctx.Done():
// we still want to calculate freed size and return the data
canceled = true
break deleteImagesLoop
default:
}
deletedImages := []types.ImageDeleteResponseItem{}
refs := daemon.referenceStore.References(id.Digest())
if len(refs) > 0 {
shouldDelete := !danglingOnly
if !shouldDelete {
hasTag := false
for _, ref := range refs {
if _, ok := ref.(reference.NamedTagged); ok {
hasTag = true
break
}
}
// Only delete if it's untagged (i.e. repo:<none>)
shouldDelete = !hasTag
}
if shouldDelete {
for _, ref := range refs {
imgDel, err := daemon.ImageDelete(ref.String(), false, true)
if imageDeleteFailed(ref.String(), err) {
continue
}
deletedImages = append(deletedImages, imgDel...)
}
}
} else {
hex := id.Digest().Hex()
imgDel, err := daemon.ImageDelete(hex, false, true)
if imageDeleteFailed(hex, err) {
continue
}
deletedImages = append(deletedImages, imgDel...)
}
rep.ImagesDeleted = append(rep.ImagesDeleted, deletedImages...)
}
// Compute how much space was freed
for _, d := range rep.ImagesDeleted {
if d.Deleted != "" {
chid := layer.ChainID(d.Deleted)
if l, ok := allLayers[chid]; ok {
diffSize, err := l.DiffSize()
if err != nil {
logrus.Warnf("failed to get layer %s size: %v", chid, err)
continue
}
rep.SpaceReclaimed += uint64(diffSize)
}
}
}
if canceled {
logrus.Debugf("ImagesPrune operation cancelled: %#v", *rep)
}
return rep, nil
}
func imageDeleteFailed(ref string, err error) bool {
switch {
case err == nil:
return false
case errdefs.IsConflict(err):
return true
default:
logrus.Warnf("failed to prune image %s: %v", ref, err)
return true
}
}
// localNetworksPrune removes unused local networks
func (daemon *Daemon) localNetworksPrune(ctx context.Context, pruneFilters filters.Args) *types.NetworksPruneReport {
rep := &types.NetworksPruneReport{}