0ct0pu5/moby
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

LCOW: Refactor to multiple layer-stores based on feedback

Browse source 
Signed-off-by: John Howard <jhoward@microsoft.com>
This commit is contained in:
John Howard 2017-09-19 12:14:46 -07:00
parent ce8e529e18
commit afd305c4b5
54 changed files with 373 additions and 376 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
builder/builder.go
View file

@ -102,6 +102,6 @@ type Image interface {
type ReleaseableLayer interface {
Release() error
Mount() (containerfs.ContainerFS, error)
Commit(platform string) (ReleaseableLayer, error)
Commit() (ReleaseableLayer, error)
DiffID() layer.DiffID
}

3
builder/dockerfile/builder.go
View file

@ -357,7 +357,7 @@ func addNodesForLabelOption(dockerfile *parser.Node, labels map[string]string) {
// coming from the query parameter of the same name.
//
// TODO: Remove?
func BuildFromConfig(config *container.Config, changes []string) (*container.Config, error) {
func BuildFromConfig(config *container.Config, changes []string, os string) (*container.Config, error) {
if len(changes) == 0 {
return config, nil
}
@ -395,6 +395,7 @@ func BuildFromConfig(config *container.Config, changes []string) (*container.Con
// We make mutations to the configuration, ensure we have a copy
dispatchRequest.state.runConfig = copyRunConfig(config)
dispatchRequest.state.imageID = config.Image
dispatchRequest.state.operatingSystem = os
for _, cmd := range commands {
err := dispatch(dispatchRequest, cmd)
if err != nil {

2
builder/dockerfile/containerbackend.go
View file

@ -28,7 +28,7 @@ func newContainerManager(docker builder.ExecBackend) *containerManager {
}
// Create a container
func (c *containerManager) Create(runConfig *container.Config, hostConfig *container.HostConfig, platform string) (container.ContainerCreateCreatedBody, error) {
func (c *containerManager) Create(runConfig *container.Config, hostConfig *container.HostConfig) (container.ContainerCreateCreatedBody, error) {
container, err := c.backend.ContainerCreate(types.ContainerCreateConfig{
Config: runConfig,
HostConfig: hostConfig,

13
builder/dockerfile/dispatchers.go
View file

@ -261,8 +261,8 @@ func dispatchOnbuild(d dispatchRequest, c *instructions.OnbuildCommand) error {
func dispatchWorkdir(d dispatchRequest, c *instructions.WorkdirCommand) error {
runConfig := d.state.runConfig
var err error
optionsOS := system.ParsePlatform(d.builder.options.Platform).OS
runConfig.WorkingDir, err = normalizeWorkdir(optionsOS, runConfig.WorkingDir, c.Path)
baseImageOS := system.ParsePlatform(d.state.operatingSystem).OS
runConfig.WorkingDir, err = normalizeWorkdir(baseImageOS, runConfig.WorkingDir, c.Path)
if err != nil {
return err
}
@ -278,7 +278,7 @@ func dispatchWorkdir(d dispatchRequest, c *instructions.WorkdirCommand) error {
}
comment := "WORKDIR " + runConfig.WorkingDir
runConfigWithCommentCmd := copyRunConfig(runConfig, withCmdCommentString(comment, optionsOS))
runConfigWithCommentCmd := copyRunConfig(runConfig, withCmdCommentString(comment, baseImageOS))
containerID, err := d.builder.probeAndCreate(d.state, runConfigWithCommentCmd)
if err != nil || containerID == "" {
return err
@ -290,10 +290,10 @@ func dispatchWorkdir(d dispatchRequest, c *instructions.WorkdirCommand) error {
return d.builder.commitContainer(d.state, containerID, runConfigWithCommentCmd)
}
func resolveCmdLine(cmd instructions.ShellDependantCmdLine, runConfig *container.Config, platform string) []string {
func resolveCmdLine(cmd instructions.ShellDependantCmdLine, runConfig *container.Config, os string) []string {
result := cmd.CmdLine
if cmd.PrependShell && result != nil {
result = append(getShell(runConfig, platform), result...)
result = append(getShell(runConfig, os), result...)
}
return result
}
@ -311,8 +311,7 @@ func resolveCmdLine(cmd instructions.ShellDependantCmdLine, runConfig *container
func dispatchRun(d dispatchRequest, c *instructions.RunCommand) error {
stateRunConfig := d.state.runConfig
optionsOS := system.ParsePlatform(d.builder.options.Platform).OS
cmdFromArgs := resolveCmdLine(c.ShellDependantCmdLine, stateRunConfig, optionsOS)
cmdFromArgs := resolveCmdLine(c.ShellDependantCmdLine, stateRunConfig, d.state.operatingSystem)
buildArgs := d.state.buildArgs.FilterAllowed(stateRunConfig.Env)
saveCmd := cmdFromArgs

18
builder/dockerfile/evaluator.go
View file

@ -104,13 +104,14 @@ func dispatch(d dispatchRequest, cmd instructions.Command) (err error) {
// dispatchState is a data object which is modified by dispatchers
type dispatchState struct {
runConfig *container.Config
maintainer string
cmdSet bool
imageID string
baseImage builder.Image
stageName string
buildArgs *buildArgs
runConfig *container.Config
maintainer string
cmdSet bool
imageID string
baseImage builder.Image
stageName string
buildArgs *buildArgs
operatingSystem string
}
func newDispatchState(baseArgs *buildArgs) *dispatchState {
@ -213,6 +214,7 @@ func (s *dispatchState) hasFromImage() bool {
func (s *dispatchState) beginStage(stageName string, image builder.Image) {
s.stageName = stageName
s.imageID = image.ImageID()
s.operatingSystem = image.OperatingSystem()
if image.RunConfig() != nil {
// copy avoids referencing the same instance when 2 stages have the same base
@ -229,7 +231,7 @@ func (s *dispatchState) beginStage(stageName string, image builder.Image) {
// Add the default PATH to runConfig.ENV if one exists for the operating system and there
// is no PATH set. Note that Windows containers on Windows won't have one as it's set by HCS
func (s *dispatchState) setDefaultPath() {
defaultPath := system.DefaultPathEnv(s.baseImage.OperatingSystem())
defaultPath := system.DefaultPathEnv(s.operatingSystem)
if defaultPath == "" {
return
}

9
builder/dockerfile/internals.go
View file

@ -125,8 +125,7 @@ func (b *Builder) commitContainer(dispatchState *dispatchState, id string, conta
}
func (b *Builder) exportImage(state *dispatchState, imageMount *imageMount, runConfig *container.Config) error {
optionsPlatform := system.ParsePlatform(b.options.Platform)
newLayer, err := imageMount.Layer().Commit(optionsPlatform.OS)
newLayer, err := imageMount.Layer().Commit()
if err != nil {
return err
}
@ -502,15 +501,13 @@ func (b *Builder) probeAndCreate(dispatchState *dispatchState, runConfig *contai
}
// Set a log config to override any default value set on the daemon
hostConfig := &container.HostConfig{LogConfig: defaultLogConfig}
optionsPlatform := system.ParsePlatform(b.options.Platform)
container, err := b.containerManager.Create(runConfig, hostConfig, optionsPlatform.OS)
container, err := b.containerManager.Create(runConfig, hostConfig)
return container.ID, err
}
func (b *Builder) create(runConfig *container.Config) (string, error) {
hostConfig := hostConfigFromOptions(b.options)
optionsPlatform := system.ParsePlatform(b.options.Platform)
container, err := b.containerManager.Create(runConfig, hostConfig, optionsPlatform.OS)
container, err := b.containerManager.Create(runConfig, hostConfig)
if err != nil {
return "", err
}

2
builder/dockerfile/mockbackend_test.go
View file

@ -127,7 +127,7 @@ func (l *mockLayer) Mount() (containerfs.ContainerFS, error) {
return containerfs.NewLocalContainerFS("mountPath"), nil
}
func (l *mockLayer) Commit(string) (builder.ReleaseableLayer, error) {
func (l *mockLayer) Commit() (builder.ReleaseableLayer, error) {
return nil, nil
}

21
daemon/build.go
View file

@ -23,7 +23,6 @@ type releaseableLayer struct {
layerStore layer.Store
roLayer layer.Layer
rwLayer layer.RWLayer
os string
}
func (rl *releaseableLayer) Mount() (containerfs.ContainerFS, error) {
@ -35,7 +34,7 @@ func (rl *releaseableLayer) Mount() (containerfs.ContainerFS, error) {
}
mountID := stringid.GenerateRandomID()
rl.rwLayer, err = rl.layerStore.CreateRWLayer(mountID, chainID, rl.os, nil)
rl.rwLayer, err = rl.layerStore.CreateRWLayer(mountID, chainID, nil)
if err != nil {
return nil, errors.Wrap(err, "failed to create rwlayer")
}
@ -55,7 +54,7 @@ func (rl *releaseableLayer) Mount() (containerfs.ContainerFS, error) {
return mountPath, nil
}
func (rl *releaseableLayer) Commit(os string) (builder.ReleaseableLayer, error) {
func (rl *releaseableLayer) Commit() (builder.ReleaseableLayer, error) {
var chainID layer.ChainID
if rl.roLayer != nil {
chainID = rl.roLayer.ChainID()
@ -67,12 +66,12 @@ func (rl *releaseableLayer) Commit(os string) (builder.ReleaseableLayer, error)
}
defer stream.Close()
newLayer, err := rl.layerStore.Register(stream, chainID, os)
newLayer, err := rl.layerStore.Register(stream, chainID)
if err != nil {
return nil, err
}
// TODO: An optimization woudld be to handle empty layers before returning
return &releaseableLayer{layerStore: rl.layerStore, roLayer: newLayer, os: os}, nil
return &releaseableLayer{layerStore: rl.layerStore, roLayer: newLayer}, nil
}
func (rl *releaseableLayer) DiffID() layer.DiffID {
@ -128,9 +127,9 @@ func (rl *releaseableLayer) releaseROLayer() error {
return err
}
func newReleasableLayerForImage(img *image.Image, layerStore layer.Store, os string) (builder.ReleaseableLayer, error) {
func newReleasableLayerForImage(img *image.Image, layerStore layer.Store) (builder.ReleaseableLayer, error) {
if img == nil || img.RootFS.ChainID() == "" {
return &releaseableLayer{layerStore: layerStore, os: os}, nil
return &releaseableLayer{layerStore: layerStore}, nil
}
// Hold a reference to the image layer so that it can't be removed before
// it is released
@ -138,7 +137,7 @@ func newReleasableLayerForImage(img *image.Image, layerStore layer.Store, os str
if err != nil {
return nil, errors.Wrapf(err, "failed to get layer for image %s", img.ImageID())
}
return &releaseableLayer{layerStore: layerStore, roLayer: roLayer, os: os}, nil
return &releaseableLayer{layerStore: layerStore, roLayer: roLayer}, nil
}
// TODO: could this use the regular daemon PullImage ?
@ -172,7 +171,7 @@ func (daemon *Daemon) pullForBuilder(ctx context.Context, name string, authConfi
// leaking of layers.
func (daemon *Daemon) GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ReleaseableLayer, error) {
if refOrID == "" {
layer, err := newReleasableLayerForImage(nil, daemon.layerStore, opts.OS)
layer, err := newReleasableLayerForImage(nil, daemon.layerStores[opts.OS])
return nil, layer, err
}
@ -183,7 +182,7 @@ func (daemon *Daemon) GetImageAndReleasableLayer(ctx context.Context, refOrID st
}
// TODO: shouldn't we error out if error is different from "not found" ?
if image != nil {
layer, err := newReleasableLayerForImage(image, daemon.layerStore, image.OperatingSystem())
layer, err := newReleasableLayerForImage(image, daemon.layerStores[image.OperatingSystem()])
return image, layer, err
}
}
@ -192,7 +191,7 @@ func (daemon *Daemon) GetImageAndReleasableLayer(ctx context.Context, refOrID st
if err != nil {
return nil, nil, err
}
layer, err := newReleasableLayerForImage(image, daemon.layerStore, image.OperatingSystem())
layer, err := newReleasableLayerForImage(image, daemon.layerStores[image.OperatingSystem()])
return image, layer, err
}

12
daemon/commit.go
View file

@ -154,7 +154,7 @@ func (daemon *Daemon) Commit(name string, c *backend.ContainerCommitConfig) (str
c.Config = container.Config
}
newConfig, err := dockerfile.BuildFromConfig(c.Config, c.Changes)
newConfig, err := dockerfile.BuildFromConfig(c.Config, c.Changes, container.OS)
if err != nil {
return "", err
}
@ -186,11 +186,11 @@ func (daemon *Daemon) Commit(name string, c *backend.ContainerCommitConfig) (str
}
}
l, err := daemon.layerStore.Register(rwTar, parent.RootFS.ChainID(), container.OS)
l, err := daemon.layerStores[container.OS].Register(rwTar, parent.RootFS.ChainID())
if err != nil {
return "", err
}
defer layer.ReleaseAndLog(daemon.layerStore, l)
defer layer.ReleaseAndLog(daemon.layerStores[container.OS], l)
containerConfig := c.ContainerConfig
if containerConfig == nil {
@ -251,13 +251,13 @@ func (daemon *Daemon) Commit(name string, c *backend.ContainerCommitConfig) (str
}
func (daemon *Daemon) exportContainerRw(container *container.Container) (arch io.ReadCloser, err error) {
rwlayer, err := daemon.stores[container.OS].layerStore.GetRWLayer(container.ID)
rwlayer, err := daemon.layerStores[container.OS].GetRWLayer(container.ID)
if err != nil {
return nil, err
}
defer func() {
if err != nil {
daemon.stores[container.OS].layerStore.ReleaseRWLayer(rwlayer)
daemon.layerStores[container.OS].ReleaseRWLayer(rwlayer)
}
}()
@ -278,7 +278,7 @@ func (daemon *Daemon) exportContainerRw(container *container.Container) (arch io
return ioutils.NewReadCloserWrapper(archive, func() error {
archive.Close()
err = rwlayer.Unmount()
daemon.stores[container.OS].layerStore.ReleaseRWLayer(rwlayer)
daemon.layerStores[container.OS].ReleaseRWLayer(rwlayer)
return err
}),
nil

2
daemon/create.go
View file

@ -270,7 +270,7 @@ func (daemon *Daemon) setRWLayer(container *container.Container) error {
StorageOpt: container.HostConfig.StorageOpt,
}
rwLayer, err := daemon.layerStore.CreateRWLayer(container.ID, layerID, container.OS, rwLayerOpts)
rwLayer, err := daemon.layerStores[container.OS].CreateRWLayer(container.ID, layerID, rwLayerOpts)
if err != nil {
return err
}

57
daemon/daemon.go
View file

@ -97,7 +97,7 @@ type Daemon struct {
referenceStore refstore.Store
imageStore image.Store
imageRoot string
layerStore layer.Store
layerStores map[string]layer.Store // By operating system
distributionMetadataStore dmetadata.Store
PluginStore *plugin.Store // todo: remove
pluginManager *plugin.Manager
@ -159,7 +159,7 @@ func (daemon *Daemon) restore() error {
// Ignore the container if it does not support the current driver being used by the graph
currentDriverForContainerOS := daemon.graphDrivers[container.OS]
if (container.Driver == "" && currentDriverForContainerOS == "aufs") || container.Driver == currentDriverForContainerOS {
rwlayer, err := daemon.layerStore.GetRWLayer(container.ID)
rwlayer, err := daemon.layerStores[container.OS].GetRWLayer(container.ID)
if err != nil {
logrus.Errorf("Failed to load container mount %v: %v", id, err)
continue
@ -703,6 +703,7 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe
// be set through an environment variable, a daemon start parameter, or chosen through
// initialization of the layerstore through driver priority order for example.
d.graphDrivers = make(map[string]string)
d.layerStores = make(map[string]layer.Store)
if runtime.GOOS == "windows" {
d.graphDrivers[runtime.GOOS] = "windowsfilter"
if system.LCOWSupported() {
@ -746,22 +747,25 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe
return nil, errors.Wrap(err, "couldn't create plugin manager")
}
d.layerStore, err = layer.NewStoreFromOptions(layer.StoreOptions{
Root: config.Root,
MetadataStorePathTemplate: filepath.Join(config.Root, "image", "%s", "layerdb"),
GraphDrivers: d.graphDrivers,
GraphDriverOptions: config.GraphOptions,
IDMappings: idMappings,
PluginGetter: d.PluginStore,
ExperimentalEnabled: config.Experimental,
})
if err != nil {
return nil, err
for operatingSystem, gd := range d.graphDrivers {
d.layerStores[operatingSystem], err = layer.NewStoreFromOptions(layer.StoreOptions{
Root: config.Root,
MetadataStorePathTemplate: filepath.Join(config.Root, "image", "%s", "layerdb"),
GraphDriver: gd,
GraphDriverOptions: config.GraphOptions,
IDMappings: idMappings,
PluginGetter: d.PluginStore,
ExperimentalEnabled: config.Experimental,
OS: operatingSystem,
})
if err != nil {
return nil, err
}
}
// As layerstore may set the driver
// As layerstore initialization may set the driver
for os := range d.graphDrivers {
d.graphDrivers[os] = d.layerStore.DriverName(os)
d.graphDrivers[os] = d.layerStores[os].DriverName()
}
// Configure and validate the kernels security support. Note this is a Linux/FreeBSD
@ -771,7 +775,7 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe
}
logrus.Debugf("Max Concurrent Downloads: %d", *config.MaxConcurrentDownloads)
d.downloadManager = xfer.NewLayerDownloadManager(d.layerStore, *config.MaxConcurrentDownloads)
d.downloadManager = xfer.NewLayerDownloadManager(d.layerStores, *config.MaxConcurrentDownloads)
logrus.Debugf("Max Concurrent Uploads: %d", *config.MaxConcurrentUploads)
d.uploadManager = xfer.NewLayerUploadManager(*config.MaxConcurrentUploads)
@ -780,7 +784,12 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe
if err != nil {
return nil, err
}
d.imageStore, err = image.NewImageStore(ifs, d.layerStore)
lgrMap := make(map[string]image.LayerGetReleaser)
for os, ls := range d.layerStores {
lgrMap[os] = ls
}
d.imageStore, err = image.NewImageStore(ifs, lgrMap)
if err != nil {
return nil, err
}
@ -829,7 +838,7 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe
// No content-addressability migration on Windows as it never supported pre-CA
if runtime.GOOS != "windows" {
migrationStart := time.Now()
if err := v1.Migrate(config.Root, d.graphDrivers[runtime.GOOS], d.layerStore, d.imageStore, rs, d.distributionMetadataStore); err != nil {
if err := v1.Migrate(config.Root, d.graphDrivers[runtime.GOOS], d.layerStores[runtime.GOOS], d.imageStore, rs, d.distributionMetadataStore); err != nil {
logrus.Errorf("Graph migration failed: %q. Your old graph data was found to be too inconsistent for upgrading to content-addressable storage. Some of the old data was probably not upgraded. We recommend starting over with a clean storage directory if possible.", err)
}
logrus.Infof("Graph migration to content-addressability took %.2f seconds", time.Since(migrationStart).Seconds())
@ -988,7 +997,7 @@ func (daemon *Daemon) Shutdown() error {
logrus.Errorf("Stop container error: %v", err)
return
}
if mountid, err := daemon.layerStore.GetMountID(c.ID); err == nil {
if mountid, err := daemon.layerStores[c.OS].GetMountID(c.ID); err == nil {
daemon.cleanupMountsByID(mountid)
}
logrus.Debugf("container stopped %s", c.ID)
@ -1001,8 +1010,12 @@ func (daemon *Daemon) Shutdown() error {
}
}
if err := daemon.layerStore.Cleanup(); err != nil {
logrus.Errorf("Error during layer Store.Cleanup(): %v", err)
for os, ls := range daemon.layerStores {
if ls != nil {
if err := ls.Cleanup(); err != nil {
logrus.Errorf("Error during layer Store.Cleanup(): %v %s", err, os)
}
}
}
// If we are part of a cluster, clean up cluster's stuff
@ -1083,7 +1096,7 @@ func (daemon *Daemon) Subnets() ([]net.IPNet, []net.IPNet) {
// GraphDriverName returns the name of the graph driver used by the layer.Store
func (daemon *Daemon) GraphDriverName(os string) string {
return daemon.layerStore.DriverName(os)
return daemon.layerStores[os].DriverName()
}
// prepareTempDir prepares and returns the default directory to use

2
daemon/delete.go
View file

@ -118,7 +118,7 @@ func (daemon *Daemon) cleanupContainer(container *container.Container, forceRemo
// When container creation fails and `RWLayer` has not been created yet, we
// do not call `ReleaseRWLayer`
if container.RWLayer != nil {
metadata, err := daemon.layerStore.ReleaseRWLayer(container.RWLayer)
metadata, err := daemon.layerStores[container.OS].ReleaseRWLayer(container.RWLayer)
layer.LogReleaseMetadata(metadata)
if err != nil && err != layer.ErrMountDoesNotExist && !os.IsNotExist(errors.Cause(err)) {
e := errors.Wrapf(err, "driver %q failed to remove root filesystem for %s", daemon.GraphDriverName(container.OS), container.ID)

28
daemon/disk_usage.go
View file

@ -96,21 +96,23 @@ func (daemon *Daemon) SystemDiskUsage(ctx context.Context) (*types.DiskUsage, er
// Get total layers size on disk
var allLayersSize int64
layerRefs := daemon.getLayerRefs()
allLayers := daemon.layerStore.Map()
for _, l := range allLayers {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
size, err := l.DiffSize()
if err == nil {
if _, ok := layerRefs[l.ChainID()]; ok {
allLayersSize += size
for _, ls := range daemon.layerStores {
allLayers := ls.Map()
for _, l := range allLayers {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
size, err := l.DiffSize()
if err == nil {
if _, ok := layerRefs[l.ChainID()]; ok {
allLayersSize += size
} else {
logrus.Warnf("found leaked image layer %v", l.ChainID())
}
} else {
logrus.Warnf("found leaked image layer %v", l.ChainID())
logrus.Warnf("failed to get diff size for layer %v", l.ChainID())
}
} else {
logrus.Warnf("failed to get diff size for layer %v", l.ChainID())
}
}
}

6
daemon/export.go
View file

@ -47,13 +47,13 @@ func (daemon *Daemon) ContainerExport(name string, out io.Writer) error {
}
func (daemon *Daemon) containerExport(container *container.Container) (arch io.ReadCloser, err error) {
rwlayer, err := daemon.stores[container.OS].layerStore.GetRWLayer(container.ID)
rwlayer, err := daemon.layerStores[container.OS].GetRWLayer(container.ID)
if err != nil {
return nil, err
}
defer func() {
if err != nil {
daemon.stores[container.OS].layerStore.ReleaseRWLayer(rwlayer)
daemon.layerStores[container.OS].ReleaseRWLayer(rwlayer)
}
}()
@ -74,7 +74,7 @@ func (daemon *Daemon) containerExport(container *container.Container) (arch io.R
arch = ioutils.NewReadCloserWrapper(archive, func() error {
err := archive.Close()
rwlayer.Unmount()
daemon.stores[container.OS].layerStore.ReleaseRWLayer(rwlayer)
daemon.layerStores[container.OS].ReleaseRWLayer(rwlayer)
return err
})
daemon.LogContainerEvent(container, "export")

6
daemon/getsize_unix.go
View file

@ -15,12 +15,14 @@ func (daemon *Daemon) getSize(containerID string) (int64, int64) {
err error
)
rwlayer, err := daemon.layerStore.GetRWLayer(containerID)
// Safe to index by runtime.GOOS as Unix hosts don't support multiple
// container operating systems.
rwlayer, err := daemon.layerStores[runtime.GOOS].GetRWLayer(containerID)
if err != nil {
logrus.Errorf("Failed to compute size of container rootfs %v: %v", containerID, err)
return sizeRw, sizeRootfs
}
defer daemon.layerStore.ReleaseRWLayer(rwlayer)
defer daemon.layerStores[runtime.GOOS].ReleaseRWLayer(rwlayer)
sizeRw, err = rwlayer.Size()
if err != nil {

2
daemon/graphdriver/windows/windows.go
View file

@ -931,8 +931,6 @@ func parseStorageOpt(storageOpt map[string]string) (*storageOptions, error) {
return nil, err
}
options.size = uint64(size)
default:
return nil, fmt.Errorf("Unknown storage option: %s", key)
}
}
return &options, nil

7
daemon/image.go
View file

@ -2,7 +2,6 @@ package daemon
import (
"fmt"
"runtime"
"github.com/docker/distribution/reference"
"github.com/docker/docker/errdefs"
@ -58,11 +57,7 @@ func (daemon *Daemon) GetImageIDAndOS(refOrID string) (image.ID, string, error)
if err != nil {
return "", "", errImageDoesNotExist{ref}
}
imageOS := img.OperatingSystem()
if imageOS == "" {
imageOS = runtime.GOOS
}
return id, imageOS, nil
return id, img.OperatingSystem(), nil
}
return "", "", errImageDoesNotExist{ref}

4
daemon/image_exporter.go
View file

@ -12,7 +12,7 @@ import (
// the same tag are exported. names is the set of tags to export, and
// outStream is the writer which the images are written to.
func (daemon *Daemon) ExportImage(names []string, outStream io.Writer) error {
imageExporter := tarexport.NewTarExporter(daemon.imageStore, daemon.layerStore, daemon.referenceStore, daemon)
imageExporter := tarexport.NewTarExporter(daemon.imageStore, daemon.layerStores, daemon.referenceStore, daemon)
return imageExporter.Save(names, outStream)
}
@ -20,6 +20,6 @@ func (daemon *Daemon) ExportImage(names []string, outStream io.Writer) error {
// complement of ImageExport. The input stream is an uncompressed tar
// ball containing images and metadata.
func (daemon *Daemon) LoadImage(inTar io.ReadCloser, outStream io.Writer, quiet bool) error {
imageExporter := tarexport.NewTarExporter(daemon.imageStore, daemon.layerStore, daemon.referenceStore, daemon)
imageExporter := tarexport.NewTarExporter(daemon.imageStore, daemon.layerStores, daemon.referenceStore, daemon)
return imageExporter.Load(inTar, outStream, quiet)
}

4
daemon/image_history.go
View file

@ -33,12 +33,12 @@ func (daemon *Daemon) ImageHistory(name string) ([]*image.HistoryResponseItem, e
}
rootFS.Append(img.RootFS.DiffIDs[layerCounter])
l, err := daemon.layerStore.Get(rootFS.ChainID())
l, err := daemon.layerStores[img.OperatingSystem()].Get(rootFS.ChainID())
if err != nil {
return nil, err
}
layerSize, err = l.DiffSize()
layer.ReleaseAndLog(daemon.layerStore, l)
layer.ReleaseAndLog(daemon.layerStores[img.OperatingSystem()], l)
if err != nil {
return nil, err
}

15
daemon/image_inspect.go
View file

@ -1,7 +1,6 @@
package daemon
import (
"runtime"
"time"
"github.com/docker/distribution/reference"
@ -18,12 +17,6 @@ func (daemon *Daemon) LookupImage(name string) (*types.ImageInspect, error) {
return nil, errors.Wrapf(err, "no such image: %s", name)
}
// If the image OS isn't set, assume it's the host OS
os := img.OS
if os == "" {
os = runtime.GOOS
}
refs := daemon.referenceStore.References(img.ID().Digest())
repoTags := []string{}
repoDigests := []string{}
@ -40,11 +33,11 @@ func (daemon *Daemon) LookupImage(name string) (*types.ImageInspect, error) {
var layerMetadata map[string]string
layerID := img.RootFS.ChainID()
if layerID != "" {
l, err := daemon.layerStore.Get(layerID)
l, err := daemon.layerStores[img.OperatingSystem()].Get(layerID)
if err != nil {
return nil, err
}
defer layer.ReleaseAndLog(daemon.layerStore, l)
defer layer.ReleaseAndLog(daemon.layerStores[img.OperatingSystem()], l)
size, err = l.Size()
if err != nil {
return nil, err
@ -79,7 +72,7 @@ func (daemon *Daemon) LookupImage(name string) (*types.ImageInspect, error) {
Author: img.Author,
Config: img.Config,
Architecture: img.Architecture,
Os: os,
Os: img.OperatingSystem(),
OsVersion: img.OSVersion,
Size: size,
VirtualSize: size, // TODO: field unused, deprecate
@ -89,7 +82,7 @@ func (daemon *Daemon) LookupImage(name string) (*types.ImageInspect, error) {
},
}
imageInspect.GraphDriver.Name = daemon.GraphDriverName(os)
imageInspect.GraphDriver.Name = daemon.GraphDriverName(img.OperatingSystem())
imageInspect.GraphDriver.Data = layerMetadata
return imageInspect, nil

2
daemon/image_push.go
View file

@ -51,7 +51,7 @@ func (daemon *Daemon) PushImage(ctx context.Context, image, tag string, metaHead
ReferenceStore: daemon.referenceStore,
},
ConfigMediaType: schema2.MediaTypeImageConfig,
LayerStore: distribution.NewLayerProviderFromStore(daemon.layerStore),
LayerStores: distribution.NewLayerProvidersFromStores(daemon.layerStores),
TrustKey: daemon.trustKey,
UploadManager: daemon.uploadManager,
}

14
daemon/images.go
View file

@ -116,7 +116,7 @@ func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs
layerID := img.RootFS.ChainID()
var size int64
if layerID != "" {
l, err := daemon.layerStore.Get(layerID)
l, err := daemon.layerStores[img.OperatingSystem()].Get(layerID)
if err != nil {
// The layer may have been deleted between the call to `Map()` or
// `Heads()` and the call to `Get()`, so we just ignore this error
@ -127,7 +127,7 @@ func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs
}
size, err = l.Size()
layer.ReleaseAndLog(daemon.layerStore, l)
layer.ReleaseAndLog(daemon.layerStores[img.OperatingSystem()], l)
if err != nil {
return nil, err
}
@ -179,7 +179,7 @@ func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs
// lazily init variables
if imagesMap == nil {
allContainers = daemon.List()
allLayers = daemon.layerStore.Map()
allLayers = daemon.layerStores[img.OperatingSystem()].Map()
imagesMap = make(map[*image.Image]*types.ImageSummary)
layerRefs = make(map[layer.ChainID]int)
}
@ -264,11 +264,11 @@ func (daemon *Daemon) SquashImage(id, parent string) (string, error) {
parentImg = &image.Image{RootFS: rootFS}
}
l, err := daemon.layerStore.Get(img.RootFS.ChainID())
l, err := daemon.layerStores[img.OperatingSystem()].Get(img.RootFS.ChainID())
if err != nil {
return "", errors.Wrap(err, "error getting image layer")
}
defer daemon.layerStore.Release(l)
defer daemon.layerStores[img.OperatingSystem()].Release(l)
ts, err := l.TarStreamFrom(parentChainID)
if err != nil {
@ -276,11 +276,11 @@ func (daemon *Daemon) SquashImage(id, parent string) (string, error) {
}
defer ts.Close()
newL, err := daemon.layerStore.Register(ts, parentChainID, img.OperatingSystem())
newL, err := daemon.layerStores[img.OperatingSystem()].Register(ts, parentChainID)
if err != nil {
return "", errors.Wrap(err, "error registering layer")
}
defer daemon.layerStore.Release(newL)
defer daemon.layerStores[img.OperatingSystem()].Release(newL)
newImage := *img
newImage.RootFS = nil

6
daemon/import.go
View file

@ -57,7 +57,7 @@ func (daemon *Daemon) ImportImage(src string, repository, os string, tag string,
}
}
config, err := dockerfile.BuildFromConfig(&container.Config{}, changes)
config, err := dockerfile.BuildFromConfig(&container.Config{}, changes, os)
if err != nil {
return err
}
@ -91,11 +91,11 @@ func (daemon *Daemon) ImportImage(src string, repository, os string, tag string,
if err != nil {
return err
}
l, err := daemon.layerStore.Register(inflatedLayerData, "", os)
l, err := daemon.layerStores[os].Register(inflatedLayerData, "")
if err != nil {
return err
}
defer layer.ReleaseAndLog(daemon.layerStore, l)
defer layer.ReleaseAndLog(daemon.layerStores[os], l)
created := time.Now().UTC()
imgConfig, err := json.Marshal(&image.Image{

2
daemon/info.go
View file

@ -81,7 +81,7 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) {
var ds [][2]string
drivers := ""
for os, gd := range daemon.graphDrivers {
ds = append(ds, daemon.layerStore.DriverStatus(os)...)
ds = append(ds, daemon.layerStores[os].DriverStatus()...)
drivers += gd
if len(daemon.graphDrivers) > 1 {
drivers += fmt.Sprintf(" (%s) ", os)

4
daemon/oci_windows.go
View file

@ -138,9 +138,9 @@ func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) {
max := len(img.RootFS.DiffIDs)
for i := 1; i <= max; i++ {
img.RootFS.DiffIDs = img.RootFS.DiffIDs[:i]
layerPath, err := layer.GetLayerPath(daemon.layerStore, img.RootFS.ChainID())
layerPath, err := layer.GetLayerPath(daemon.layerStores[img.OperatingSystem()], img.RootFS.ChainID())
if err != nil {
return nil, fmt.Errorf("failed to get layer path from graphdriver %s for ImageID %s - %s", daemon.layerStore, img.RootFS.ChainID(), err)
return nil, fmt.Errorf("failed to get layer path from graphdriver %s for ImageID %s - %s", daemon.layerStores[img.OperatingSystem()], img.RootFS.ChainID(), err)
}
// Reverse order, expecting parent most first
s.Windows.LayerFolders = append([]string{layerPath}, s.Windows.LayerFolders...)

7
daemon/prune.go
View file

@ -205,7 +205,12 @@ func (daemon *Daemon) ImagesPrune(ctx context.Context, pruneFilters filters.Args
}
// Filter intermediary images and get their unique size
allLayers := daemon.layerStore.Map()
allLayers := make(map[layer.ChainID]layer.Layer)
for _, ls := range daemon.layerStores {
for k, v := range ls.Map() {
allLayers[k] = v
}
}
topImages := map[image.ID]*image.Image{}
for id, img := range allImages {
select {

2
daemon/start.go
View file

@ -222,7 +222,7 @@ func (daemon *Daemon) Cleanup(container *container.Container) {
if err := daemon.conditionalUnmountOnCleanup(container); err != nil {
// FIXME: remove once reference counting for graphdrivers has been refactored
// Ensure that all the mounts are gone
if mountid, err := daemon.layerStore.GetMountID(container.ID); err == nil {
if mountid, err := daemon.layerStores[container.OS].GetMountID(container.ID); err == nil {
daemon.cleanupMountsByID(mountid)
}
}

14
distribution/config.go
View file

@ -71,8 +71,8 @@ type ImagePushConfig struct {
// ConfigMediaType is the configuration media type for
// schema2 manifests.
ConfigMediaType string
// LayerStore manages layers.
LayerStore PushLayerProvider
// LayerStores (indexed by operating system) manages layers.
LayerStores map[string]PushLayerProvider
// TrustKey is the private key for legacy signatures. This is typically
// an ephemeral key, since these signatures are no longer verified.
TrustKey libtrust.PrivateKey
@ -165,13 +165,15 @@ type storeLayerProvider struct {
ls layer.Store
}
// NewLayerProviderFromStore returns a layer provider backed by
// NewLayerProvidersFromStores returns layer providers backed by
// an instance of LayerStore. Only getting layers as gzipped
// tars is supported.
func NewLayerProviderFromStore(ls layer.Store) PushLayerProvider {
return &storeLayerProvider{
ls: ls,
func NewLayerProvidersFromStores(lss map[string]layer.Store) map[string]PushLayerProvider {
plps := make(map[string]PushLayerProvider)
for os, ls := range lss {
plps[os] = &storeLayerProvider{ls: ls}
}
return plps
}
func (p *storeLayerProvider) Get(lid layer.ChainID) (PushLayer, error) {

2
distribution/push_v1.go
View file

@ -210,7 +210,7 @@ func (p *v1Pusher) imageListForTag(imgID image.ID, dependenciesSeen map[layer.Ch
topLayerID := img.RootFS.ChainID()
pl, err := p.config.LayerStore.Get(topLayerID)
pl, err := p.config.LayerStores[img.OperatingSystem()].Get(topLayerID)
*referencedLayers = append(*referencedLayers, pl)
if err != nil {
return nil, fmt.Errorf("failed to get top layer from image: %v", err)

4
distribution/push_v2.go
View file

@ -118,12 +118,12 @@ func (p *v2Pusher) pushV2Tag(ctx context.Context, ref reference.NamedTagged, id
return fmt.Errorf("could not find image from tag %s: %v", reference.FamiliarString(ref), err)
}
rootfs, _, err := p.config.ImageStore.RootFSAndOSFromConfig(imgConfig)
rootfs, os, err := p.config.ImageStore.RootFSAndOSFromConfig(imgConfig)
if err != nil {
return fmt.Errorf("unable to get rootfs for image %s: %s", reference.FamiliarString(ref), err)
}
l, err := p.config.LayerStore.Get(rootfs.ChainID())
l, err := p.config.LayerStores[os].Get(rootfs.ChainID())
if err != nil {
return fmt.Errorf("failed to get top layer from image: %v", err)
}

32
distribution/xfer/download.go
View file

@ -4,6 +4,7 @@ import (
"errors"
"fmt"
"io"
"runtime"
"time"
"github.com/docker/distribution"
@ -22,7 +23,7 @@ const maxDownloadAttempts = 5
// registers and downloads those, taking into account dependencies between
// layers.
type LayerDownloadManager struct {
layerStore layer.Store
layerStores map[string]layer.Store
tm TransferManager
waitDuration time.Duration
}
@ -33,9 +34,9 @@ func (ldm *LayerDownloadManager) SetConcurrency(concurrency int) {
}
// NewLayerDownloadManager returns a new LayerDownloadManager.
func NewLayerDownloadManager(layerStore layer.Store, concurrencyLimit int, options ...func(*LayerDownloadManager)) *LayerDownloadManager {
func NewLayerDownloadManager(layerStores map[string]layer.Store, concurrencyLimit int, options ...func(*LayerDownloadManager)) *LayerDownloadManager {
manager := LayerDownloadManager{
layerStore: layerStore,
layerStores: layerStores,
tm: NewTransferManager(concurrencyLimit),
waitDuration: time.Second,
}
@ -104,6 +105,11 @@ func (ldm *LayerDownloadManager) Download(ctx context.Context, initialRootFS ima
downloadsByKey = make(map[string]*downloadTransfer)
)
// Assume that the operating system is the host OS if blank
if os == "" {
os = runtime.GOOS
}
rootFS := initialRootFS
for _, descriptor := range layers {
key := descriptor.Key()
@ -115,13 +121,13 @@ func (ldm *LayerDownloadManager) Download(ctx context.Context, initialRootFS ima
if err == nil {
getRootFS := rootFS
getRootFS.Append(diffID)
l, err := ldm.layerStore.Get(getRootFS.ChainID())
l, err := ldm.layerStores[os].Get(getRootFS.ChainID())
if err == nil {
// Layer already exists.
logrus.Debugf("Layer already exists: %s", descriptor.ID())
progress.Update(progressOutput, descriptor.ID(), "Already exists")
if topLayer != nil {
layer.ReleaseAndLog(ldm.layerStore, topLayer)
layer.ReleaseAndLog(ldm.layerStores[os], topLayer)
}
topLayer = l
missingLayer = false
@ -165,7 +171,7 @@ func (ldm *LayerDownloadManager) Download(ctx context.Context, initialRootFS ima
if topDownload == nil {
return rootFS, func() {
if topLayer != nil {
layer.ReleaseAndLog(ldm.layerStore, topLayer)
layer.ReleaseAndLog(ldm.layerStores[os], topLayer)
}
}, nil
}
@ -176,7 +182,7 @@ func (ldm *LayerDownloadManager) Download(ctx context.Context, initialRootFS ima
defer func() {
if topLayer != nil {
layer.ReleaseAndLog(ldm.layerStore, topLayer)
layer.ReleaseAndLog(ldm.layerStores[os], topLayer)
}
}()
@ -216,7 +222,7 @@ func (ldm *LayerDownloadManager) makeDownloadFunc(descriptor DownloadDescriptor,
return func(progressChan chan<- progress.Progress, start <-chan struct{}, inactive chan<- struct{}) Transfer {
d := &downloadTransfer{
Transfer: NewTransfer(),
layerStore: ldm.layerStore,
layerStore: ldm.layerStores[os],
}
go func() {
@ -335,9 +341,9 @@ func (ldm *LayerDownloadManager) makeDownloadFunc(descriptor DownloadDescriptor,
src = fs.Descriptor()
}
if ds, ok := d.layerStore.(layer.DescribableStore); ok {
d.layer, err = ds.RegisterWithDescriptor(inflatedLayerData, parentLayer, os, src)
d.layer, err = ds.RegisterWithDescriptor(inflatedLayerData, parentLayer, src)
} else {
d.layer, err = d.layerStore.Register(inflatedLayerData, parentLayer, os)
d.layer, err = d.layerStore.Register(inflatedLayerData, parentLayer)
}
if err != nil {
select {
@ -380,7 +386,7 @@ func (ldm *LayerDownloadManager) makeDownloadFuncFromDownload(descriptor Downloa
return func(progressChan chan<- progress.Progress, start <-chan struct{}, inactive chan<- struct{}) Transfer {
d := &downloadTransfer{
Transfer: NewTransfer(),
layerStore: ldm.layerStore,
layerStore: ldm.layerStores[os],
}
go func() {
@ -434,9 +440,9 @@ func (ldm *LayerDownloadManager) makeDownloadFuncFromDownload(descriptor Downloa
src = fs.Descriptor()
}
if ds, ok := d.layerStore.(layer.DescribableStore); ok {
d.layer, err = ds.RegisterWithDescriptor(layerReader, parentLayer, os, src)
d.layer, err = ds.RegisterWithDescriptor(layerReader, parentLayer, src)
} else {
d.layer, err = d.layerStore.Register(layerReader, parentLayer, os)
d.layer, err = d.layerStore.Register(layerReader, parentLayer)
}
if err != nil {
d.err = fmt.Errorf("failed to register layer: %v", err)

23
distribution/xfer/download_test.go
View file

@ -91,7 +91,7 @@ func (ls *mockLayerStore) Map() map[layer.ChainID]layer.Layer {
return layers
}
func (ls *mockLayerStore) Register(reader io.Reader, parentID layer.ChainID, os string) (layer.Layer, error) {
func (ls *mockLayerStore) Register(reader io.Reader, parentID layer.ChainID) (layer.Layer, error) {
return ls.RegisterWithDescriptor(reader, parentID, distribution.Descriptor{})
}
@ -131,7 +131,7 @@ func (ls *mockLayerStore) Get(chainID layer.ChainID) (layer.Layer, error) {
func (ls *mockLayerStore) Release(l layer.Layer) ([]layer.Metadata, error) {
return []layer.Metadata{}, nil
}
func (ls *mockLayerStore) CreateRWLayer(string, layer.ChainID, string, *layer.CreateRWLayerOpts) (layer.RWLayer, error) {
func (ls *mockLayerStore) CreateRWLayer(string, layer.ChainID, *layer.CreateRWLayerOpts) (layer.RWLayer, error) {
return nil, errors.New("not implemented")
}
@ -150,14 +150,18 @@ func (ls *mockLayerStore) Cleanup() error {
return nil
}
func (ls *mockLayerStore) DriverStatus(string) [][2]string {
func (ls *mockLayerStore) DriverStatus() [][2]string {
return [][2]string{}
}
func (ls *mockLayerStore) DriverName(string) string {
func (ls *mockLayerStore) DriverName() string {
return "mock"
}
func (ls *mockLayerStore) OS() string {
return runtime.GOOS
}
type mockDownloadDescriptor struct {
currentDownloads *int32
id string
@ -272,7 +276,9 @@ func TestSuccessfulDownload(t *testing.T) {
}
layerStore := &mockLayerStore{make(map[layer.ChainID]*mockLayer)}
ldm := NewLayerDownloadManager(layerStore, maxDownloadConcurrency, func(m *LayerDownloadManager) { m.waitDuration = time.Millisecond })
lsMap := make(map[string]layer.Store)
lsMap[runtime.GOOS] = layerStore
ldm := NewLayerDownloadManager(lsMap, maxDownloadConcurrency, func(m *LayerDownloadManager) { m.waitDuration = time.Millisecond })
progressChan := make(chan progress.Progress)
progressDone := make(chan struct{})
@ -291,7 +297,7 @@ func TestSuccessfulDownload(t *testing.T) {
firstDescriptor := descriptors[0].(*mockDownloadDescriptor)
// Pre-register the first layer to simulate an already-existing layer
l, err := layerStore.Register(firstDescriptor.mockTarStream(), "", runtime.GOOS)
l, err := layerStore.Register(firstDescriptor.mockTarStream(), "")
if err != nil {
t.Fatal(err)
}
@ -334,8 +340,9 @@ func TestSuccessfulDownload(t *testing.T) {
func TestCancelledDownload(t *testing.T) {
layerStore := &mockLayerStore{make(map[layer.ChainID]*mockLayer)}
ldm := NewLayerDownloadManager(layerStore, maxDownloadConcurrency, func(m *LayerDownloadManager) { m.waitDuration = time.Millisecond })
lsMap := make(map[string]layer.Store)
lsMap[runtime.GOOS] = layerStore
ldm := NewLayerDownloadManager(lsMap, maxDownloadConcurrency, func(m *LayerDownloadManager) { m.waitDuration = time.Millisecond })
progressChan := make(chan progress.Progress)
progressDone := make(chan struct{})

14
image/store.go
View file

@ -41,16 +41,16 @@ type imageMeta struct {
type store struct {
sync.RWMutex
ls LayerGetReleaser
lss map[string]LayerGetReleaser
images map[ID]*imageMeta
fs StoreBackend
digestSet *digestset.Set
}
// NewImageStore returns new store object for given layer store
func NewImageStore(fs StoreBackend, ls LayerGetReleaser) (Store, error) {
// NewImageStore returns new store object for given set of layer stores
func NewImageStore(fs StoreBackend, lss map[string]LayerGetReleaser) (Store, error) {
is := &store{
ls: ls,
lss: lss,
images: make(map[ID]*imageMeta),
fs: fs,
digestSet: digestset.NewSet(),
@ -73,7 +73,7 @@ func (is *store) restore() error {
}
var l layer.Layer
if chainID := img.RootFS.ChainID(); chainID != "" {
l, err = is.ls.Get(chainID)
l, err = is.lss[img.OperatingSystem()].Get(chainID)
if err != nil {
return err
}
@ -148,7 +148,7 @@ func (is *store) Create(config []byte) (ID, error) {
var l layer.Layer
if layerID != "" {
l, err = is.ls.Get(layerID)
l, err = is.lss[img.OperatingSystem()].Get(layerID)
if err != nil {
return "", errors.Wrapf(err, "failed to get layer %s", layerID)
}
@ -231,7 +231,7 @@ func (is *store) Delete(id ID) ([]layer.Metadata, error) {
is.fs.Delete(id.Digest())
if imageMeta.layer != nil {
return is.ls.Release(imageMeta.layer)
return is.lss[imageMeta.layer.OS()].Release(imageMeta.layer)
}
return nil, nil
}

9
image/store_test.go
View file

@ -1,6 +1,7 @@
package image
import (
"runtime"
"testing"
"github.com/docker/docker/internal/testutil"
@ -25,7 +26,9 @@ func TestRestore(t *testing.T) {
err = fs.SetMetadata(id2, "parent", []byte(id1))
assert.NoError(t, err)
is, err := NewImageStore(fs, &mockLayerGetReleaser{})
mlgrMap := make(map[string]LayerGetReleaser)
mlgrMap[runtime.GOOS] = &mockLayerGetReleaser{}
is, err := NewImageStore(fs, mlgrMap)
assert.NoError(t, err)
assert.Len(t, is.Map(), 2)
@ -142,7 +145,9 @@ func TestParentReset(t *testing.T) {
func defaultImageStore(t *testing.T) (Store, func()) {
fsBackend, cleanup := defaultFSStoreBackend(t)
store, err := NewImageStore(fsBackend, &mockLayerGetReleaser{})
mlgrMap := make(map[string]LayerGetReleaser)
mlgrMap[runtime.GOOS] = &mockLayerGetReleaser{}
store, err := NewImageStore(fsBackend, mlgrMap)
assert.NoError(t, err)
return store, cleanup

17
image/tarexport/load.go
View file

@ -107,14 +107,14 @@ func (l *tarexporter) Load(inTar io.ReadCloser, outStream io.Writer, quiet bool)
}
r := rootFS
r.Append(diffID)
newLayer, err := l.ls.Get(r.ChainID())
newLayer, err := l.lss[os].Get(r.ChainID())
if err != nil {
newLayer, err = l.loadLayer(layerPath, rootFS, diffID.String(), os, m.LayerSources[diffID], progressOutput)
if err != nil {
return err
}
}
defer layer.ReleaseAndLog(l.ls, newLayer)
defer layer.ReleaseAndLog(l.lss[os], newLayer)
if expected, actual := diffID, newLayer.DiffID(); expected != actual {
return fmt.Errorf("invalid diffID for layer %d: expected %q, got %q", i, expected, actual)
}
@ -205,10 +205,10 @@ func (l *tarexporter) loadLayer(filename string, rootFS image.RootFS, id string,
}
defer inflatedLayerData.Close()
if ds, ok := l.ls.(layer.DescribableStore); ok {
return ds.RegisterWithDescriptor(inflatedLayerData, rootFS.ChainID(), os, foreignSrc)
if ds, ok := l.lss[os].(layer.DescribableStore); ok {
return ds.RegisterWithDescriptor(inflatedLayerData, rootFS.ChainID(), foreignSrc)
}
return l.ls.Register(inflatedLayerData, rootFS.ChainID(), os)
return l.lss[os].Register(inflatedLayerData, rootFS.ChainID())
}
func (l *tarexporter) setLoadedTag(ref reference.Named, imgID digest.Digest, outStream io.Writer) error {
@ -302,6 +302,9 @@ func (l *tarexporter) legacyLoadImage(oldID, sourceDir string, loadedMap map[str
if err := checkCompatibleOS(img.OS); err != nil {
return err
}
if img.OS == "" {
img.OS = runtime.GOOS
}
var parentID image.ID
if img.Parent != "" {
@ -335,7 +338,7 @@ func (l *tarexporter) legacyLoadImage(oldID, sourceDir string, loadedMap map[str
if err != nil {
return err
}
newLayer, err := l.loadLayer(layerPath, *rootFS, oldID, "", distribution.Descriptor{}, progressOutput)
newLayer, err := l.loadLayer(layerPath, *rootFS, oldID, img.OS, distribution.Descriptor{}, progressOutput)
if err != nil {
return err
}
@ -356,7 +359,7 @@ func (l *tarexporter) legacyLoadImage(oldID, sourceDir string, loadedMap map[str
return err
}
metadata, err := l.ls.Release(newLayer)
metadata, err := l.lss[img.OS].Release(newLayer)
layer.LogReleaseMetadata(metadata)
if err != nil {
return err

21
image/tarexport/save.go
View file

@ -7,6 +7,7 @@ import (
"io/ioutil"
"os"
"path/filepath"
"runtime"
"time"
"github.com/docker/distribution"
@ -153,7 +154,11 @@ func (l *tarexporter) takeLayerReference(id image.ID, imgDescr *imageDescriptor)
if topLayerID == "" {
return nil
}
layer, err := l.ls.Get(topLayerID)
os := img.OS
if os == "" {
os = runtime.GOOS
}
layer, err := l.lss[os].Get(topLayerID)
if err != nil {
return err
}
@ -165,7 +170,11 @@ func (l *tarexporter) takeLayerReference(id image.ID, imgDescr *imageDescriptor)
func (l *tarexporter) releaseLayerReferences(imgDescr map[image.ID]*imageDescriptor) error {
for _, descr := range imgDescr {
if descr.layerRef != nil {
l.ls.Release(descr.layerRef)
os := descr.image.OS
if os == "" {
os = runtime.GOOS
}
l.lss[os].Release(descr.layerRef)
}
}
return nil
@ -356,11 +365,15 @@ func (s *saveSession) saveLayer(id layer.ChainID, legacyImg image.V1Image, creat
// serialize filesystem
layerPath := filepath.Join(outDir, legacyLayerFileName)
l, err := s.ls.Get(id)
operatingSystem := legacyImg.OS
if operatingSystem == "" {
operatingSystem = runtime.GOOS
}
l, err := s.lss[operatingSystem].Get(id)
if err != nil {
return distribution.Descriptor{}, err
}
defer layer.ReleaseAndLog(s.ls, l)
defer layer.ReleaseAndLog(s.lss[operatingSystem], l)
if oldPath, exists := s.diffIDPaths[l.DiffID()]; exists {
relPath, err := filepath.Rel(outDir, oldPath)

6
image/tarexport/tarexport.go
View file

@ -25,7 +25,7 @@ type manifestItem struct {
type tarexporter struct {
is image.Store
ls layer.Store
lss map[string]layer.Store
rs refstore.Store
loggerImgEvent LogImageEvent
}
@ -37,10 +37,10 @@ type LogImageEvent interface {
}
// NewTarExporter returns new Exporter for tar packages
func NewTarExporter(is image.Store, ls layer.Store, rs refstore.Store, loggerImgEvent LogImageEvent) image.Exporter {
func NewTarExporter(is image.Store, lss map[string]layer.Store, rs refstore.Store, loggerImgEvent LogImageEvent) image.Exporter {
return &tarexporter{
is: is,
ls: ls,
lss: lss,
rs: rs,
loggerImgEvent: loggerImgEvent,
}

9
layer/empty.go
View file

@ -6,7 +6,6 @@ import (
"fmt"
"io"
"io/ioutil"
"runtime"
)
// DigestSHA256EmptyTar is the canonical sha256 digest of empty tar file -
@ -44,6 +43,10 @@ func (el *emptyLayer) Parent() Layer {
return nil
}
func (el *emptyLayer) OS() string {
return ""
}
func (el *emptyLayer) Size() (size int64, err error) {
return 0, nil
}
@ -56,10 +59,6 @@ func (el *emptyLayer) Metadata() (map[string]string, error) {
return make(map[string]string), nil
}
func (el *emptyLayer) OS() string {
return runtime.GOOS
}
// IsEmpty returns true if the layer is an EmptyLayer
func IsEmpty(diffID DiffID) bool {
return diffID == DigestSHA256EmptyTar

11
layer/layer.go
View file

@ -186,25 +186,26 @@ type CreateRWLayerOpts struct {
// Store represents a backend for managing both
// read-only and read-write layers.
type Store interface {
Register(io.Reader, ChainID, string) (Layer, error)
Register(io.Reader, ChainID) (Layer, error)
Get(ChainID) (Layer, error)
Map() map[ChainID]Layer
Release(Layer) ([]Metadata, error)
CreateRWLayer(id string, parent ChainID, os string, opts *CreateRWLayerOpts) (RWLayer, error)
CreateRWLayer(id string, parent ChainID, opts *CreateRWLayerOpts) (RWLayer, error)
GetRWLayer(id string) (RWLayer, error)
GetMountID(id string) (string, error)
ReleaseRWLayer(RWLayer) ([]Metadata, error)
Cleanup() error
DriverStatus(os string) [][2]string
DriverName(os string) string
DriverStatus() [][2]string
DriverName() string
OS() string
}
// DescribableStore represents a layer store capable of storing
// descriptors for layers.
type DescribableStore interface {
RegisterWithDescriptor(io.Reader, ChainID, string, distribution.Descriptor) (Layer, error)
RegisterWithDescriptor(io.Reader, ChainID, distribution.Descriptor) (Layer, error)
}
// MetadataTransaction represents functions for setting layer metadata

158
layer/layer_store.go
View file

@ -5,7 +5,6 @@ import (
"fmt"
"io"
"io/ioutil"
"runtime"
"sync"
"github.com/docker/distribution"
@ -29,76 +28,70 @@ const maxLayerDepth = 125
type layerStore struct {
store MetadataStore
drivers map[string]graphdriver.Driver
useTarSplit map[string]bool
driver graphdriver.Driver
useTarSplit bool
layerMap map[ChainID]*roLayer
layerL sync.Mutex
mounts map[string]*mountedLayer
mountL sync.Mutex
os string
}
// StoreOptions are the options used to create a new Store instance
type StoreOptions struct {
Root string
GraphDrivers map[string]string
MetadataStorePathTemplate string
GraphDriver string
GraphDriverOptions []string
IDMappings *idtools.IDMappings
PluginGetter plugingetter.PluginGetter
ExperimentalEnabled bool
OS string
}
// NewStoreFromOptions creates a new Store instance
func NewStoreFromOptions(options StoreOptions) (Store, error) {
drivers := make(map[string]graphdriver.Driver)
for os, drivername := range options.GraphDrivers {
var err error
drivers[os], err = graphdriver.New(drivername,
options.PluginGetter,
graphdriver.Options{
Root: options.Root,
DriverOptions: options.GraphDriverOptions,
UIDMaps: options.IDMappings.UIDs(),
GIDMaps: options.IDMappings.GIDs(),
ExperimentalEnabled: options.ExperimentalEnabled,
})
if err != nil {
return nil, fmt.Errorf("error initializing graphdriver: %v", err)
}
logrus.Debugf("Initialized graph driver %s", drivername)
driver, err := graphdriver.New(options.GraphDriver, options.PluginGetter, graphdriver.Options{
Root: options.Root,
DriverOptions: options.GraphDriverOptions,
UIDMaps: options.IDMappings.UIDs(),
GIDMaps: options.IDMappings.GIDs(),
ExperimentalEnabled: options.ExperimentalEnabled,
})
if err != nil {
return nil, fmt.Errorf("error initializing graphdriver: %v", err)
}
logrus.Debugf("Initialized graph driver %s", driver)
fms, err := NewFSMetadataStore(fmt.Sprintf(options.MetadataStorePathTemplate, options.GraphDrivers[runtime.GOOS]))
fms, err := NewFSMetadataStore(fmt.Sprintf(options.MetadataStorePathTemplate, driver))
if err != nil {
return nil, err
}
return NewStoreFromGraphDrivers(fms, drivers)
return NewStoreFromGraphDriver(fms, driver, options.OS)
}
// NewStoreFromGraphDrivers creates a new Store instance using the provided
// metadata store and graph drivers. The metadata store will be used to restore
// NewStoreFromGraphDriver creates a new Store instance using the provided
// metadata store and graph driver. The metadata store will be used to restore
// the Store.
func NewStoreFromGraphDrivers(store MetadataStore, drivers map[string]graphdriver.Driver) (Store, error) {
useTarSplit := make(map[string]bool)
for os, driver := range drivers {
caps := graphdriver.Capabilities{}
if capDriver, ok := driver.(graphdriver.CapabilityDriver); ok {
caps = capDriver.Capabilities()
}
useTarSplit[os] = !caps.ReproducesExactDiffs
func NewStoreFromGraphDriver(store MetadataStore, driver graphdriver.Driver, os string) (Store, error) {
if !system.IsOSSupported(os) {
return nil, fmt.Errorf("failed to initialize layer store as operating system '%s' is not supported", os)
}
caps := graphdriver.Capabilities{}
if capDriver, ok := driver.(graphdriver.CapabilityDriver); ok {
caps = capDriver.Capabilities()
}
ls := &layerStore{
store: store,
drivers: drivers,
driver: driver,
layerMap: map[ChainID]*roLayer{},
mounts: map[string]*mountedLayer{},
useTarSplit: useTarSplit,
useTarSplit: !caps.ReproducesExactDiffs,
os: os,
}
ids, mounts, err := store.List()
@ -162,6 +155,10 @@ func (ls *layerStore) loadLayer(layer ChainID) (*roLayer, error) {
return nil, fmt.Errorf("failed to get operating system for %s: %s", layer, err)
}
if os != ls.os {
return nil, fmt.Errorf("failed to load layer with os %s into layerstore for %s", os, ls.os)
}
cl = &roLayer{
chainID: layer,
diffID: diff,
@ -170,7 +167,6 @@ func (ls *layerStore) loadLayer(layer ChainID) (*roLayer, error) {
layerStore: ls,
references: map[Layer]struct{}{},
descriptor: descriptor,
os: os,
}
if parent != "" {
@ -234,7 +230,7 @@ func (ls *layerStore) applyTar(tx MetadataTransaction, ts io.Reader, parent stri
tr := io.TeeReader(ts, digester.Hash())
rdr := tr
if ls.useTarSplit[layer.os] {
if ls.useTarSplit {
tsw, err := tx.TarSplitWriter(true)
if err != nil {
return err
@ -250,7 +246,7 @@ func (ls *layerStore) applyTar(tx MetadataTransaction, ts io.Reader, parent stri
}
}
applySize, err := ls.drivers[layer.os].ApplyDiff(layer.cacheID, parent, rdr)
applySize, err := ls.driver.ApplyDiff(layer.cacheID, parent, rdr)
if err != nil {
return err
}
@ -266,11 +262,11 @@ func (ls *layerStore) applyTar(tx MetadataTransaction, ts io.Reader, parent stri
return nil
}
func (ls *layerStore) Register(ts io.Reader, parent ChainID, os string) (Layer, error) {
return ls.registerWithDescriptor(ts, parent, os, distribution.Descriptor{})
func (ls *layerStore) Register(ts io.Reader, parent ChainID) (Layer, error) {
return ls.registerWithDescriptor(ts, parent, distribution.Descriptor{})
}
func (ls *layerStore) registerWithDescriptor(ts io.Reader, parent ChainID, os string, descriptor distribution.Descriptor) (Layer, error) {
func (ls *layerStore) registerWithDescriptor(ts io.Reader, parent ChainID, descriptor distribution.Descriptor) (Layer, error) {
// err is used to hold the error which will always trigger
// cleanup of creates sources but may not be an error returned
// to the caller (already exists).
@ -298,14 +294,6 @@ func (ls *layerStore) registerWithDescriptor(ts io.Reader, parent ChainID, os st
}
}
// Validate the operating system is valid
if os == "" {
os = runtime.GOOS
}
if err := system.ValidatePlatform(system.ParsePlatform(os)); err != nil {
return nil, err
}
// Create new roLayer
layer := &roLayer{
parent: p,
@ -314,10 +302,9 @@ func (ls *layerStore) registerWithDescriptor(ts io.Reader, parent ChainID, os st
layerStore: ls,
references: map[Layer]struct{}{},
descriptor: descriptor,
os: os,
}
if err = ls.drivers[os].Create(layer.cacheID, pid, nil); err != nil {
if err = ls.driver.Create(layer.cacheID, pid, nil); err != nil {
return nil, err
}
@ -329,7 +316,7 @@ func (ls *layerStore) registerWithDescriptor(ts io.Reader, parent ChainID, os st
defer func() {
if err != nil {
logrus.Debugf("Cleaning up layer %s: %v", layer.cacheID, err)
if err := ls.drivers[os].Remove(layer.cacheID); err != nil {
if err := ls.driver.Remove(layer.cacheID); err != nil {
logrus.Errorf("Error cleaning up cache layer %s: %v", layer.cacheID, err)
}
if err := tx.Cancel(); err != nil {
@ -413,7 +400,7 @@ func (ls *layerStore) Map() map[ChainID]Layer {
}
func (ls *layerStore) deleteLayer(layer *roLayer, metadata *Metadata) error {
err := ls.drivers[layer.os].Remove(layer.cacheID)
err := ls.driver.Remove(layer.cacheID)
if err != nil {
return err
}
@ -483,7 +470,7 @@ func (ls *layerStore) Release(l Layer) ([]Metadata, error) {
return ls.releaseLayer(layer)
}
func (ls *layerStore) CreateRWLayer(name string, parent ChainID, os string, opts *CreateRWLayerOpts) (RWLayer, error) {
func (ls *layerStore) CreateRWLayer(name string, parent ChainID, opts *CreateRWLayerOpts) (RWLayer, error) {
var (
storageOpt map[string]string
initFunc MountInit
@ -523,21 +510,16 @@ func (ls *layerStore) CreateRWLayer(name string, parent ChainID, os string, opts
}()
}
// Ensure the operating system is set to the host OS if not populated.
if os == "" {
os = runtime.GOOS
}
m = &mountedLayer{
name: name,
parent: p,
mountID: ls.mountID(name),
layerStore: ls,
references: map[RWLayer]*referencedRWLayer{},
os: os,
}
if initFunc != nil {
pid, err = ls.initMount(m.mountID, m.os, pid, mountLabel, initFunc, storageOpt)
pid, err = ls.initMount(m.mountID, pid, mountLabel, initFunc, storageOpt)
if err != nil {
return nil, err
}
@ -548,7 +530,7 @@ func (ls *layerStore) CreateRWLayer(name string, parent ChainID, os string, opts
StorageOpt: storageOpt,
}
if err = ls.drivers[os].CreateReadWrite(m.mountID, pid, createOpts); err != nil {
if err = ls.driver.CreateReadWrite(m.mountID, pid, createOpts); err != nil {
return nil, err
}
if err = ls.saveMount(m); err != nil {
@ -597,14 +579,14 @@ func (ls *layerStore) ReleaseRWLayer(l RWLayer) ([]Metadata, error) {
return []Metadata{}, nil
}
if err := ls.drivers[l.OS()].Remove(m.mountID); err != nil {
if err := ls.driver.Remove(m.mountID); err != nil {
logrus.Errorf("Error removing mounted layer %s: %s", m.name, err)
m.retakeReference(l)
return nil, err
}
if m.initID != "" {
if err := ls.drivers[l.OS()].Remove(m.initID); err != nil {
if err := ls.driver.Remove(m.initID); err != nil {
logrus.Errorf("Error removing init layer %s: %s", m.name, err)
m.retakeReference(l)
return nil, err
@ -650,7 +632,7 @@ func (ls *layerStore) saveMount(mount *mountedLayer) error {
return nil
}
func (ls *layerStore) initMount(graphID, os, parent, mountLabel string, initFunc MountInit, storageOpt map[string]string) (string, error) {
func (ls *layerStore) initMount(graphID, parent, mountLabel string, initFunc MountInit, storageOpt map[string]string) (string, error) {
// Use "<graph-id>-init" to maintain compatibility with graph drivers
// which are expecting this layer with this special name. If all
// graph drivers can be updated to not rely on knowing about this layer
@ -662,20 +644,20 @@ func (ls *layerStore) initMount(graphID, os, parent, mountLabel string, initFunc
StorageOpt: storageOpt,
}
if err := ls.drivers[os].CreateReadWrite(initID, parent, createOpts); err != nil {
if err := ls.driver.CreateReadWrite(initID, parent, createOpts); err != nil {
return "", err
}
p, err := ls.drivers[os].Get(initID, "")
p, err := ls.driver.Get(initID, "")
if err != nil {
return "", err
}
if err := initFunc(p); err != nil {
ls.drivers[os].Put(initID)
ls.driver.Put(initID)
return "", err
}
if err := ls.drivers[os].Put(initID); err != nil {
if err := ls.driver.Put(initID); err != nil {
return "", err
}
@ -683,13 +665,13 @@ func (ls *layerStore) initMount(graphID, os, parent, mountLabel string, initFunc
}
func (ls *layerStore) getTarStream(rl *roLayer) (io.ReadCloser, error) {
if !ls.useTarSplit[rl.os] {
if !ls.useTarSplit {
var parentCacheID string
if rl.parent != nil {
parentCacheID = rl.parent.cacheID
}
return ls.drivers[rl.os].Diff(rl.cacheID, parentCacheID)
return ls.driver.Diff(rl.cacheID, parentCacheID)
}
r, err := ls.store.TarSplitReader(rl.chainID)
@ -699,7 +681,7 @@ func (ls *layerStore) getTarStream(rl *roLayer) (io.ReadCloser, error) {
pr, pw := io.Pipe()
go func() {
err := ls.assembleTarTo(rl.cacheID, rl.os, r, nil, pw)
err := ls.assembleTarTo(rl.cacheID, r, nil, pw)
if err != nil {
pw.CloseWithError(err)
} else {
@ -710,10 +692,10 @@ func (ls *layerStore) getTarStream(rl *roLayer) (io.ReadCloser, error) {
return pr, nil
}
func (ls *layerStore) assembleTarTo(graphID, os string, metadata io.ReadCloser, size *int64, w io.Writer) error {
diffDriver, ok := ls.drivers[os].(graphdriver.DiffGetterDriver)
func (ls *layerStore) assembleTarTo(graphID string, metadata io.ReadCloser, size *int64, w io.Writer) error {
diffDriver, ok := ls.driver.(graphdriver.DiffGetterDriver)
if !ok {
diffDriver = &naiveDiffPathDriver{ls.drivers[os]}
diffDriver = &naiveDiffPathDriver{ls.driver}
}
defer metadata.Close()
@ -732,27 +714,19 @@ func (ls *layerStore) assembleTarTo(graphID, os string, metadata io.ReadCloser,
}
func (ls *layerStore) Cleanup() error {
var err error
for _, driver := range ls.drivers {
if e := driver.Cleanup(); e != nil {
err = fmt.Errorf("%s - %s", err.Error(), e.Error())
}
}
return err
return ls.driver.Cleanup()
}
func (ls *layerStore) DriverStatus(os string) [][2]string {
if os == "" {
os = runtime.GOOS
}
return ls.drivers[os].Status()
func (ls *layerStore) DriverStatus() [][2]string {
return ls.driver.Status()
}
func (ls *layerStore) DriverName(os string) string {
if os == "" {
os = runtime.GOOS
}
return ls.drivers[os].String()
func (ls *layerStore) DriverName() string {
return ls.driver.String()
}
func (ls *layerStore) OS() string {
return ls.os
}
type naiveDiffPathDriver struct {

4
layer/layer_store_windows.go
View file

@ -6,6 +6,6 @@ import (
"github.com/docker/distribution"
)
func (ls *layerStore) RegisterWithDescriptor(ts io.Reader, parent ChainID, os string, descriptor distribution.Descriptor) (Layer, error) {
return ls.registerWithDescriptor(ts, parent, os, descriptor)
func (ls *layerStore) RegisterWithDescriptor(ts io.Reader, parent ChainID, descriptor distribution.Descriptor) (Layer, error) {
return ls.registerWithDescriptor(ts, parent, descriptor)
}

32
layer/layer_test.go
View file

@ -73,9 +73,7 @@ func newTestStore(t *testing.T) (Store, string, func()) {
if err != nil {
t.Fatal(err)
}
graphs := make(map[string]graphdriver.Driver)
graphs[runtime.GOOS] = graph
ls, err := NewStoreFromGraphDrivers(fms, graphs)
ls, err := NewStoreFromGraphDriver(fms, graph, runtime.GOOS)
if err != nil {
t.Fatal(err)
}
@ -90,7 +88,7 @@ type layerInit func(root containerfs.ContainerFS) error
func createLayer(ls Store, parent ChainID, layerFunc layerInit) (Layer, error) {
containerID := stringid.GenerateRandomID()
mount, err := ls.CreateRWLayer(containerID, parent, runtime.GOOS, nil)
mount, err := ls.CreateRWLayer(containerID, parent, nil)
if err != nil {
return nil, err
}
@ -110,7 +108,7 @@ func createLayer(ls Store, parent ChainID, layerFunc layerInit) (Layer, error) {
}
defer ts.Close()
layer, err := ls.Register(ts, parent, runtime.GOOS)
layer, err := ls.Register(ts, parent)
if err != nil {
return nil, err
}
@ -279,7 +277,7 @@ func TestMountAndRegister(t *testing.T) {
size, _ := layer.Size()
t.Logf("Layer size: %d", size)
mount2, err := ls.CreateRWLayer("new-test-mount", layer.ChainID(), runtime.GOOS, nil)
mount2, err := ls.CreateRWLayer("new-test-mount", layer.ChainID(), nil)
if err != nil {
t.Fatal(err)
}
@ -387,7 +385,7 @@ func TestStoreRestore(t *testing.T) {
t.Fatal(err)
}
m, err := ls.CreateRWLayer("some-mount_name", layer3.ChainID(), runtime.GOOS, nil)
m, err := ls.CreateRWLayer("some-mount_name", layer3.ChainID(), nil)
if err != nil {
t.Fatal(err)
}
@ -405,7 +403,7 @@ func TestStoreRestore(t *testing.T) {
t.Fatal(err)
}
ls2, err := NewStoreFromGraphDrivers(ls.(*layerStore).store, ls.(*layerStore).drivers)
ls2, err := NewStoreFromGraphDriver(ls.(*layerStore).store, ls.(*layerStore).driver, runtime.GOOS)
if err != nil {
t.Fatal(err)
}
@ -418,7 +416,7 @@ func TestStoreRestore(t *testing.T) {
assertLayerEqual(t, layer3b, layer3)
// Create again with same name, should return error
if _, err := ls2.CreateRWLayer("some-mount_name", layer3b.ChainID(), runtime.GOOS, nil); err == nil {
if _, err := ls2.CreateRWLayer("some-mount_name", layer3b.ChainID(), nil); err == nil {
t.Fatal("Expected error creating mount with same name")
} else if err != ErrMountNameConflict {
t.Fatal(err)
@ -500,13 +498,13 @@ func TestTarStreamStability(t *testing.T) {
t.Fatal(err)
}
layer1, err := ls.Register(bytes.NewReader(tar1), "", runtime.GOOS)
layer1, err := ls.Register(bytes.NewReader(tar1), "")
if err != nil {
t.Fatal(err)
}
// hack layer to add file
p, err := ls.(*layerStore).drivers[runtime.GOOS].Get(layer1.(*referencedCacheLayer).cacheID, "")
p, err := ls.(*layerStore).driver.Get(layer1.(*referencedCacheLayer).cacheID, "")
if err != nil {
t.Fatal(err)
}
@ -515,11 +513,11 @@ func TestTarStreamStability(t *testing.T) {
t.Fatal(err)
}
if err := ls.(*layerStore).drivers[runtime.GOOS].Put(layer1.(*referencedCacheLayer).cacheID); err != nil {
if err := ls.(*layerStore).driver.Put(layer1.(*referencedCacheLayer).cacheID); err != nil {
t.Fatal(err)
}
layer2, err := ls.Register(bytes.NewReader(tar2), layer1.ChainID(), runtime.GOOS)
layer2, err := ls.Register(bytes.NewReader(tar2), layer1.ChainID())
if err != nil {
t.Fatal(err)
}
@ -687,12 +685,12 @@ func TestRegisterExistingLayer(t *testing.T) {
t.Fatal(err)
}
layer2a, err := ls.Register(bytes.NewReader(tar1), layer1.ChainID(), runtime.GOOS)
layer2a, err := ls.Register(bytes.NewReader(tar1), layer1.ChainID())
if err != nil {
t.Fatal(err)
}
layer2b, err := ls.Register(bytes.NewReader(tar1), layer1.ChainID(), runtime.GOOS)
layer2b, err := ls.Register(bytes.NewReader(tar1), layer1.ChainID())
if err != nil {
t.Fatal(err)
}
@ -727,12 +725,12 @@ func TestTarStreamVerification(t *testing.T) {
t.Fatal(err)
}
layer1, err := ls.Register(bytes.NewReader(tar1), "", runtime.GOOS)
layer1, err := ls.Register(bytes.NewReader(tar1), "")
if err != nil {
t.Fatal(err)
}
layer2, err := ls.Register(bytes.NewReader(tar2), "", runtime.GOOS)
layer2, err := ls.Register(bytes.NewReader(tar2), "")
if err != nil {
t.Fatal(err)
}

3
layer/layer_unix_test.go
View file

@ -3,7 +3,6 @@
package layer
import (
"runtime"
"testing"
)
@ -13,7 +12,7 @@ func graphDiffSize(ls Store, l Layer) (int64, error) {
if cl.parent != nil {
parent = cl.parent.cacheID
}
return ls.(*layerStore).drivers[runtime.GOOS].DiffSize(cl.cacheID, parent)
return ls.(*layerStore).driver.DiffSize(cl.cacheID, parent)
}
// Unix as Windows graph driver does not support Changes which is indirectly

6
layer/layer_windows.go
View file

@ -25,15 +25,15 @@ func GetLayerPath(s Store, layer ChainID) (string, error) {
return "", ErrLayerDoesNotExist
}
if layerGetter, ok := ls.drivers[rl.os].(Getter); ok {
if layerGetter, ok := ls.driver.(Getter); ok {
return layerGetter.GetLayerPath(rl.cacheID)
}
path, err := ls.drivers[rl.os].Get(rl.cacheID, "")
path, err := ls.driver.Get(rl.cacheID, "")
if err != nil {
return "", err
}
if err := ls.drivers[rl.os].Put(rl.cacheID); err != nil {
if err := ls.driver.Put(rl.cacheID); err != nil {
return "", err
}

22
layer/migration.go
View file

@ -6,7 +6,6 @@ import (
"fmt"
"io"
"os"
"runtime"
"github.com/opencontainers/go-digest"
"github.com/sirupsen/logrus"
@ -17,7 +16,7 @@ import (
// CreateRWLayerByGraphID creates a RWLayer in the layer store using
// the provided name with the given graphID. To get the RWLayer
// after migration the layer may be retrieved by the given name.
func (ls *layerStore) CreateRWLayerByGraphID(name, graphID, os string, parent ChainID) (err error) {
func (ls *layerStore) CreateRWLayerByGraphID(name, graphID string, parent ChainID) (err error) {
ls.mountL.Lock()
defer ls.mountL.Unlock()
m, ok := ls.mounts[name]
@ -32,11 +31,7 @@ func (ls *layerStore) CreateRWLayerByGraphID(name, graphID, os string, parent Ch
return nil
}
// Ensure the operating system is set to the host OS if not populated.
if os == "" {
os = runtime.GOOS
}
if !ls.drivers[os].Exists(graphID) {
if !ls.driver.Exists(graphID) {
return fmt.Errorf("graph ID does not exist: %q", graphID)
}
@ -65,12 +60,11 @@ func (ls *layerStore) CreateRWLayerByGraphID(name, graphID, os string, parent Ch
mountID: graphID,
layerStore: ls,
references: map[RWLayer]*referencedRWLayer{},
os: os,
}
// Check for existing init layer
initID := fmt.Sprintf("%s-init", graphID)
if ls.drivers[os].Exists(initID) {
if ls.driver.Exists(initID) {
m.initID = initID
}
@ -101,10 +95,7 @@ func (ls *layerStore) ChecksumForGraphID(id, parent, oldTarDataPath, newTarDataP
}
dgst := digest.Canonical.Digester()
// Note - we use the host OS here. This is a safe assumption as its during migration, and
// no host OS which supports migration also supports multiple image OS's. In other words,
// it's only on Linux, not on Windows.
err = ls.assembleTarTo(id, runtime.GOOS, uncompressed, &size, dgst.Hash())
err = ls.assembleTarTo(id, uncompressed, &size, dgst.Hash())
if err != nil {
return
}
@ -120,10 +111,7 @@ func (ls *layerStore) ChecksumForGraphID(id, parent, oldTarDataPath, newTarDataP
}
func (ls *layerStore) checksumForGraphIDNoTarsplit(id, parent, newTarDataPath string) (diffID DiffID, size int64, err error) {
// Note - we use the host OS here. This is a safe assumption as its during migration, and
// no host OS which supports migration also supports multiple image OS's. In other words,
// it's only on Linux, not on Windows.
rawarchive, err := ls.drivers[runtime.GOOS].Diff(id, parent)
rawarchive, err := ls.driver.Diff(id, parent)
if err != nil {
return
}

22
layer/migration_test.go
View file

@ -94,9 +94,7 @@ func TestLayerMigration(t *testing.T) {
if err != nil {
t.Fatal(err)
}
graphs := make(map[string]graphdriver.Driver)
graphs[runtime.GOOS] = graph
ls, err := NewStoreFromGraphDrivers(fms, graphs)
ls, err := NewStoreFromGraphDriver(fms, graph, runtime.GOOS)
if err != nil {
t.Fatal(err)
}
@ -112,14 +110,14 @@ func TestLayerMigration(t *testing.T) {
t.Fatal(err)
}
layer1b, err := ls.Register(bytes.NewReader(tar1), "", runtime.GOOS)
layer1b, err := ls.Register(bytes.NewReader(tar1), "")
if err != nil {
t.Fatal(err)
}
assertReferences(t, layer1a, layer1b)
// Attempt register, should be same
layer2a, err := ls.Register(bytes.NewReader(tar2), layer1a.ChainID(), runtime.GOOS)
layer2a, err := ls.Register(bytes.NewReader(tar2), layer1a.ChainID())
if err != nil {
t.Fatal(err)
}
@ -224,9 +222,7 @@ func TestLayerMigrationNoTarsplit(t *testing.T) {
if err != nil {
t.Fatal(err)
}
graphs := make(map[string]graphdriver.Driver)
graphs[runtime.GOOS] = graph
ls, err := NewStoreFromGraphDrivers(fms, graphs)
ls, err := NewStoreFromGraphDriver(fms, graph, runtime.GOOS)
if err != nil {
t.Fatal(err)
}
@ -242,7 +238,7 @@ func TestLayerMigrationNoTarsplit(t *testing.T) {
t.Fatal(err)
}
layer1b, err := ls.Register(bytes.NewReader(tar1), "", runtime.GOOS)
layer1b, err := ls.Register(bytes.NewReader(tar1), "")
if err != nil {
t.Fatal(err)
}
@ -250,7 +246,7 @@ func TestLayerMigrationNoTarsplit(t *testing.T) {
assertReferences(t, layer1a, layer1b)
// Attempt register, should be same
layer2a, err := ls.Register(bytes.NewReader(tar2), layer1a.ChainID(), runtime.GOOS)
layer2a, err := ls.Register(bytes.NewReader(tar2), layer1a.ChainID())
if err != nil {
t.Fatal(err)
}
@ -312,7 +308,7 @@ func TestMountMigration(t *testing.T) {
t.Fatal(err)
}
graph := ls.(*layerStore).drivers[runtime.GOOS]
graph := ls.(*layerStore).driver
layer1, err := createLayer(ls, "", initWithFiles(baseFiles...))
if err != nil {
@ -338,7 +334,7 @@ func TestMountMigration(t *testing.T) {
t.Fatal(err)
}
if err := ls.(*layerStore).CreateRWLayerByGraphID("migration-mount", containerID, runtime.GOOS, layer1.ChainID()); err != nil {
if err := ls.(*layerStore).CreateRWLayerByGraphID("migration-mount", containerID, layer1.ChainID()); err != nil {
t.Fatal(err)
}
@ -384,7 +380,7 @@ func TestMountMigration(t *testing.T) {
Kind: archive.ChangeAdd,
})
if _, err := ls.CreateRWLayer("migration-mount", layer1.ChainID(), runtime.GOOS, nil); err == nil {
if _, err := ls.CreateRWLayer("migration-mount", layer1.ChainID(), nil); err == nil {
t.Fatal("Expected error creating mount with same name")
} else if err != ErrMountNameConflict {
t.Fatal(err)

6
layer/mount_test.go
View file

@ -35,7 +35,7 @@ func TestMountInit(t *testing.T) {
rwLayerOpts := &CreateRWLayerOpts{
InitFunc: mountInit,
}
m, err := ls.CreateRWLayer("fun-mount", layer.ChainID(), runtime.GOOS, rwLayerOpts)
m, err := ls.CreateRWLayer("fun-mount", layer.ChainID(), rwLayerOpts)
if err != nil {
t.Fatal(err)
}
@ -95,7 +95,7 @@ func TestMountSize(t *testing.T) {
InitFunc: mountInit,
}
m, err := ls.CreateRWLayer("mount-size", layer.ChainID(), runtime.GOOS, rwLayerOpts)
m, err := ls.CreateRWLayer("mount-size", layer.ChainID(), rwLayerOpts)
if err != nil {
t.Fatal(err)
}
@ -147,7 +147,7 @@ func TestMountChanges(t *testing.T) {
InitFunc: mountInit,
}
m, err := ls.CreateRWLayer("mount-changes", layer.ChainID(), runtime.GOOS, rwLayerOpts)
m, err := ls.CreateRWLayer("mount-changes", layer.ChainID(), rwLayerOpts)
if err != nil {
t.Fatal(err)
}

20
layer/mounted_layer.go
View file

@ -2,7 +2,6 @@ package layer
import (
"io"
"runtime"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/containerfs"
@ -15,7 +14,6 @@ type mountedLayer struct {
parent *roLayer
path string
layerStore *layerStore
os string
references map[RWLayer]*referencedRWLayer
}
@ -31,7 +29,7 @@ func (ml *mountedLayer) cacheParent() string {
}
func (ml *mountedLayer) TarStream() (io.ReadCloser, error) {
return ml.layerStore.drivers[ml.OS()].Diff(ml.mountID, ml.cacheParent())
return ml.layerStore.driver.Diff(ml.mountID, ml.cacheParent())
}
func (ml *mountedLayer) Name() string {
@ -49,23 +47,19 @@ func (ml *mountedLayer) Parent() Layer {
}
func (ml *mountedLayer) OS() string {
// For backwards compatibility, return the host OS if not set.
if ml.os == "" {
return runtime.GOOS
}
return ml.os
return ml.layerStore.os
}
func (ml *mountedLayer) Size() (int64, error) {
return ml.layerStore.drivers[ml.OS()].DiffSize(ml.mountID, ml.cacheParent())
return ml.layerStore.driver.DiffSize(ml.mountID, ml.cacheParent())
}
func (ml *mountedLayer) Changes() ([]archive.Change, error) {
return ml.layerStore.drivers[ml.OS()].Changes(ml.mountID, ml.cacheParent())
return ml.layerStore.driver.Changes(ml.mountID, ml.cacheParent())
}
func (ml *mountedLayer) Metadata() (map[string]string, error) {
return ml.layerStore.drivers[ml.OS()].GetMetadata(ml.mountID)
return ml.layerStore.driver.GetMetadata(ml.mountID)
}
func (ml *mountedLayer) getReference() RWLayer {
@ -100,11 +94,11 @@ type referencedRWLayer struct {
}
func (rl *referencedRWLayer) Mount(mountLabel string) (containerfs.ContainerFS, error) {
return rl.layerStore.drivers[rl.OS()].Get(rl.mountedLayer.mountID, mountLabel)
return rl.layerStore.driver.Get(rl.mountedLayer.mountID, mountLabel)
}
// Unmount decrements the activity count and unmounts the underlying layer
// Callers should only call `Unmount` once per call to `Mount`, even on error.
func (rl *referencedRWLayer) Unmount() error {
return rl.layerStore.drivers[rl.OS()].Put(rl.mountedLayer.mountID)
return rl.layerStore.driver.Put(rl.mountedLayer.mountID)
}

15
layer/ro_layer.go
View file

@ -16,7 +16,6 @@ type roLayer struct {
size int64
layerStore *layerStore
descriptor distribution.Descriptor
os string
referenceCount int
references map[Layer]struct{}
@ -52,7 +51,7 @@ func (rl *roLayer) TarStreamFrom(parent ChainID) (io.ReadCloser, error) {
if parent != ChainID("") && parentCacheID == "" {
return nil, fmt.Errorf("layer ID '%s' is not a parent of the specified layer: cannot provide diff to non-parent", parent)
}
return rl.layerStore.drivers[rl.OS()].Diff(rl.cacheID, parentCacheID)
return rl.layerStore.driver.Diff(rl.cacheID, parentCacheID)
}
func (rl *roLayer) ChainID() ChainID {
@ -70,6 +69,10 @@ func (rl *roLayer) Parent() Layer {
return rl.parent
}
func (rl *roLayer) OS() string {
return rl.layerStore.os
}
func (rl *roLayer) Size() (size int64, err error) {
if rl.parent != nil {
size, err = rl.parent.Size()
@ -86,7 +89,7 @@ func (rl *roLayer) DiffSize() (size int64, err error) {
}
func (rl *roLayer) Metadata() (map[string]string, error) {
return rl.layerStore.drivers[rl.OS()].GetMetadata(rl.cacheID)
return rl.layerStore.driver.GetMetadata(rl.cacheID)
}
type referencedCacheLayer struct {
@ -143,7 +146,11 @@ func storeLayer(tx MetadataTransaction, layer *roLayer) error {
return err
}
}
return tx.SetOS(layer.os)
if err := tx.SetOS(layer.layerStore.os); err != nil {
return err
}
return nil
}
func newVerifiedReadCloser(rc io.ReadCloser, dgst digest.Digest) (io.ReadCloser, error) {

9
layer/ro_layer_unix.go
View file

@ -1,9 +0,0 @@
// +build !windows
package layer
import "runtime"
func (rl *roLayer) OS() string {
return runtime.GOOS
}

7
layer/ro_layer_windows.go
View file

@ -7,10 +7,3 @@ var _ distribution.Describable = &roLayer{}
func (rl *roLayer) Descriptor() distribution.Descriptor {
return rl.descriptor
}
func (rl *roLayer) OS() string {
if rl.os == "" {
return "windows"
}
return rl.os
}

14
migrate/v1/migratev1_test.go
View file

@ -87,14 +87,15 @@ func TestMigrateContainers(t *testing.T) {
t.Fatal(err)
}
ls := &mockMounter{}
ifs, err := image.NewFSStoreBackend(filepath.Join(tmpdir, "imagedb"))
if err != nil {
t.Fatal(err)
}
is, err := image.NewImageStore(ifs, ls)
ls := &mockMounter{}
mmMap := make(map[string]image.LayerGetReleaser)
mmMap[runtime.GOOS] = ls
is, err := image.NewImageStore(ifs, mmMap)
if err != nil {
t.Fatal(err)
}
@ -165,14 +166,15 @@ func TestMigrateImages(t *testing.T) {
t.Fatal(err)
}
ls := &mockRegistrar{}
ifs, err := image.NewFSStoreBackend(filepath.Join(tmpdir, "imagedb"))
if err != nil {
t.Fatal(err)
}
is, err := image.NewImageStore(ifs, ls)
ls := &mockRegistrar{}
mrMap := make(map[string]image.LayerGetReleaser)
mrMap[runtime.GOOS] = ls
is, err := image.NewImageStore(ifs, mrMap)
if err != nil {
t.Fatal(err)
}

11
pkg/system/lcow.go
View file

@ -56,3 +56,14 @@ func ParsePlatform(in string) *specs.Platform {
}
return p
}
// IsOSSupported determines if an operating system is supported by the host
func IsOSSupported(os string) bool {
if runtime.GOOS == os {
return true
}
if LCOWSupported() && runtime.GOOS == "windows" && os == "linux" {
return true
}
return false
}

6
plugin/backend_linux.go
View file

@ -10,6 +10,7 @@ import (
"os"
"path"
"path/filepath"
"runtime"
"strings"
"github.com/docker/distribution/manifest/schema2"
@ -440,7 +441,8 @@ func (pm *Manager) Push(ctx context.Context, name string, metaHeader http.Header
pm: pm,
plugin: p,
}
ls := &pluginLayerProvider{
lss := make(map[string]distribution.PushLayerProvider)
lss[runtime.GOOS] = &pluginLayerProvider{
pm: pm,
plugin: p,
}
@ -463,7 +465,7 @@ func (pm *Manager) Push(ctx context.Context, name string, metaHeader http.Header
RequireSchema2: true,
},
ConfigMediaType: schema2.MediaTypePluginConfig,
LayerStore: ls,
LayerStores: lss,
UploadManager: uploadManager,
}