beenull/moby
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Move to a single tag-store

Browse source 
Signed-off-by: John Howard <jhoward@microsoft.com>
This commit is contained in:
John Howard 2017-08-17 15:43:36 -07:00
parent 8bee1e9a3b
commit 7b9a8f460b
14 changed files with 54 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
daemon/daemon.go
View file

@ -75,7 +75,6 @@ type daemonStore struct {
imageStore image.Store
layerStore layer.Store
distributionMetadataStore dmetadata.Store
referenceStore refstore.Store
}
// Daemon holds information about the Docker daemon.
@ -103,6 +102,7 @@ type Daemon struct {
shutdown bool
idMappings *idtools.IDMappings
stores map[string]daemonStore // By container target platform
referenceStore refstore.Store
PluginStore *plugin.Store // todo: remove
pluginManager *plugin.Manager
linkIndex *linkIndex
@ -691,7 +691,6 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe
d.downloadManager = xfer.NewLayerDownloadManager(lsMap, *config.MaxConcurrentDownloads)
logrus.Debugf("Max Concurrent Uploads: %d", *config.MaxConcurrentUploads)
d.uploadManager = xfer.NewLayerUploadManager(*config.MaxConcurrentUploads)
for platform, ds := range d.stores {
imageRoot := filepath.Join(config.Root, "image", ds.graphDriver)
ifs, err := image.NewFSStoreBackend(filepath.Join(imageRoot, "imagedb"))
@ -728,18 +727,30 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe
eventsService := events.New()
// We have a single tag/reference store for the daemon globally. However, it's
// stored under the graphdriver. On host platforms which only support a single
// container OS, but multiple selectable graphdrivers, this means depending on which
// graphdriver is chosen, the global reference store is under there. For
// platforms which support multiple container operating systems, this is slightly
// more problematic as where does the global ref store get located? Fortunately,
// for Windows, which is currently the only daemon supporting multiple container
// operating systems, the list of graphdrivers available isn't user configurable.
// For backwards compatibility, we just put it under the windowsfilter
// directory regardless.
refStoreLocation := filepath.Join(d.stores[runtime.GOOS].imageRoot, `repositories.json`)
rs, err := refstore.NewReferenceStore(refStoreLocation)
if err != nil {
return nil, fmt.Errorf("Couldn't create reference store repository: %s", err)
}
d.referenceStore = rs
for platform, ds := range d.stores {
dms, err := dmetadata.NewFSMetadataStore(filepath.Join(ds.imageRoot, "distribution"), platform)
if err != nil {
return nil, err
}
rs, err := refstore.NewReferenceStore(filepath.Join(ds.imageRoot, "repositories.json"), platform)
if err != nil {
return nil, fmt.Errorf("Couldn't create Tag store repositories: %s", err)
}
ds.distributionMetadataStore = dms
ds.referenceStore = rs
d.stores[platform] = ds
// No content-addressability migration on Windows as it never supported pre-CA

2
daemon/disk_usage.go
View file

@ -20,7 +20,7 @@ func (daemon *Daemon) getLayerRefs(platform string) map[layer.ChainID]int {
layerRefs := map[layer.ChainID]int{}
for id, img := range tmpImages {
dgst := digest.Digest(id)
if len(daemon.stores[platform].referenceStore.References(dgst)) == 0 && len(daemon.stores[platform].imageStore.Children(id)) != 0 {
if len(daemon.referenceStore.References(dgst)) == 0 && len(daemon.stores[platform].imageStore.Children(id)) != 0 {
continue
}

14
daemon/image.go
View file

@ -2,6 +2,7 @@ package daemon
import (
"fmt"
"runtime"
"github.com/docker/distribution/reference"
"github.com/docker/docker/image"
@ -45,18 +46,25 @@ func (daemon *Daemon) GetImageIDAndPlatform(refOrID string) (image.ID, string, e
return "", "", errImageDoesNotExist{ref}
}
if digest, err := daemon.referenceStore.Get(namedRef); err == nil {
// Search the image stores to get the platform, defaulting to host OS.
imagePlatform := runtime.GOOS
id := image.IDFromDigest(digest)
for platform := range daemon.stores {
if id, err := daemon.stores[platform].referenceStore.Get(namedRef); err == nil {
return image.IDFromDigest(id), platform, nil
if img, err := daemon.stores[platform].imageStore.Get(id); err == nil {
imagePlatform = img.Platform()
break
}
}
return id, imagePlatform, nil
}
// deprecated: repo:shortid https://github.com/docker/docker/pull/799
if tagged, ok := namedRef.(reference.Tagged); ok {
if tag := tagged.Tag(); stringid.IsShortID(stringid.TruncateID(tag)) {
for platform := range daemon.stores {
if id, err := daemon.stores[platform].imageStore.Search(tag); err == nil {
for _, storeRef := range daemon.stores[platform].referenceStore.References(id.Digest()) {
for _, storeRef := range daemon.referenceStore.References(id.Digest()) {
if storeRef.Name() == namedRef.Name() {
return id, platform, nil
}

12
daemon/image_delete.go
View file

@ -70,7 +70,7 @@ func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.I
return nil, err
}
repoRefs := daemon.stores[platform].referenceStore.References(imgID.Digest())
repoRefs := daemon.referenceStore.References(imgID.Digest())
var removedRepositoryRef bool
if !isImageIDPrefix(imgID.String(), imageRef) {
@ -104,7 +104,7 @@ func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.I
daemon.LogImageEvent(imgID.String(), imgID.String(), "untag")
records = append(records, untaggedRecord)
repoRefs = daemon.stores[platform].referenceStore.References(imgID.Digest())
repoRefs = daemon.referenceStore.References(imgID.Digest())
// If a tag reference was removed and the only remaining
// references to the same repository are digest references,
@ -237,7 +237,7 @@ func (daemon *Daemon) removeImageRef(platform string, ref reference.Named) (refe
// Ignore the boolean value returned, as far as we're concerned, this
// is an idempotent operation and it's okay if the reference didn't
// exist in the first place.
_, err := daemon.stores[platform].referenceStore.Delete(ref)
_, err := daemon.referenceStore.Delete(ref)
return ref, err
}
@ -248,7 +248,7 @@ func (daemon *Daemon) removeImageRef(platform string, ref reference.Named) (refe
// daemon's event service. An "Untagged" types.ImageDeleteResponseItem is added to the
// given list of records.
func (daemon *Daemon) removeAllReferencesToImageID(imgID image.ID, platform string, records *[]types.ImageDeleteResponseItem) error {
imageRefs := daemon.stores[platform].referenceStore.References(imgID.Digest())
imageRefs := daemon.referenceStore.References(imgID.Digest())
for _, imageRef := range imageRefs {
parsedRef, err := daemon.removeImageRef(platform, imageRef)
@ -383,7 +383,7 @@ func (daemon *Daemon) checkImageDeleteConflict(imgID image.ID, platform string,
}
// Check if any repository tags/digest reference this image.
if mask&conflictActiveReference != 0 && len(daemon.stores[platform].referenceStore.References(imgID.Digest())) > 0 {
if mask&conflictActiveReference != 0 && len(daemon.referenceStore.References(imgID.Digest())) > 0 {
return &imageDeleteConflict{
imgID: imgID,
message: "image is referenced in multiple repositories",
@ -411,5 +411,5 @@ func (daemon *Daemon) checkImageDeleteConflict(imgID image.ID, platform string,
// that there are no repository references to the given image and it has no
// child images.
func (daemon *Daemon) imageIsDangling(imgID image.ID, platform string) bool {
return !(len(daemon.stores[platform].referenceStore.References(imgID.Digest())) > 0 || len(daemon.stores[platform].imageStore.Children(imgID)) > 0)
return !(len(daemon.referenceStore.References(imgID.Digest())) > 0 || len(daemon.stores[platform].imageStore.Children(imgID)) > 0)
}

4
daemon/image_exporter.go
View file

@ -19,7 +19,7 @@ func (daemon *Daemon) ExportImage(names []string, outStream io.Writer) error {
if system.LCOWSupported() {
platform = "linux"
}
imageExporter := tarexport.NewTarExporter(daemon.stores[platform].imageStore, daemon.stores[platform].layerStore, daemon.stores[platform].referenceStore, daemon)
imageExporter := tarexport.NewTarExporter(daemon.stores[platform].imageStore, daemon.stores[platform].layerStore, daemon.referenceStore, daemon)
return imageExporter.Save(names, outStream)
}
@ -32,6 +32,6 @@ func (daemon *Daemon) LoadImage(inTar io.ReadCloser, outStream io.Writer, quiet
if system.LCOWSupported() {
platform = "linux"
}
imageExporter := tarexport.NewTarExporter(daemon.stores[platform].imageStore, daemon.stores[platform].layerStore, daemon.stores[platform].referenceStore, daemon)
imageExporter := tarexport.NewTarExporter(daemon.stores[platform].imageStore, daemon.stores[platform].layerStore, daemon.referenceStore, daemon)
return imageExporter.Load(inTar, outStream, quiet)
}

2
daemon/image_history.go
View file

@ -69,7 +69,7 @@ func (daemon *Daemon) ImageHistory(name string) ([]*image.HistoryResponseItem, e
h.ID = id.String()
var tags []string
for _, r := range daemon.stores[platform].referenceStore.References(id.Digest()) {
for _, r := range daemon.referenceStore.References(id.Digest()) {
if _, ok := r.(reference.NamedTagged); ok {
tags = append(tags, reference.FamiliarString(r))
}

2
daemon/image_inspect.go
View file

@ -24,7 +24,7 @@ func (daemon *Daemon) LookupImage(name string) (*types.ImageInspect, error) {
platform = runtime.GOOS
}
refs := daemon.stores[platform].referenceStore.References(img.ID().Digest())
refs := daemon.referenceStore.References(img.ID().Digest())
repoTags := []string{}
repoDigests := []string{}
for _, ref := range refs {

2
daemon/image_pull.go
View file

@ -74,7 +74,7 @@ func (daemon *Daemon) pullImageWithReference(ctx context.Context, ref reference.
ImageEventLogger: daemon.LogImageEvent,
MetadataStore: daemon.stores[platform].distributionMetadataStore,
ImageStore: distribution.NewImageConfigStoreFromStore(daemon.stores[platform].imageStore),
ReferenceStore: daemon.stores[platform].referenceStore,
ReferenceStore: daemon.referenceStore,
},
DownloadManager: daemon.downloadManager,
Schema2Types: distribution.ImageTypes,

2
daemon/image_push.go
View file

@ -56,7 +56,7 @@ func (daemon *Daemon) PushImage(ctx context.Context, image, tag string, metaHead
ImageEventLogger: daemon.LogImageEvent,
MetadataStore: daemon.stores[platform].distributionMetadataStore,
ImageStore: distribution.NewImageConfigStoreFromStore(daemon.stores[platform].imageStore),
ReferenceStore: daemon.stores[platform].referenceStore,
ReferenceStore: daemon.referenceStore,
},
ConfigMediaType: schema2.MediaTypeImageConfig,
LayerStore: distribution.NewLayerProviderFromStore(daemon.stores[platform].layerStore),

2
daemon/image_tag.go
View file

@ -28,7 +28,7 @@ func (daemon *Daemon) TagImage(imageName, repository, tag string) error {
// TagImageWithReference adds the given reference to the image ID provided.
func (daemon *Daemon) TagImageWithReference(imageID image.ID, platform string, newTag reference.Named) error {
if err := daemon.stores[platform].referenceStore.AddTag(newTag, imageID.Digest(), true); err != nil {
if err := daemon.referenceStore.AddTag(newTag, imageID.Digest(), true); err != nil {
return err
}

2
daemon/images.go
View file

@ -149,7 +149,7 @@ func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs
newImage := newImage(img, size)
for _, ref := range daemon.stores[platform].referenceStore.References(id.Digest()) {
for _, ref := range daemon.referenceStore.References(id.Digest()) {
if imageFilters.Include("reference") {
var found bool
var matchErr error

4
daemon/prune.go
View file

@ -221,7 +221,7 @@ func (daemon *Daemon) ImagesPrune(ctx context.Context, pruneFilters filters.Args
return nil, ctx.Err()
default:
dgst := digest.Digest(id)
if len(daemon.stores[platform].referenceStore.References(dgst)) == 0 && len(daemon.stores[platform].imageStore.Children(id)) != 0 {
if len(daemon.referenceStore.References(dgst)) == 0 && len(daemon.stores[platform].imageStore.Children(id)) != 0 {
continue
}
if !until.IsZero() && img.Created.After(until) {
@ -252,7 +252,7 @@ deleteImagesLoop:
}
deletedImages := []types.ImageDeleteResponseItem{}
refs := daemon.stores[platform].referenceStore.References(dgst)
refs := daemon.referenceStore.References(dgst)
if len(refs) > 0 {
shouldDelete := !danglingOnly
if !shouldDelete {

8
reference/store.go
View file

@ -26,7 +26,7 @@ type Association struct {
ID digest.Digest
}
// Store provides the set of methods which can operate on a tag store.
// Store provides the set of methods which can operate on a reference store.
type Store interface {
References(id digest.Digest) []reference.Named
ReferencesByName(ref reference.Named) []Association
@ -46,9 +46,6 @@ type store struct {
// referencesByIDCache is a cache of references indexed by ID, to speed
// up References.
referencesByIDCache map[digest.Digest]map[string]reference.Named
// platform is the container target platform for this store (which may be
// different to the host operating system
platform string
}
// Repository maps tags to digests. The key is a stringified Reference,
@ -73,7 +70,7 @@ func (a lexicalAssociations) Less(i, j int) bool {
// NewReferenceStore creates a new reference store, tied to a file path where
// the set of references are serialized in JSON format.
func NewReferenceStore(jsonPath, platform string) (Store, error) {
func NewReferenceStore(jsonPath string) (Store, error) {
abspath, err := filepath.Abs(jsonPath)
if err != nil {
return nil, err
@ -83,7 +80,6 @@ func NewReferenceStore(jsonPath, platform string) (Store, error) {
jsonPath: abspath,
Repositories: make(map[string]repository),
referencesByIDCache: make(map[digest.Digest]map[string]reference.Named),
platform: platform,
}
// Load the json file if it exists, otherwise create it.
if err := store.reload(); os.IsNotExist(err) {

9
reference/store_test.go
View file

@ -5,7 +5,6 @@ import (
"io/ioutil"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
@ -41,7 +40,7 @@ func TestLoad(t *testing.T) {
}
jsonFile.Close()
store, err := NewReferenceStore(jsonFile.Name(), runtime.GOOS)
store, err := NewReferenceStore(jsonFile.Name())
if err != nil {
t.Fatalf("error creating tag store: %v", err)
}
@ -70,7 +69,7 @@ func TestSave(t *testing.T) {
jsonFile.Close()
defer os.RemoveAll(jsonFile.Name())
store, err := NewReferenceStore(jsonFile.Name(), runtime.GOOS)
store, err := NewReferenceStore(jsonFile.Name())
if err != nil {
t.Fatalf("error creating tag store: %v", err)
}
@ -112,7 +111,7 @@ func TestAddDeleteGet(t *testing.T) {
jsonFile.Close()
defer os.RemoveAll(jsonFile.Name())
store, err := NewReferenceStore(jsonFile.Name(), runtime.GOOS)
store, err := NewReferenceStore(jsonFile.Name())
if err != nil {
t.Fatalf("error creating tag store: %v", err)
}
@ -329,7 +328,7 @@ func TestInvalidTags(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "tag-store-test")
defer os.RemoveAll(tmpDir)
store, err := NewReferenceStore(filepath.Join(tmpDir, "repositories.json"), runtime.GOOS)
store, err := NewReferenceStore(filepath.Join(tmpDir, "repositories.json"))
if err != nil {
t.Fatalf("error creating tag store: %v", err)
}