0ct0pu5/moby
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #47449 from vvoland/c8d-list-single

Browse source 
c8d/list: Add test and combine size
This commit is contained in:
Paweł Gronowski 2024-03-07 18:49:19 +01:00 • committed by GitHub
commit f4c696eef1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
20 changed files with 623 additions and 194 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
daemon/containerd/image.go
View file

@ -178,7 +178,7 @@ func (i *ImageService) GetImageManifest(ctx context.Context, refOrID string, opt
platform = platforms.Only(*options.Platform)
}
cs := i.client.ContentStore()
cs := i.content
img, err := i.resolveImage(ctx, refOrID)
if err != nil {
@ -237,7 +237,7 @@ func (i *ImageService) GetImageManifest(ctx context.Context, refOrID string, opt
func (i *ImageService) size(ctx context.Context, desc ocispec.Descriptor, platform platforms.MatchComparer) (int64, error) {
var size int64
cs := i.client.ContentStore()
cs := i.content
handler := containerdimages.LimitManifests(containerdimages.ChildrenHandler(cs), platform, 1)
var wh containerdimages.HandlerFunc = func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
@ -360,7 +360,7 @@ func (i *ImageService) resolveImage(ctx context.Context, refOrID string) (contai
// pointing to the same repository as the given reference.
func (i *ImageService) getAllImagesWithRepository(ctx context.Context, ref reference.Named) ([]containerdimages.Image, error) {
nameFilter := "^" + regexp.QuoteMeta(ref.Name()) + ":" + reference.TagRegexp.String() + "$"
return i.client.ImageService().List(ctx, "name~="+strconv.Quote(nameFilter))
return i.images.List(ctx, "name~="+strconv.Quote(nameFilter))
}
func imageFamiliarName(img containerdimages.Image) string {
@ -378,7 +378,7 @@ func imageFamiliarName(img containerdimages.Image) string {
// targeting the specified digest.
// If images have different values, an errdefs.Conflict error will be returned.
func (i *ImageService) getImageLabelByDigest(ctx context.Context, target digest.Digest, labelKey string) (string, error) {
imgs, err := i.client.ImageService().List(ctx, "target.digest=="+target.String()+",labels."+labelKey)
imgs, err := i.images.List(ctx, "target.digest=="+target.String()+",labels."+labelKey)
if err != nil {
return "", errdefs.System(err)
}

14
daemon/containerd/image_builder.go
View file

@ -202,12 +202,12 @@ func newROLayerForImage(ctx context.Context, imgDesc *ocispec.Descriptor, i *Ima
platMatcher = platforms.Only(*platform)
}
confDesc, err := containerdimages.Config(ctx, i.client.ContentStore(), *imgDesc, platMatcher)
confDesc, err := containerdimages.Config(ctx, i.content, *imgDesc, platMatcher)
if err != nil {
return nil, err
}
diffIDs, err := containerdimages.RootFS(ctx, i.client.ContentStore(), confDesc)
diffIDs, err := containerdimages.RootFS(ctx, i.content, confDesc)
if err != nil {
return nil, err
}
@ -446,7 +446,7 @@ func (i *ImageService) CreateImage(ctx context.Context, config []byte, parent st
if err != nil {
return nil, err
}
parentImageManifest, err := containerdimages.Manifest(ctx, i.client.ContentStore(), parentDesc, platforms.Default())
parentImageManifest, err := containerdimages.Manifest(ctx, i.content, parentDesc, platforms.Default())
if err != nil {
return nil, err
}
@ -455,7 +455,7 @@ func (i *ImageService) CreateImage(ctx context.Context, config []byte, parent st
parentDigest = parentDesc.Digest
}
cs := i.client.ContentStore()
cs := i.content
ra, err := cs.ReaderAt(ctx, ocispec.Descriptor{Digest: layerDigest})
if err != nil {
@ -504,7 +504,7 @@ func (i *ImageService) createImageOCI(ctx context.Context, imgToCreate imagespec
}
}()
manifestDesc, ccDesc, err := writeContentsForImage(ctx, i.snapshotter, i.client.ContentStore(), imgToCreate, layers, containerConfig)
manifestDesc, ccDesc, err := writeContentsForImage(ctx, i.snapshotter, i.content, imgToCreate, layers, containerConfig)
if err != nil {
return "", err
}
@ -523,13 +523,13 @@ func (i *ImageService) createImageOCI(ctx context.Context, imgToCreate imagespec
img.Labels[imageLabelClassicBuilderFromScratch] = "1"
}
createdImage, err := i.client.ImageService().Update(ctx, img)
createdImage, err := i.images.Update(ctx, img)
if err != nil {
if !cerrdefs.IsNotFound(err) {
return "", err
}
if createdImage, err = i.client.ImageService().Create(ctx, img); err != nil {
if createdImage, err = i.images.Create(ctx, img); err != nil {
return "", fmt.Errorf("failed to create new image: %w", err)
}
}

2
daemon/containerd/image_commit.go
View file

@ -38,7 +38,7 @@ with adaptations to match the Moby data model and services.
// CommitImage creates a new image from a commit config.
func (i *ImageService) CommitImage(ctx context.Context, cc backend.CommitConfig) (image.ID, error) {
container := i.containers.Get(cc.ContainerID)
cs := i.client.ContentStore()
cs := i.content
var parentManifest ocispec.Manifest
var parentImage imagespec.DockerOCIImage

2
daemon/containerd/image_delete.go
View file

@ -375,7 +375,7 @@ func (i *ImageService) imageDeleteHelper(ctx context.Context, img images.Image,
CreatedAt: time.Now(),
Labels: img.Labels,
}
if _, err = i.client.ImageService().Create(ctx, img); err != nil && !cerrdefs.IsAlreadyExists(err) {
if _, err = i.images.Create(ctx, img); err != nil && !cerrdefs.IsAlreadyExists(err) {
return fmt.Errorf("failed to create dangling image: %w", err)
}
}

5
daemon/containerd/image_history.go
View file

@ -84,13 +84,12 @@ func (i *ImageService) ImageHistory(ctx context.Context, name string) ([]*imaget
return imgs
}
is := i.client.ImageService()
currentImg := img
for _, h := range history {
dgst := currentImg.Target.Digest.String()
h.ID = dgst
imgs, err := is.List(ctx, "target.digest=="+dgst)
imgs, err := i.images.List(ctx, "target.digest=="+dgst)
if err != nil {
return nil, err
}
@ -157,5 +156,5 @@ func (i *ImageService) getParentsByBuilderLabel(ctx context.Context, img contain
return nil, nil
}
return i.client.ImageService().List(ctx, "target.digest=="+dgst.String())
return i.images.List(ctx, "target.digest=="+dgst.String())
}

8
daemon/containerd/image_import.go
View file

@ -66,7 +66,7 @@ func (i *ImageService) ImportImage(ctx context.Context, ref reference.Named, pla
return "", errdefs.InvalidParameter(err)
}
cs := i.client.ContentStore()
cs := i.content
compressedDigest, uncompressedDigest, mt, err := saveArchive(ctx, cs, layerReader)
if err != nil {
@ -299,11 +299,9 @@ func writeBlobAndReturnDigest(ctx context.Context, cs content.Store, mt string,
// saveImage creates an image in the ImageService or updates it if it exists.
func (i *ImageService) saveImage(ctx context.Context, img images.Image) error {
is := i.client.ImageService()
if _, err := is.Update(ctx, img); err != nil {
if _, err := i.images.Update(ctx, img); err != nil {
if cerrdefs.IsNotFound(err) {
if _, err := is.Create(ctx, img); err != nil {
if _, err := i.images.Create(ctx, img); err != nil {
return errdefs.Unknown(err)
}
} else {

255
daemon/containerd/image_list.go
View file

@ -11,6 +11,7 @@ import (
cerrdefs "github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/labels"
"github.com/containerd/containerd/platforms"
cplatforms "github.com/containerd/containerd/platforms"
"github.com/containerd/containerd/snapshots"
"github.com/containerd/log"
@ -21,7 +22,6 @@ import (
timetypes "github.com/docker/docker/api/types/time"
"github.com/docker/docker/container"
"github.com/docker/docker/errdefs"
"github.com/docker/docker/image"
dockerspec "github.com/moby/docker-image-spec/specs-go/v1"
"github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/identity"
@ -59,7 +59,6 @@ func (r byCreated) Less(i, j int) bool { return r[i].Created < r[j].Created }
// Images returns a filtered list of images.
//
// TODO(thaJeztah): implement opts.ContainerCount (used for docker system df); see https://github.com/moby/moby/issues/43853
// TODO(thaJeztah): verify behavior of `RepoDigests` and `RepoTags` for images without (untagged) or multiple tags; see https://github.com/moby/moby/issues/43861
// TODO(thaJeztah): verify "Size" vs "VirtualSize" in images; see https://github.com/moby/moby/issues/43862
func (i *ImageService) Images(ctx context.Context, opts imagetypes.ListOptions) ([]*imagetypes.Summary, error) {
@ -72,13 +71,13 @@ func (i *ImageService) Images(ctx context.Context, opts imagetypes.ListOptions)
return nil, err
}
imgs, err := i.client.ImageService().List(ctx)
imgs, err := i.images.List(ctx)
if err != nil {
return nil, err
}
// TODO(thaJeztah): do we need to take multiple snapshotters into account? See https://github.com/moby/moby/issues/45273
snapshotter := i.client.SnapshotService(i.snapshotter)
snapshotter := i.snapshotterService(i.snapshotter)
sizeCache := make(map[digest.Digest]int64)
snapshotSizeFn := func(d digest.Digest) (int64, error) {
if s, ok := sizeCache[d]; ok {
@ -93,17 +92,15 @@ func (i *ImageService) Images(ctx context.Context, opts imagetypes.ListOptions)
}
var (
allContainers []*container.Container
summaries = make([]*imagetypes.Summary, 0, len(imgs))
root []*[]digest.Digest
layers map[digest.Digest]int
summaries = make([]*imagetypes.Summary, 0, len(imgs))
root []*[]digest.Digest
layers map[digest.Digest]int
)
if opts.SharedSize {
root = make([]*[]digest.Digest, 0, len(imgs))
layers = make(map[digest.Digest]int)
}
contentStore := i.client.ContentStore()
uniqueImages := map[digest.Digest]images.Image{}
tagsByDigest := map[digest.Digest][]string{}
intermediateImages := map[digest.Digest]struct{}{}
@ -155,88 +152,17 @@ func (i *ImageService) Images(ctx context.Context, opts imagetypes.ListOptions)
tagsByDigest[dgst] = append(tagsByDigest[dgst], reference.FamiliarString(ref))
}
if opts.ContainerCount {
allContainers = i.containers.List()
}
type tempImage struct {
img *ImageManifest
indexPlatform *ocispec.Platform
dockerImage *dockerspec.DockerOCIImage
}
for _, img := range uniqueImages {
var presentImages []tempImage
err := i.walkImageManifests(ctx, img, func(img *ImageManifest) error {
if isPseudo, err := img.IsPseudoImage(ctx); isPseudo || err != nil {
return err
}
available, err := img.CheckContentAvailable(ctx)
if err != nil {
log.G(ctx).WithFields(log.Fields{
"error": err,
"manifest": img.Target(),
"image": img.Name(),
}).Warn("checking availability of platform specific manifest failed")
return nil
}
if !available {
return nil
}
conf, err := img.Config(ctx)
if err != nil {
return err
}
var dockerImage dockerspec.DockerOCIImage
if err := readConfig(ctx, contentStore, conf, &dockerImage); err != nil {
return err
}
presentImages = append(presentImages, tempImage{
img: img,
indexPlatform: img.Target().Platform,
dockerImage: &dockerImage,
})
return nil
})
if err != nil {
return nil, err
}
if len(presentImages) == 0 {
// TODO we should probably show *something* for images we've pulled
// but are 100% shallow or an empty manifest list/index
// ("tianon/scratch:index" is an empty example image index and
// "tianon/scratch:list" is an empty example manifest list)
continue
}
sort.SliceStable(presentImages, func(i, j int) bool {
platformFromIndexOrConfig := func(idx int) ocispec.Platform {
if presentImages[i].indexPlatform != nil {
return *presentImages[i].indexPlatform
}
return presentImages[i].dockerImage.Platform
}
return platformMatcher.Less(platformFromIndexOrConfig(i), platformFromIndexOrConfig(j))
})
best := presentImages[0].img
image, chainIDs, err := i.singlePlatformImage(ctx, contentStore, tagsByDigest[best.RealTarget.Digest], best, opts, allContainers)
if err != nil {
image, allChainsIDs, err := i.imageSummary(ctx, img, platformMatcher, opts, tagsByDigest)
if err != nil || image == nil {
return nil, err
}
summaries = append(summaries, image)
if opts.SharedSize {
root = append(root, &chainIDs)
for _, id := range chainIDs {
root = append(root, &allChainsIDs)
for _, id := range allChainsIDs {
layers[id] = layers[id] + 1
}
}
@ -257,36 +183,157 @@ func (i *ImageService) Images(ctx context.Context, opts imagetypes.ListOptions)
return summaries, nil
}
func (i *ImageService) singlePlatformImage(ctx context.Context, contentStore content.Store, repoTags []string, imageManifest *ImageManifest, opts imagetypes.ListOptions, allContainers []*container.Container) (*imagetypes.Summary, []digest.Digest, error) {
diffIDs, err := imageManifest.RootFS(ctx)
// imageSummary returns a summary of the image, including the total size of the image and all its platforms.
// It also returns the chainIDs of all the layers of the image (including all its platforms).
// All return values will be nil if the image should be skipped.
func (i *ImageService) imageSummary(ctx context.Context, img images.Image, platformMatcher platforms.MatchComparer,
opts imagetypes.ListOptions, tagsByDigest map[digest.Digest][]string,
) (_ *imagetypes.Summary, allChainIDs []digest.Digest, _ error) {
// Total size of the image including all its platform
var totalSize int64
// ChainIDs of all the layers of the image (including all its platform)
var allChainsIDs []digest.Digest
// Count of containers using the image
var containersCount int64
// Single platform image manifest preferred by the platform matcher
var best *ImageManifest
var bestPlatform ocispec.Platform
err := i.walkImageManifests(ctx, img, func(img *ImageManifest) error {
if isPseudo, err := img.IsPseudoImage(ctx); isPseudo || err != nil {
return nil
}
available, err := img.CheckContentAvailable(ctx)
if err != nil {
log.G(ctx).WithFields(log.Fields{
"error": err,
"manifest": img.Target(),
"image": img.Name(),
}).Warn("checking availability of platform specific manifest failed")
return nil
}
if !available {
return nil
}
conf, err := img.Config(ctx)
if err != nil {
return err
}
var dockerImage dockerspec.DockerOCIImage
if err := readConfig(ctx, i.content, conf, &dockerImage); err != nil {
return err
}
target := img.Target()
chainIDs, err := img.RootFS(ctx)
if err != nil {
return err
}
ts, _, err := i.singlePlatformSize(ctx, img)
if err != nil {
return err
}
totalSize += ts
allChainsIDs = append(allChainsIDs, chainIDs...)
if opts.ContainerCount {
i.containers.ApplyAll(func(c *container.Container) {
if c.ImageManifest != nil && c.ImageManifest.Digest == target.Digest {
containersCount++
}
})
}
var platform ocispec.Platform
if target.Platform != nil {
platform = *target.Platform
} else {
platform = dockerImage.Platform
}
// Filter out platforms that don't match the requested platform. Do it
// after the size, container count and chainIDs are summed up to have
// the single combined entry still represent the whole multi-platform
// image.
if !platformMatcher.Match(platform) {
return nil
}
if best == nil || platformMatcher.Less(platform, bestPlatform) {
best = img
bestPlatform = platform
}
return nil
})
if err != nil {
return nil, nil, errors.Wrapf(err, "failed to get rootfs of image %s", imageManifest.Name())
return nil, nil, err
}
if best == nil {
// TODO we should probably show *something* for images we've pulled
// but are 100% shallow or an empty manifest list/index
// ("tianon/scratch:index" is an empty example image index and
// "tianon/scratch:list" is an empty example manifest list)
return nil, nil, nil
}
image, err := i.singlePlatformImage(ctx, i.content, tagsByDigest[best.RealTarget.Digest], best)
if err != nil {
return nil, nil, err
}
image.Size = totalSize
if opts.ContainerCount {
image.Containers = containersCount
}
return image, allChainsIDs, nil
}
func (i *ImageService) singlePlatformSize(ctx context.Context, imgMfst *ImageManifest) (totalSize int64, contentSize int64, _ error) {
// TODO(thaJeztah): do we need to take multiple snapshotters into account? See https://github.com/moby/moby/issues/45273
snapshotter := i.client.SnapshotService(i.snapshotter)
snapshotter := i.snapshotterService(i.snapshotter)
diffIDs, err := imgMfst.RootFS(ctx)
if err != nil {
return -1, -1, errors.Wrapf(err, "failed to get rootfs of image %s", imgMfst.Name())
}
imageSnapshotID := identity.ChainID(diffIDs).String()
unpackedUsage, err := calculateSnapshotTotalUsage(ctx, snapshotter, imageSnapshotID)
if err != nil {
if !cerrdefs.IsNotFound(err) {
log.G(ctx).WithError(err).WithFields(log.Fields{
"image": imageManifest.Name(),
"image": imgMfst.Name(),
"snapshotID": imageSnapshotID,
}).Warn("failed to calculate unpacked size of image")
}
unpackedUsage = snapshots.Usage{Size: 0}
}
contentSize, err := imageManifest.Size(ctx)
contentSize, err = imgMfst.Size(ctx)
if err != nil {
return nil, nil, err
return -1, -1, err
}
// totalSize is the size of the image's packed layers and snapshots
// (unpacked layers) combined.
totalSize := contentSize + unpackedUsage.Size
totalSize = contentSize + unpackedUsage.Size
return totalSize, contentSize, nil
}
func (i *ImageService) singlePlatformImage(ctx context.Context, contentStore content.Store, repoTags []string, imageManifest *ImageManifest) (*imagetypes.Summary, error) {
var repoDigests []string
rawImg := imageManifest.Metadata()
target := rawImg.Target.Digest
@ -318,11 +365,16 @@ func (i *ImageService) singlePlatformImage(ctx context.Context, contentStore con
cfgDesc, err := imageManifest.Image.Config(ctx)
if err != nil {
return nil, nil, err
return nil, err
}
var cfg configLabels
if err := readConfig(ctx, contentStore, cfgDesc, &cfg); err != nil {
return nil, nil, err
return nil, err
}
totalSize, _, err := i.singlePlatformSize(ctx, imageManifest)
if err != nil {
return nil, errors.Wrapf(err, "failed to calculate size of image %s", imageManifest.Name())
}
summary := &imagetypes.Summary{
@ -343,18 +395,7 @@ func (i *ImageService) singlePlatformImage(ctx context.Context, contentStore con
summary.Created = cfg.Created.Unix()
}
if opts.ContainerCount {
// Get container count
var containers int64
for _, c := range allContainers {
if c.ImageID == image.ID(target.String()) {
containers++
}
}
summary.Containers = containers
}
return summary, identity.ChainIDs(diffIDs), nil
return summary, nil
}
type imageFilterFunc func(image images.Image) bool
@ -426,7 +467,7 @@ func (i *ImageService) setupFilters(ctx context.Context, imageFilters filters.Ar
return nil, err
}
labelFn, err := setupLabelFilter(i.client.ContentStore(), imageFilters)
labelFn, err := setupLabelFilter(i.content, imageFilters)
if err != nil {
return nil, err
}

243
daemon/containerd/image_list_test.go Normal file
View file

@ -0,0 +1,243 @@
package containerd
import (
"context"
"fmt"
"os"
"path/filepath"
"sort"
"testing"
"github.com/containerd/containerd"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/metadata"
"github.com/containerd/containerd/namespaces"
"github.com/containerd/containerd/snapshots"
"github.com/containerd/log/logtest"
imagetypes "github.com/docker/docker/api/types/image"
daemonevents "github.com/docker/docker/daemon/events"
"github.com/docker/docker/internal/testutils/specialimage"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"gotest.tools/v3/assert"
is "gotest.tools/v3/assert/cmp"
)
func imagesFromIndex(index ...*ocispec.Index) []images.Image {
var imgs []images.Image
for _, idx := range index {
for _, desc := range idx.Manifests {
imgs = append(imgs, images.Image{
Name: desc.Annotations["io.containerd.image.name"],
Target: desc,
})
}
}
return imgs
}
func TestImageList(t *testing.T) {
ctx := namespaces.WithNamespace(context.TODO(), "testing")
blobsDir := t.TempDir()
multilayer, err := specialimage.MultiLayer(blobsDir)
assert.NilError(t, err)
twoplatform, err := specialimage.TwoPlatform(blobsDir)
assert.NilError(t, err)
cs := &blobsDirContentStore{blobs: filepath.Join(blobsDir, "blobs/sha256")}
snapshotter := &testSnapshotterService{}
for _, tc := range []struct {
name string
images []images.Image
opts imagetypes.ListOptions
check func(*testing.T, []*imagetypes.Summary) // Change the type of the check function
}{
{
name: "one multi-layer image",
images: imagesFromIndex(multilayer),
check: func(t *testing.T, all []*imagetypes.Summary) { // Change the type of the check function
assert.Check(t, is.Len(all, 1))
assert.Check(t, is.Equal(all[0].ID, multilayer.Manifests[0].Digest.String()))
assert.Check(t, is.DeepEqual(all[0].RepoTags, []string{"multilayer:latest"}))
},
},
{
name: "one image with two platforms is still one entry",
images: imagesFromIndex(twoplatform),
check: func(t *testing.T, all []*imagetypes.Summary) { // Change the type of the check function
assert.Check(t, is.Len(all, 1))
assert.Check(t, is.Equal(all[0].ID, twoplatform.Manifests[0].Digest.String()))
assert.Check(t, is.DeepEqual(all[0].RepoTags, []string{"twoplatform:latest"}))
},
},
{
name: "two images are two entries",
images: imagesFromIndex(multilayer, twoplatform),
check: func(t *testing.T, all []*imagetypes.Summary) { // Change the type of the check function
assert.Check(t, is.Len(all, 2))
assert.Check(t, is.Equal(all[0].ID, multilayer.Manifests[0].Digest.String()))
assert.Check(t, is.DeepEqual(all[0].RepoTags, []string{"multilayer:latest"}))
assert.Check(t, is.Equal(all[1].ID, twoplatform.Manifests[0].Digest.String()))
assert.Check(t, is.DeepEqual(all[1].RepoTags, []string{"twoplatform:latest"}))
},
},
} {
tc := tc
t.Run(tc.name, func(t *testing.T) {
ctx := logtest.WithT(ctx, t)
mdb := newTestDB(ctx, t)
snapshotters := map[string]snapshots.Snapshotter{
containerd.DefaultSnapshotter: snapshotter,
}
service := &ImageService{
images: metadata.NewImageStore(mdb),
containers: emptyTestContainerStore(),
content: cs,
eventsService: daemonevents.New(),
snapshotterServices: snapshotters,
snapshotter: containerd.DefaultSnapshotter,
}
// containerd.Image gets the services directly from containerd.Client
// so we need to create a "fake" containerd.Client with the test services.
c8dCli, err := containerd.New("", containerd.WithServices(
containerd.WithImageStore(service.images),
containerd.WithContentStore(cs),
containerd.WithSnapshotters(snapshotters),
))
assert.NilError(t, err)
service.client = c8dCli
for _, img := range tc.images {
_, err := service.images.Create(ctx, img)
assert.NilError(t, err)
}
all, err := service.Images(ctx, tc.opts)
assert.NilError(t, err)
sort.Slice(all, func(i, j int) bool {
firstTag := func(idx int) string {
if len(all[idx].RepoTags) > 0 {
return all[idx].RepoTags[0]
}
return ""
}
return firstTag(i) < firstTag(j)
})
tc.check(t, all)
})
}
}
type blobsDirContentStore struct {
blobs string
}
type fileReaderAt struct {
*os.File
}
func (f *fileReaderAt) Size() int64 {
fi, err := f.Stat()
if err != nil {
return -1
}
return fi.Size()
}
func (s *blobsDirContentStore) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content.ReaderAt, error) {
p := filepath.Join(s.blobs, desc.Digest.Encoded())
r, err := os.Open(p)
if err != nil {
return nil, err
}
return &fileReaderAt{r}, nil
}
func (s *blobsDirContentStore) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) {
return nil, fmt.Errorf("read-only")
}
func (s *blobsDirContentStore) Status(ctx context.Context, _ string) (content.Status, error) {
return content.Status{}, fmt.Errorf("not implemented")
}
func (s *blobsDirContentStore) Delete(ctx context.Context, dgst digest.Digest) error {
return fmt.Errorf("read-only")
}
func (s *blobsDirContentStore) ListStatuses(ctx context.Context, filters ...string) ([]content.Status, error) {
return nil, nil
}
func (s *blobsDirContentStore) Abort(ctx context.Context, ref string) error {
return fmt.Errorf("not implemented")
}
func (s *blobsDirContentStore) Walk(ctx context.Context, fn content.WalkFunc, filters ...string) error {
entries, err := os.ReadDir(s.blobs)
if err != nil {
return err
}
for _, e := range entries {
if e.IsDir() {
continue
}
d := digest.FromString(e.Name())
if d == "" {
continue
}
stat, err := e.Info()
if err != nil {
return err
}
if err := fn(content.Info{Digest: d, Size: stat.Size()}); err != nil {
return err
}
}
return nil
}
func (s *blobsDirContentStore) Info(ctx context.Context, dgst digest.Digest) (content.Info, error) {
f, err := os.Open(filepath.Join(s.blobs, dgst.Encoded()))
if err != nil {
return content.Info{}, err
}
defer f.Close()
stat, err := f.Stat()
if err != nil {
return content.Info{}, err
}
return content.Info{
Digest: dgst,
Size: stat.Size(),
}, nil
}
func (s *blobsDirContentStore) Update(ctx context.Context, info content.Info, fieldpaths ...string) (content.Info, error) {
return content.Info{}, fmt.Errorf("read-only")
}

6
daemon/containerd/image_pull.go
View file

@ -90,7 +90,7 @@ func (i *ImageService) pullTag(ctx context.Context, ref reference.Named, platfor
})
opts = append(opts, containerd.WithImageHandler(h))
pp := pullProgress{store: i.client.ContentStore(), showExists: true}
pp := pullProgress{store: i.content, showExists: true}
finishProgress := jobs.showProgress(ctx, out, pp)
var outNewImg *containerd.Image
@ -140,7 +140,7 @@ func (i *ImageService) pullTag(ctx context.Context, ref reference.Named, platfor
sentPullingFrom = true
}
available, _, _, missing, err := images.Check(ctx, i.client.ContentStore(), desc, p)
available, _, _, missing, err := images.Check(ctx, i.content, desc, p)
if err != nil {
return nil, err
}
@ -187,7 +187,7 @@ func (i *ImageService) pullTag(ctx context.Context, ref reference.Named, platfor
logger.Info("image pulled")
// The pull succeeded, so try to remove any dangling image we have for this target
err = i.client.ImageService().Delete(compatcontext.WithoutCancel(ctx), danglingImageName(img.Target().Digest))
err = i.images.Delete(compatcontext.WithoutCancel(ctx), danglingImageName(img.Target().Digest))
if err != nil && !cerrdefs.IsNotFound(err) {
// Image pull succeeded, but cleaning up the dangling image failed. Ignore the
// error to not mark the pull as failed.

4
daemon/containerd/image_push.go
View file

@ -91,7 +91,7 @@ func (i *ImageService) pushRef(ctx context.Context, targetRef reference.Named, m
}
}()
img, err := i.client.ImageService().Get(ctx, targetRef.String())
img, err := i.images.Get(ctx, targetRef.String())
if err != nil {
if cerrdefs.IsNotFound(err) {
return errdefs.NotFound(fmt.Errorf("tag does not exist: %s", reference.FamiliarString(targetRef)))
@ -100,7 +100,7 @@ func (i *ImageService) pushRef(ctx context.Context, targetRef reference.Named, m
}
target := img.Target
store := i.client.ContentStore()
store := i.content
resolver, tracker := i.newResolverFromAuthConfig(ctx, authConfig, targetRef)
pp := pushProgress{Tracker: tracker}

2
daemon/containerd/image_snapshot.go
View file

@ -26,7 +26,7 @@ func (i *ImageService) PrepareSnapshot(ctx context.Context, id string, parentIma
return err
}
cs := i.client.ContentStore()
cs := i.content
matcher := matchAllWithPreference(platforms.Default())
if platform != nil {

7
daemon/containerd/image_tag.go
View file

@ -28,8 +28,7 @@ func (i *ImageService) TagImage(ctx context.Context, imageID image.ID, newTag re
Labels: targetImage.Labels,
}
is := i.client.ImageService()
_, err = is.Create(ctx, newImg)
_, err = i.images.Create(ctx, newImg)
if err != nil {
if !cerrdefs.IsAlreadyExists(err) {
return errdefs.System(errors.Wrapf(err, "failed to create image with name %s and target %s", newImg.Name, newImg.Target.Digest.String()))
@ -54,7 +53,7 @@ func (i *ImageService) TagImage(ctx context.Context, imageID image.ID, newTag re
return errors.Wrapf(err, "failed to delete previous image %s", replacedImg.Name)
}
if _, err = is.Create(compatcontext.WithoutCancel(ctx), newImg); err != nil {
if _, err = i.images.Create(compatcontext.WithoutCancel(ctx), newImg); err != nil {
return errdefs.System(errors.Wrapf(err, "failed to create an image %s with target %s after deleting the existing one",
newImg.Name, imageID.String()))
}
@ -69,7 +68,7 @@ func (i *ImageService) TagImage(ctx context.Context, imageID image.ID, newTag re
defer i.LogImageEvent(imageID.String(), reference.FamiliarString(newTag), events.ActionTag)
// Delete the source dangling image, as it's no longer dangling.
if err := is.Delete(compatcontext.WithoutCancel(ctx), danglingImageName(targetImage.Target.Digest)); err != nil {
if err := i.images.Delete(compatcontext.WithoutCancel(ctx), danglingImageName(targetImage.Target.Digest)); err != nil {
logger.WithError(err).Warn("unexpected error when deleting dangling image")
}

13
daemon/containerd/image_test.go
View file

@ -10,6 +10,7 @@ import (
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/metadata"
"github.com/containerd/containerd/namespaces"
"github.com/containerd/containerd/snapshots"
"github.com/containerd/log/logtest"
"github.com/distribution/reference"
dockerimages "github.com/docker/docker/daemon/images"
@ -296,3 +297,15 @@ func newTestDB(ctx context.Context, t *testing.T) *metadata.DB {
return mdb
}
type testSnapshotterService struct {
snapshots.Snapshotter
}
func (s *testSnapshotterService) Stat(ctx context.Context, key string) (snapshots.Info, error) {
return snapshots.Info{}, nil
}
func (s *testSnapshotterService) Usage(ctx context.Context, key string) (snapshots.Usage, error) {
return snapshots.Usage{}, nil
}

42
daemon/containerd/service.go
View file

@ -27,17 +27,18 @@ import (
// ImageService implements daemon.ImageService
type ImageService struct {
client *containerd.Client
images images.Store
content content.Store
containers container.Store
snapshotter string
registryHosts docker.RegistryHosts
registryService registryResolver
eventsService *daemonevents.Events
pruneRunning atomic.Bool
refCountMounter snapshotter.Mounter
idMapping idtools.IdentityMapping
client *containerd.Client
images images.Store
content content.Store
containers container.Store
snapshotterServices map[string]snapshots.Snapshotter
snapshotter string
registryHosts docker.RegistryHosts
registryService registryResolver
eventsService *daemonevents.Events
pruneRunning atomic.Bool
refCountMounter snapshotter.Mounter
idMapping idtools.IdentityMapping
}
type registryResolver interface {
@ -61,9 +62,12 @@ type ImageServiceConfig struct {
// NewService creates a new ImageService.
func NewService(config ImageServiceConfig) *ImageService {
return &ImageService{
client: config.Client,
images: config.Client.ImageService(),
content: config.Client.ContentStore(),
client: config.Client,
images: config.Client.ImageService(),
content: config.Client.ContentStore(),
snapshotterServices: map[string]snapshots.Snapshotter{
config.Snapshotter: config.Client.SnapshotService(config.Snapshotter),
},
containers: config.Containers,
snapshotter: config.Snapshotter,
registryHosts: config.RegistryHosts,
@ -74,6 +78,16 @@ func NewService(config ImageServiceConfig) *ImageService {
}
}
func (i *ImageService) snapshotterService(snapshotter string) snapshots.Snapshotter {
s, ok := i.snapshotterServices[snapshotter]
if !ok {
s = i.client.SnapshotService(snapshotter)
i.snapshotterServices[snapshotter] = s
}
return s
}
// DistributionServices return services controlling daemon image storage.
func (i *ImageService) DistributionServices() dimages.DistributionServices {
return dimages.DistributionServices{}

3
integration-cli/docker_utils_test.go
View file

@ -473,7 +473,8 @@ func loadSpecialImage(c *testing.T, imageFunc specialimage.SpecialImageFunc) str
imgDir := filepath.Join(tmpDir, "image")
assert.NilError(c, os.Mkdir(imgDir, 0o755))
assert.NilError(c, imageFunc(imgDir))
_, err := imageFunc(imgDir)
assert.NilError(c, err)
rc, err := archive.TarWithOptions(imgDir, &archive.TarOptions{})
assert.NilError(c, err)

18
internal/testutils/specialimage/dangling.go
View file

@ -4,6 +4,8 @@ import (
"os"
"path/filepath"
"strings"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
const danglingImageManifestDigest = "sha256:16d365089e5c10e1673ee82ab5bba38ade9b763296ad918bd24b42a1156c5456" // #nosec G101 -- ignoring: Potential hardcoded credentials (gosec)
@ -12,30 +14,30 @@ const danglingImageConfigDigest = "sha256:0df1207206e5288f4a989a2f13d1f5b3c4e704
// Dangling creates an image with no layers and no tag.
// It also has an extra org.mobyproject.test.specialimage=1 label set.
// Layout: OCI.
func Dangling(dir string) error {
func Dangling(dir string) (*ocispec.Index, error) {
if err := os.WriteFile(filepath.Join(dir, "index.json"), []byte(`{"schemaVersion":2,"manifests":[{"mediaType":"application/vnd.docker.distribution.manifest.v2+json","digest":"sha256:16d365089e5c10e1673ee82ab5bba38ade9b763296ad918bd24b42a1156c5456","size":264,"annotations":{"org.opencontainers.image.created":"2023-05-19T08:00:44Z"},"platform":{"architecture":"amd64","os":"linux"}}]}`), 0o644); err != nil {
return err
return nil, err
}
if err := os.WriteFile(filepath.Join(dir, "manifest.json"), []byte(`[{"Config":"blobs/sha256/0df1207206e5288f4a989a2f13d1f5b3c4e70467702c1d5d21dfc9f002b7bd43","RepoTags":null,"Layers":null}]`), 0o644); err != nil {
return err
return nil, err
}
if err := os.Mkdir(filepath.Join(dir, "blobs"), 0o755); err != nil {
return err
return nil, err
}
blobsDir := filepath.Join(dir, "blobs", "sha256")
if err := os.Mkdir(blobsDir, 0o755); err != nil {
return err
return nil, err
}
if err := os.WriteFile(filepath.Join(blobsDir, strings.TrimPrefix(danglingImageManifestDigest, "sha256:")), []byte(`{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/vnd.docker.container.image.v1+json","digest":"sha256:0df1207206e5288f4a989a2f13d1f5b3c4e70467702c1d5d21dfc9f002b7bd43","size":390},"layers":[]}`), 0o644); err != nil {
return err
return nil, err
}
if err := os.WriteFile(filepath.Join(blobsDir, strings.TrimPrefix(danglingImageConfigDigest, "sha256:")), []byte(`{"architecture":"amd64","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"WorkingDir":"/","Labels":{"org.mobyproject.test.specialimage":"1"},"OnBuild":null},"created":null,"history":[{"created_by":"LABEL org.mobyproject.test.specialimage=1","comment":"buildkit.dockerfile.v0","empty_layer":true}],"os":"linux","rootfs":{"type":"layers","diff_ids":null}}`), 0o644); err != nil {
return err
return nil, err
}
return nil
return nil, nil
}

24
internal/testutils/specialimage/emptyfs.go
View file

@ -4,53 +4,55 @@ import (
"io"
"os"
"path/filepath"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
// EmptyFS builds an image with an empty rootfs.
// Layout: Legacy Docker Archive
// See https://github.com/docker/docker/pull/5262
// and also https://github.com/docker/docker/issues/4242
func EmptyFS(dir string) error {
func EmptyFS(dir string) (*ocispec.Index, error) {
if err := os.WriteFile(filepath.Join(dir, "manifest.json"), []byte(`[{"Config":"11f64303f0f7ffdc71f001788132bca5346831939a956e3e975c93267d89a16d.json","RepoTags":["emptyfs:latest"],"Layers":["511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/layer.tar"]}]`), 0o644); err != nil {
return err
return nil, err
}
if err := os.Mkdir(filepath.Join(dir, "blobs"), 0o755); err != nil {
return err
return nil, err
}
blobsDir := filepath.Join(dir, "511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158")
if err := os.Mkdir(blobsDir, 0o755); err != nil {
return err
return nil, err
}
if err := os.WriteFile(filepath.Join(dir, "VERSION"), []byte(`1.0`), 0o644); err != nil {
return err
return nil, err
}
if err := os.WriteFile(filepath.Join(dir, "repositories"), []byte(`{"emptyfs":{"latest":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158"}}`), 0o644); err != nil {
return err
return nil, err
}
if err := os.WriteFile(filepath.Join(dir, "11f64303f0f7ffdc71f001788132bca5346831939a956e3e975c93267d89a16d.json"), []byte(`{"architecture":"x86_64","comment":"Imported from -","container_config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":null,"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2013-06-13T14:03:50.821769-07:00","docker_version":"0.4.0","history":[{"created":"2013-06-13T14:03:50.821769-07:00","comment":"Imported from -"}],"rootfs":{"type":"layers","diff_ids":["sha256:84ff92691f909a05b224e1c56abb4864f01b4f8e3c854e4bb4c7baf1d3f6d652"]}}`), 0o644); err != nil {
return err
return nil, err
}
if err := os.WriteFile(filepath.Join(blobsDir, "json"), []byte(`{"id":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158","comment":"Imported from -","created":"2013-06-13T14:03:50.821769-07:00","container_config":{"Hostname":"","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":null,"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":null},"docker_version":"0.4.0","architecture":"x86_64","Size":0}`+"\n"), 0o644); err != nil {
return err
return nil, err
}
layerFile, err := os.OpenFile(filepath.Join(blobsDir, "layer.tar"), os.O_CREATE|os.O_WRONLY, 0o644)
if err != nil {
return err
return nil, err
}
defer layerFile.Close()
// 10240 NUL bytes is a valid empty tar archive.
_, err = io.Copy(layerFile, io.LimitReader(zeroReader{}, 10240))
if err != nil {
return err
return nil, err
}
return nil
return nil, nil
}
type zeroReader struct{}

5
internal/testutils/specialimage/load.go
View file

@ -12,15 +12,16 @@ import (
"github.com/docker/docker/client"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/jsonmessage"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"gotest.tools/v3/assert"
)
type SpecialImageFunc func(string) error
type SpecialImageFunc func(string) (*ocispec.Index, error)
func Load(ctx context.Context, t *testing.T, apiClient client.APIClient, imageFunc SpecialImageFunc) string {
tempDir := t.TempDir()
err := imageFunc(tempDir)
_, err := imageFunc(tempDir)
assert.NilError(t, err)
rc, err := archive.TarWithOptions(tempDir, &archive.TarOptions{})

39
internal/testutils/specialimage/multilayer.go
View file

@ -16,20 +16,20 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
func MultiLayer(dir string) error {
func MultiLayer(dir string) (*ocispec.Index, error) {
const imageRef = "multilayer:latest"
layer1Desc, err := writeLayerWithOneFile(dir, "foo", []byte("1"))
if err != nil {
return err
return nil, err
}
layer2Desc, err := writeLayerWithOneFile(dir, "bar", []byte("2"))
if err != nil {
return err
return nil, err
}
layer3Desc, err := writeLayerWithOneFile(dir, "hello", []byte("world"))
if err != nil {
return err
return nil, err
}
configDesc, err := writeJsonBlob(dir, ocispec.MediaTypeImageConfig, ocispec.Image{
@ -43,7 +43,7 @@ func MultiLayer(dir string) error {
},
})
if err != nil {
return err
return nil, err
}
manifest := ocispec.Manifest{
@ -53,7 +53,7 @@ func MultiLayer(dir string) error {
}
legacyManifests := []manifestItem{
manifestItem{
{
Config: blobPath(configDesc),
RepoTags: []string{imageRef},
Layers: []string{blobPath(layer1Desc), blobPath(layer2Desc), blobPath(layer3Desc)},
@ -62,7 +62,7 @@ func MultiLayer(dir string) error {
ref, err := reference.ParseNormalizedNamed(imageRef)
if err != nil {
return err
return nil, err
}
return singlePlatformImage(dir, ref, manifest, legacyManifests)
}
@ -74,10 +74,10 @@ type manifestItem struct {
Layers []string
}
func singlePlatformImage(dir string, ref reference.Named, manifest ocispec.Manifest, legacyManifests []manifestItem) error {
func singlePlatformImage(dir string, ref reference.Named, manifest ocispec.Manifest, legacyManifests []manifestItem) (*ocispec.Index, error) {
manifestDesc, err := writeJsonBlob(dir, ocispec.MediaTypeImageManifest, manifest)
if err != nil {
return err
return nil, err
}
if ref != nil {
@ -90,25 +90,24 @@ func singlePlatformImage(dir string, ref reference.Named, manifest ocispec.Manif
}
}
if err := writeJson(ocispec.Index{
idx := ocispec.Index{
Versioned: specs.Versioned{SchemaVersion: 2},
MediaType: ocispec.MediaTypeImageIndex,
Manifests: []ocispec.Descriptor{manifestDesc},
}, filepath.Join(dir, "index.json")); err != nil {
return err
}
if err != nil {
return err
if err := writeJson(idx, filepath.Join(dir, "index.json")); err != nil {
return nil, err
}
if err := writeJson(legacyManifests, filepath.Join(dir, "manifest.json")); err != nil {
return err
}
if err != nil {
return err
return nil, err
}
return os.WriteFile(filepath.Join(dir, "oci-layout"), []byte(`{"imageLayoutVersion": "1.0.0"}`), 0o644)
err = os.WriteFile(filepath.Join(dir, "oci-layout"), []byte(`{"imageLayoutVersion": "1.0.0"}`), 0o644)
if err != nil {
return nil, err
}
return &idx, nil
}
func fileArchive(dir string, name string, content []byte) (io.ReadCloser, error) {

117
internal/testutils/specialimage/twoplatform.go Normal file
View file

@ -0,0 +1,117 @@
package specialimage
import (
"os"
"path/filepath"
"github.com/containerd/containerd/platforms"
"github.com/distribution/reference"
"github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/specs-go"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
func TwoPlatform(dir string) (*ocispec.Index, error) {
const imageRef = "twoplatform:latest"
layer1Desc, err := writeLayerWithOneFile(dir, "bash", []byte("layer1"))
if err != nil {
return nil, err
}
layer2Desc, err := writeLayerWithOneFile(dir, "bash", []byte("layer2"))
if err != nil {
return nil, err
}
config1Desc, err := writeJsonBlob(dir, ocispec.MediaTypeImageConfig, ocispec.Image{
Platform: platforms.MustParse("linux/amd64"),
Config: ocispec.ImageConfig{
Env: []string{"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"},
},
RootFS: ocispec.RootFS{
Type: "layers",
DiffIDs: []digest.Digest{layer1Desc.Digest},
},
})
if err != nil {
return nil, err
}
manifest1Desc, err := writeJsonBlob(dir, ocispec.MediaTypeImageManifest, ocispec.Manifest{
MediaType: ocispec.MediaTypeImageManifest,
Config: config1Desc,
Layers: []ocispec.Descriptor{layer1Desc},
})
if err != nil {
return nil, err
}
config2Desc, err := writeJsonBlob(dir, ocispec.MediaTypeImageConfig, ocispec.Image{
Platform: platforms.MustParse("linux/arm64"),
Config: ocispec.ImageConfig{
Env: []string{"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"},
},
RootFS: ocispec.RootFS{
Type: "layers",
DiffIDs: []digest.Digest{layer1Desc.Digest},
},
})
if err != nil {
return nil, err
}
manifest2Desc, err := writeJsonBlob(dir, ocispec.MediaTypeImageManifest, ocispec.Manifest{
MediaType: ocispec.MediaTypeImageManifest,
Config: config2Desc,
Layers: []ocispec.Descriptor{layer2Desc},
})
if err != nil {
return nil, err
}
index := ocispec.Index{
Versioned: specs.Versioned{SchemaVersion: 2},
MediaType: ocispec.MediaTypeImageIndex,
Manifests: []ocispec.Descriptor{manifest1Desc, manifest2Desc},
}
ref, err := reference.ParseNormalizedNamed(imageRef)
if err != nil {
return nil, err
}
return multiPlatformImage(dir, ref, index)
}
func multiPlatformImage(dir string, ref reference.Named, target ocispec.Index) (*ocispec.Index, error) {
targetDesc, err := writeJsonBlob(dir, ocispec.MediaTypeImageIndex, target)
if err != nil {
return nil, err
}
if ref != nil {
targetDesc.Annotations = map[string]string{
"io.containerd.image.name": ref.String(),
}
if tagged, ok := ref.(reference.Tagged); ok {
targetDesc.Annotations[ocispec.AnnotationRefName] = tagged.Tag()
}
}
index := ocispec.Index{
Versioned: specs.Versioned{SchemaVersion: 2},
MediaType: ocispec.MediaTypeImageIndex,
Manifests: []ocispec.Descriptor{targetDesc},
}
if err := writeJson(index, filepath.Join(dir, "index.json")); err != nil {
return nil, err
}
err = os.WriteFile(filepath.Join(dir, "oci-layout"), []byte(`{"imageLayoutVersion": "1.0.0"}`), 0o644)
if err != nil {
return nil, err
}
return &index, nil
}