123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441 |
- package distribution
- import (
- "context"
- "encoding/json"
- "os"
- "strings"
- "sync"
- "testing"
- "github.com/containerd/containerd/content"
- "github.com/containerd/containerd/content/local"
- cerrdefs "github.com/containerd/containerd/errdefs"
- "github.com/containerd/containerd/remotes"
- "github.com/distribution/reference"
- "github.com/docker/distribution"
- "github.com/docker/distribution/manifest/manifestlist"
- "github.com/docker/distribution/manifest/ocischema"
- "github.com/docker/distribution/manifest/schema1"
- "github.com/docker/distribution/manifest/schema2"
- "github.com/google/go-cmp/cmp/cmpopts"
- "github.com/opencontainers/go-digest"
- ocispec "github.com/opencontainers/image-spec/specs-go/v1"
- "github.com/pkg/errors"
- "gotest.tools/v3/assert"
- "gotest.tools/v3/assert/cmp"
- )
- type mockManifestGetter struct {
- manifests map[digest.Digest]distribution.Manifest
- gets int
- }
- func (m *mockManifestGetter) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) {
- m.gets++
- manifest, ok := m.manifests[dgst]
- if !ok {
- return nil, distribution.ErrManifestUnknown{Tag: dgst.String()}
- }
- return manifest, nil
- }
- func (m *mockManifestGetter) Exists(ctx context.Context, dgst digest.Digest) (bool, error) {
- _, ok := m.manifests[dgst]
- return ok, nil
- }
- type memoryLabelStore struct {
- mu sync.Mutex
- labels map[digest.Digest]map[string]string
- }
- // Get returns all the labels for the given digest
- func (s *memoryLabelStore) Get(dgst digest.Digest) (map[string]string, error) {
- s.mu.Lock()
- labels := s.labels[dgst]
- s.mu.Unlock()
- return labels, nil
- }
- // Set sets all the labels for a given digest
- func (s *memoryLabelStore) Set(dgst digest.Digest, labels map[string]string) error {
- s.mu.Lock()
- if s.labels == nil {
- s.labels = make(map[digest.Digest]map[string]string)
- }
- s.labels[dgst] = labels
- s.mu.Unlock()
- return nil
- }
- // Update replaces the given labels for a digest,
- // a key with an empty value removes a label.
- func (s *memoryLabelStore) Update(dgst digest.Digest, update map[string]string) (map[string]string, error) {
- s.mu.Lock()
- defer s.mu.Unlock()
- labels, ok := s.labels[dgst]
- if !ok {
- labels = map[string]string{}
- }
- for k, v := range update {
- labels[k] = v
- }
- if s.labels == nil {
- s.labels = map[digest.Digest]map[string]string{}
- }
- s.labels[dgst] = labels
- return labels, nil
- }
- type testingContentStoreWrapper struct {
- ContentStore
- errorOnWriter error
- errorOnCommit error
- }
- func (s *testingContentStoreWrapper) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) {
- if s.errorOnWriter != nil {
- return nil, s.errorOnWriter
- }
- w, err := s.ContentStore.Writer(ctx, opts...)
- if err != nil {
- return nil, err
- }
- if s.errorOnCommit != nil {
- w = &testingContentWriterWrapper{w, s.errorOnCommit}
- }
- return w, nil
- }
- type testingContentWriterWrapper struct {
- content.Writer
- err error
- }
- func (w *testingContentWriterWrapper) Commit(ctx context.Context, size int64, dgst digest.Digest, opts ...content.Opt) error {
- if w.err != nil {
- // The contract for `Commit` is to always close.
- // Since this is returning early before hitting the real `Commit`, we should close it here.
- w.Close()
- return w.err
- }
- return w.Writer.Commit(ctx, size, dgst, opts...)
- }
- func TestManifestStore(t *testing.T) {
- ociManifest := &ocispec.Manifest{}
- serialized, err := json.Marshal(ociManifest)
- assert.NilError(t, err)
- dgst := digest.Canonical.FromBytes(serialized)
- setupTest := func(t *testing.T) (reference.Named, ocispec.Descriptor, *mockManifestGetter, *manifestStore, content.Store, func(*testing.T)) {
- root, err := os.MkdirTemp("", strings.ReplaceAll(t.Name(), "/", "_"))
- assert.NilError(t, err)
- defer func() {
- if t.Failed() {
- os.RemoveAll(root)
- }
- }()
- cs, err := local.NewLabeledStore(root, &memoryLabelStore{})
- assert.NilError(t, err)
- mg := &mockManifestGetter{manifests: make(map[digest.Digest]distribution.Manifest)}
- store := &manifestStore{local: cs, remote: mg}
- desc := ocispec.Descriptor{Digest: dgst, MediaType: ocispec.MediaTypeImageManifest, Size: int64(len(serialized))}
- ref, err := reference.Parse("foo/bar")
- assert.NilError(t, err)
- return ref.(reference.Named), desc, mg, store, cs, func(t *testing.T) {
- assert.Check(t, os.RemoveAll(root))
- }
- }
- ctx := context.Background()
- m, _, err := distribution.UnmarshalManifest(ocispec.MediaTypeImageManifest, serialized)
- assert.NilError(t, err)
- writeManifest := func(t *testing.T, cs ContentStore, desc ocispec.Descriptor, opts ...content.Opt) {
- ingestKey := remotes.MakeRefKey(ctx, desc)
- w, err := cs.Writer(ctx, content.WithDescriptor(desc), content.WithRef(ingestKey))
- assert.NilError(t, err)
- defer func() {
- if err := w.Close(); err != nil {
- t.Log(err)
- }
- if t.Failed() {
- if err := cs.Abort(ctx, ingestKey); err != nil {
- t.Log(err)
- }
- }
- }()
- _, err = w.Write(serialized)
- assert.NilError(t, err)
- err = w.Commit(ctx, desc.Size, desc.Digest, opts...)
- assert.NilError(t, err)
- }
- // All tests should end up with no active ingest
- checkIngest := func(t *testing.T, cs content.Store, desc ocispec.Descriptor) {
- ingestKey := remotes.MakeRefKey(ctx, desc)
- _, err := cs.Status(ctx, ingestKey)
- assert.Check(t, cerrdefs.IsNotFound(err), err)
- }
- t.Run("no remote or local", func(t *testing.T) {
- ref, desc, _, store, cs, teardown := setupTest(t)
- defer teardown(t)
- _, err = store.Get(ctx, desc, ref)
- checkIngest(t, cs, desc)
- // This error is what our digest getter returns when it doesn't know about the manifest
- assert.Error(t, err, distribution.ErrManifestUnknown{Tag: dgst.String()}.Error())
- })
- t.Run("no local cache", func(t *testing.T) {
- ref, desc, mg, store, cs, teardown := setupTest(t)
- defer teardown(t)
- mg.manifests[desc.Digest] = m
- m2, err := store.Get(ctx, desc, ref)
- checkIngest(t, cs, desc)
- assert.NilError(t, err)
- assert.Check(t, cmp.DeepEqual(m, m2, cmpopts.IgnoreUnexported(ocischema.DeserializedManifest{})))
- assert.Check(t, cmp.Equal(mg.gets, 1))
- i, err := cs.Info(ctx, desc.Digest)
- assert.NilError(t, err)
- assert.Check(t, cmp.Equal(i.Digest, desc.Digest))
- distKey, distSource := makeDistributionSourceLabel(ref)
- assert.Check(t, hasDistributionSource(i.Labels[distKey], distSource))
- // Now check again, this should not hit the remote
- m2, err = store.Get(ctx, desc, ref)
- checkIngest(t, cs, desc)
- assert.NilError(t, err)
- assert.Check(t, cmp.DeepEqual(m, m2, cmpopts.IgnoreUnexported(ocischema.DeserializedManifest{})))
- assert.Check(t, cmp.Equal(mg.gets, 1))
- t.Run("digested", func(t *testing.T) {
- ref, err := reference.WithDigest(ref, desc.Digest)
- assert.NilError(t, err)
- _, err = store.Get(ctx, desc, ref)
- assert.NilError(t, err)
- })
- })
- t.Run("with local cache", func(t *testing.T) {
- ref, desc, mg, store, cs, teardown := setupTest(t)
- defer teardown(t)
- // first add the manifest to the coontent store
- writeManifest(t, cs, desc)
- // now do the get
- m2, err := store.Get(ctx, desc, ref)
- checkIngest(t, cs, desc)
- assert.NilError(t, err)
- assert.Check(t, cmp.DeepEqual(m, m2, cmpopts.IgnoreUnexported(ocischema.DeserializedManifest{})))
- assert.Check(t, cmp.Equal(mg.gets, 0))
- i, err := cs.Info(ctx, desc.Digest)
- assert.NilError(t, err)
- assert.Check(t, cmp.Equal(i.Digest, desc.Digest))
- })
- // This is for the case of pull by digest where we don't know the media type of the manifest until it's actually pulled.
- t.Run("unknown media type", func(t *testing.T) {
- t.Run("no cache", func(t *testing.T) {
- ref, desc, mg, store, cs, teardown := setupTest(t)
- defer teardown(t)
- mg.manifests[desc.Digest] = m
- desc.MediaType = ""
- m2, err := store.Get(ctx, desc, ref)
- checkIngest(t, cs, desc)
- assert.NilError(t, err)
- assert.Check(t, cmp.DeepEqual(m, m2, cmpopts.IgnoreUnexported(ocischema.DeserializedManifest{})))
- assert.Check(t, cmp.Equal(mg.gets, 1))
- })
- t.Run("with cache", func(t *testing.T) {
- t.Run("cached manifest has media type", func(t *testing.T) {
- ref, desc, mg, store, cs, teardown := setupTest(t)
- defer teardown(t)
- writeManifest(t, cs, desc)
- desc.MediaType = ""
- m2, err := store.Get(ctx, desc, ref)
- checkIngest(t, cs, desc)
- assert.NilError(t, err)
- assert.Check(t, cmp.DeepEqual(m, m2, cmpopts.IgnoreUnexported(ocischema.DeserializedManifest{})))
- assert.Check(t, cmp.Equal(mg.gets, 0))
- })
- t.Run("cached manifest has no media type", func(t *testing.T) {
- ref, desc, mg, store, cs, teardown := setupTest(t)
- defer teardown(t)
- desc.MediaType = ""
- writeManifest(t, cs, desc)
- m2, err := store.Get(ctx, desc, ref)
- checkIngest(t, cs, desc)
- assert.NilError(t, err)
- assert.Check(t, cmp.DeepEqual(m, m2, cmpopts.IgnoreUnexported(ocischema.DeserializedManifest{})))
- assert.Check(t, cmp.Equal(mg.gets, 0))
- })
- })
- })
- // Test that if there is an error with the content store, for whatever
- // reason, that doesn't stop us from getting the manifest.
- //
- // Also makes sure the ingests are aborted.
- t.Run("error persisting manifest", func(t *testing.T) {
- t.Run("error on writer", func(t *testing.T) {
- ref, desc, mg, store, cs, teardown := setupTest(t)
- defer teardown(t)
- mg.manifests[desc.Digest] = m
- csW := &testingContentStoreWrapper{ContentStore: store.local, errorOnWriter: errors.New("random error")}
- store.local = csW
- m2, err := store.Get(ctx, desc, ref)
- checkIngest(t, cs, desc)
- assert.NilError(t, err)
- assert.Check(t, cmp.DeepEqual(m, m2, cmpopts.IgnoreUnexported(ocischema.DeserializedManifest{})))
- assert.Check(t, cmp.Equal(mg.gets, 1))
- _, err = cs.Info(ctx, desc.Digest)
- // Nothing here since we couldn't persist
- assert.Check(t, cerrdefs.IsNotFound(err), err)
- })
- t.Run("error on commit", func(t *testing.T) {
- ref, desc, mg, store, cs, teardown := setupTest(t)
- defer teardown(t)
- mg.manifests[desc.Digest] = m
- csW := &testingContentStoreWrapper{ContentStore: store.local, errorOnCommit: errors.New("random error")}
- store.local = csW
- m2, err := store.Get(ctx, desc, ref)
- checkIngest(t, cs, desc)
- assert.NilError(t, err)
- assert.Check(t, cmp.DeepEqual(m, m2, cmpopts.IgnoreUnexported(ocischema.DeserializedManifest{})))
- assert.Check(t, cmp.Equal(mg.gets, 1))
- _, err = cs.Info(ctx, desc.Digest)
- // Nothing here since we couldn't persist
- assert.Check(t, cerrdefs.IsNotFound(err), err)
- })
- })
- }
- func TestDetectManifestBlobMediaType(t *testing.T) {
- type testCase struct {
- json []byte
- expected string
- }
- cases := map[string]testCase{
- "mediaType is set": {[]byte(`{"mediaType": "bananas"}`), "bananas"},
- "oci manifest": {[]byte(`{"config": {}}`), ocispec.MediaTypeImageManifest},
- "schema1": {[]byte(`{"fsLayers": []}`), schema1.MediaTypeManifest},
- "oci index fallback": {[]byte(`{}`), ocispec.MediaTypeImageIndex},
- // Make sure we prefer mediaType
- "mediaType and config set": {[]byte(`{"mediaType": "bananas", "config": {}}`), "bananas"},
- "mediaType and fsLayers set": {[]byte(`{"mediaType": "bananas", "fsLayers": []}`), "bananas"},
- }
- for name, tc := range cases {
- t.Run(name, func(t *testing.T) {
- mt, err := detectManifestBlobMediaType(tc.json)
- assert.NilError(t, err)
- assert.Equal(t, mt, tc.expected)
- })
- }
- }
- func TestDetectManifestBlobMediaTypeInvalid(t *testing.T) {
- type testCase struct {
- json []byte
- expected string
- }
- cases := map[string]testCase{
- "schema 1 mediaType with manifests": {
- []byte(`{"mediaType": "` + schema1.MediaTypeManifest + `","manifests":[]}`),
- `media-type: "application/vnd.docker.distribution.manifest.v1+json" should not have "manifests" or "layers"`,
- },
- "schema 1 mediaType with layers": {
- []byte(`{"mediaType": "` + schema1.MediaTypeManifest + `","layers":[]}`),
- `media-type: "application/vnd.docker.distribution.manifest.v1+json" should not have "manifests" or "layers"`,
- },
- "schema 2 mediaType with manifests": {
- []byte(`{"mediaType": "` + schema2.MediaTypeManifest + `","manifests":[]}`),
- `media-type: "application/vnd.docker.distribution.manifest.v2+json" should not have "manifests" or "fsLayers"`,
- },
- "schema 2 mediaType with fsLayers": {
- []byte(`{"mediaType": "` + schema2.MediaTypeManifest + `","fsLayers":[]}`),
- `media-type: "application/vnd.docker.distribution.manifest.v2+json" should not have "manifests" or "fsLayers"`,
- },
- "oci manifest mediaType with manifests": {
- []byte(`{"mediaType": "` + ocispec.MediaTypeImageManifest + `","manifests":[]}`),
- `media-type: "application/vnd.oci.image.manifest.v1+json" should not have "manifests" or "fsLayers"`,
- },
- "manifest list mediaType with fsLayers": {
- []byte(`{"mediaType": "` + manifestlist.MediaTypeManifestList + `","fsLayers":[]}`),
- `media-type: "application/vnd.docker.distribution.manifest.list.v2+json" should not have "config", "layers", or "fsLayers"`,
- },
- "index mediaType with layers": {
- []byte(`{"mediaType": "` + ocispec.MediaTypeImageIndex + `","layers":[]}`),
- `media-type: "application/vnd.oci.image.index.v1+json" should not have "config", "layers", or "fsLayers"`,
- },
- "index mediaType with config": {
- []byte(`{"mediaType": "` + ocispec.MediaTypeImageIndex + `","config":{}}`),
- `media-type: "application/vnd.oci.image.index.v1+json" should not have "config", "layers", or "fsLayers"`,
- },
- "config and manifests": {
- []byte(`{"config":{}, "manifests":[]}`),
- `media-type: cannot determine`,
- },
- "layers and manifests": {
- []byte(`{"layers":[], "manifests":[]}`),
- `media-type: cannot determine`,
- },
- "layers and fsLayers": {
- []byte(`{"layers":[], "fsLayers":[]}`),
- `media-type: cannot determine`,
- },
- "fsLayers and manifests": {
- []byte(`{"fsLayers":[], "manifests":[]}`),
- `media-type: cannot determine`,
- },
- "config and fsLayers": {
- []byte(`{"config":{}, "fsLayers":[]}`),
- `media-type: cannot determine`,
- },
- }
- for name, tc := range cases {
- t.Run(name, func(t *testing.T) {
- mt, err := detectManifestBlobMediaType(tc.json)
- assert.Error(t, err, tc.expected)
- assert.Equal(t, mt, "")
- })
- }
- }
|