Merge pull request #405 from thaJeztah/19.03_backport_oci_regression

[19.03 backport] Use ocischema package instead of custom handler

distribution/oci.go

@@ -1,29 +0,0 @@
-package distribution
-
-import (
-	"fmt"
-
-	"github.com/docker/distribution"
-	"github.com/docker/distribution/manifest/schema2"
-	digest "github.com/opencontainers/go-digest"
-	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-)
-
-func init() {
-	// TODO: Remove this registration if distribution is included with OCI support
-
-	ocischemaFunc := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) {
-		m := new(schema2.DeserializedManifest)
-		err := m.UnmarshalJSON(b)
-		if err != nil {
-			return nil, distribution.Descriptor{}, err
-		}
-
-		dgst := digest.FromBytes(b)
-		return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: ocispec.MediaTypeImageManifest}, err
-	}
-	err := distribution.RegisterManifestSchema(ocispec.MediaTypeImageManifest, ocischemaFunc)
-	if err != nil {
-		panic(fmt.Sprintf("Unable to register manifest: %s", err))
-	}
-}

distribution/pull_v2.go

@@ -14,6 +14,7 @@ import (
 	"github.com/containerd/containerd/platforms"
 	"github.com/docker/distribution"
 	"github.com/docker/distribution/manifest/manifestlist"
+	"github.com/docker/distribution/manifest/ocischema"
 	"github.com/docker/distribution/manifest/schema1"
 	"github.com/docker/distribution/manifest/schema2"
 	"github.com/docker/distribution/reference"
@@ -410,6 +411,11 @@ func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named, platform
 		if err != nil {
 			return false, err
 		}
+	case *ocischema.DeserializedManifest:
+		id, manifestDigest, err = p.pullOCI(ctx, ref, v, platform)
+		if err != nil {
+			return false, err
+		}
 	case *manifestlist.DeserializedManifestList:
 		id, manifestDigest, err = p.pullManifestList(ctx, ref, v, platform)
 		if err != nil {
@@ -557,24 +563,18 @@ func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Reference, unv
 	return imageID, manifestDigest, nil
 }
 
-func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *schema2.DeserializedManifest, platform *specs.Platform) (id digest.Digest, manifestDigest digest.Digest, err error) {
-	manifestDigest, err = schema2ManifestDigest(ref, mfst)
-	if err != nil {
-		return "", "", err
-	}
-
-	target := mfst.Target()
+func (p *v2Puller) pullSchema2Layers(ctx context.Context, target distribution.Descriptor, layers []distribution.Descriptor, platform *specs.Platform) (id digest.Digest, err error) {
 	if _, err := p.config.ImageStore.Get(target.Digest); err == nil {
 		// If the image already exists locally, no need to pull
 		// anything.
-		return target.Digest, manifestDigest, nil
+		return target.Digest, nil
 	}
 
 	var descriptors []xfer.DownloadDescriptor
 
 	// Note that the order of this loop is in the direction of bottom-most
 	// to top-most, so that the downloads slice gets ordered correctly.
-	for _, d := range mfst.Layers {
+	for _, d := range layers {
 		layerDescriptor := &v2LayerDescriptor{
 			digest:            d.Digest,
 			repo:              p.repo,
@@ -629,23 +629,23 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s
 	if runtime.GOOS == "windows" {
 		configJSON, configRootFS, configPlatform, err = receiveConfig(p.config.ImageStore, configChan, configErrChan)
 		if err != nil {
-			return "", "", err
+			return "", err
 		}
 		if configRootFS == nil {
-			return "", "", errRootFSInvalid
+			return "", errRootFSInvalid
 		}
 		if err := checkImageCompatibility(configPlatform.OS, configPlatform.OSVersion); err != nil {
-			return "", "", err
+			return "", err
 		}
 
 		if len(descriptors) != len(configRootFS.DiffIDs) {
-			return "", "", errRootFSMismatch
+			return "", errRootFSMismatch
 		}
 		if platform == nil {
 			// Early bath if the requested OS doesn't match that of the configuration.
 			// This avoids doing the download, only to potentially fail later.
 			if !system.IsOSSupported(configPlatform.OS) {
-				return "", "", fmt.Errorf("cannot download image with operating system %q when requesting %q", configPlatform.OS, layerStoreOS)
+				return "", fmt.Errorf("cannot download image with operating system %q when requesting %q", configPlatform.OS, layerStoreOS)
 			}
 			layerStoreOS = configPlatform.OS
 		}
@@ -692,14 +692,14 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s
 			case <-downloadsDone:
 			case <-layerErrChan:
 			}
-			return "", "", err
+			return "", err
 		}
 	}
 
 	select {
 	case <-downloadsDone:
 	case err = <-layerErrChan:
-		return "", "", err
+		return "", err
 	}
 
 	if release != nil {
@@ -711,22 +711,40 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s
 		// Otherwise the image config could be referencing layers that aren't
 		// included in the manifest.
 		if len(downloadedRootFS.DiffIDs) != len(configRootFS.DiffIDs) {
-			return "", "", errRootFSMismatch
+			return "", errRootFSMismatch
 		}
 
 		for i := range downloadedRootFS.DiffIDs {
 			if downloadedRootFS.DiffIDs[i] != configRootFS.DiffIDs[i] {
-				return "", "", errRootFSMismatch
+				return "", errRootFSMismatch
 			}
 		}
 	}
 
 	imageID, err := p.config.ImageStore.Put(configJSON)
+	if err != nil {
+		return "", err
+	}
+
+	return imageID, nil
+}
+
+func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *schema2.DeserializedManifest, platform *specs.Platform) (id digest.Digest, manifestDigest digest.Digest, err error) {
+	manifestDigest, err = schema2ManifestDigest(ref, mfst)
 	if err != nil {
 		return "", "", err
 	}
+	id, err = p.pullSchema2Layers(ctx, mfst.Target(), mfst.Layers, platform)
+	return id, manifestDigest, err
+}
 
-	return imageID, manifestDigest, nil
+func (p *v2Puller) pullOCI(ctx context.Context, ref reference.Named, mfst *ocischema.DeserializedManifest, platform *specs.Platform) (id digest.Digest, manifestDigest digest.Digest, err error) {
+	manifestDigest, err = schema2ManifestDigest(ref, mfst)
+	if err != nil {
+		return "", "", err
+	}
+	id, err = p.pullSchema2Layers(ctx, mfst.Target(), mfst.Layers, platform)
+	return id, manifestDigest, err
 }
 
 func receiveConfig(s ImageConfigStore, configChan <-chan []byte, errChan <-chan error) ([]byte, *image.RootFS, *specs.Platform, error) {
@@ -811,6 +829,12 @@ func (p *v2Puller) pullManifestList(ctx context.Context, ref reference.Named, mf
 		if err != nil {
 			return "", "", err
 		}
+	case *ocischema.DeserializedManifest:
+		platform := toOCIPlatform(manifestMatches[0].Platform)
+		id, _, err = p.pullOCI(ctx, manifestRef, v, &platform)
+		if err != nil {
+			return "", "", err
+		}
 	default:
 		return "", "", errors.New("unsupported manifest format")
 	}

vendor/github.com/docker/distribution/manifest/ocischema/builder.go

@@ -0,0 +1,107 @@
+package ocischema
+
+import (
+	"context"
+	"errors"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/manifest"
+	"github.com/opencontainers/go-digest"
+	"github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+// Builder is a type for constructing manifests.
+type Builder struct {
+	// bs is a BlobService used to publish the configuration blob.
+	bs distribution.BlobService
+
+	// configJSON references
+	configJSON []byte
+
+	// layers is a list of layer descriptors that gets built by successive
+	// calls to AppendReference.
+	layers []distribution.Descriptor
+
+	// Annotations contains arbitrary metadata relating to the targeted content.
+	annotations map[string]string
+
+	// For testing purposes
+	mediaType string
+}
+
+// NewManifestBuilder is used to build new manifests for the current schema
+// version. It takes a BlobService so it can publish the configuration blob
+// as part of the Build process, and annotations.
+func NewManifestBuilder(bs distribution.BlobService, configJSON []byte, annotations map[string]string) distribution.ManifestBuilder {
+	mb := &Builder{
+		bs:          bs,
+		configJSON:  make([]byte, len(configJSON)),
+		annotations: annotations,
+		mediaType:   v1.MediaTypeImageManifest,
+	}
+	copy(mb.configJSON, configJSON)
+
+	return mb
+}
+
+// SetMediaType assigns the passed mediatype or error if the mediatype is not a
+// valid media type for oci image manifests currently: "" or "application/vnd.oci.image.manifest.v1+json"
+func (mb *Builder) SetMediaType(mediaType string) error {
+	if mediaType != "" && mediaType != v1.MediaTypeImageManifest {
+		return errors.New("invalid media type for OCI image manifest")
+	}
+
+	mb.mediaType = mediaType
+	return nil
+}
+
+// Build produces a final manifest from the given references.
+func (mb *Builder) Build(ctx context.Context) (distribution.Manifest, error) {
+	m := Manifest{
+		Versioned: manifest.Versioned{
+			SchemaVersion: 2,
+			MediaType:     mb.mediaType,
+		},
+		Layers:      make([]distribution.Descriptor, len(mb.layers)),
+		Annotations: mb.annotations,
+	}
+	copy(m.Layers, mb.layers)
+
+	configDigest := digest.FromBytes(mb.configJSON)
+
+	var err error
+	m.Config, err = mb.bs.Stat(ctx, configDigest)
+	switch err {
+	case nil:
+		// Override MediaType, since Put always replaces the specified media
+		// type with application/octet-stream in the descriptor it returns.
+		m.Config.MediaType = v1.MediaTypeImageConfig
+		return FromStruct(m)
+	case distribution.ErrBlobUnknown:
+		// nop
+	default:
+		return nil, err
+	}
+
+	// Add config to the blob store
+	m.Config, err = mb.bs.Put(ctx, v1.MediaTypeImageConfig, mb.configJSON)
+	// Override MediaType, since Put always replaces the specified media
+	// type with application/octet-stream in the descriptor it returns.
+	m.Config.MediaType = v1.MediaTypeImageConfig
+	if err != nil {
+		return nil, err
+	}
+
+	return FromStruct(m)
+}
+
+// AppendReference adds a reference to the current ManifestBuilder.
+func (mb *Builder) AppendReference(d distribution.Describable) error {
+	mb.layers = append(mb.layers, d.Descriptor())
+	return nil
+}
+
+// References returns the current references added to this builder.
+func (mb *Builder) References() []distribution.Descriptor {
+	return mb.layers
+}

vendor/github.com/docker/distribution/manifest/ocischema/manifest.go

@@ -0,0 +1,124 @@
+package ocischema
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/manifest"
+	"github.com/opencontainers/go-digest"
+	"github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+var (
+	// SchemaVersion provides a pre-initialized version structure for this
+	// packages version of the manifest.
+	SchemaVersion = manifest.Versioned{
+		SchemaVersion: 2, // historical value here.. does not pertain to OCI or docker version
+		MediaType:     v1.MediaTypeImageManifest,
+	}
+)
+
+func init() {
+	ocischemaFunc := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) {
+		m := new(DeserializedManifest)
+		err := m.UnmarshalJSON(b)
+		if err != nil {
+			return nil, distribution.Descriptor{}, err
+		}
+
+		dgst := digest.FromBytes(b)
+		return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: v1.MediaTypeImageManifest}, err
+	}
+	err := distribution.RegisterManifestSchema(v1.MediaTypeImageManifest, ocischemaFunc)
+	if err != nil {
+		panic(fmt.Sprintf("Unable to register manifest: %s", err))
+	}
+}
+
+// Manifest defines a ocischema manifest.
+type Manifest struct {
+	manifest.Versioned
+
+	// Config references the image configuration as a blob.
+	Config distribution.Descriptor `json:"config"`
+
+	// Layers lists descriptors for the layers referenced by the
+	// configuration.
+	Layers []distribution.Descriptor `json:"layers"`
+
+	// Annotations contains arbitrary metadata for the image manifest.
+	Annotations map[string]string `json:"annotations,omitempty"`
+}
+
+// References returns the descriptors of this manifests references.
+func (m Manifest) References() []distribution.Descriptor {
+	references := make([]distribution.Descriptor, 0, 1+len(m.Layers))
+	references = append(references, m.Config)
+	references = append(references, m.Layers...)
+	return references
+}
+
+// Target returns the target of this manifest.
+func (m Manifest) Target() distribution.Descriptor {
+	return m.Config
+}
+
+// DeserializedManifest wraps Manifest with a copy of the original JSON.
+// It satisfies the distribution.Manifest interface.
+type DeserializedManifest struct {
+	Manifest
+
+	// canonical is the canonical byte representation of the Manifest.
+	canonical []byte
+}
+
+// FromStruct takes a Manifest structure, marshals it to JSON, and returns a
+// DeserializedManifest which contains the manifest and its JSON representation.
+func FromStruct(m Manifest) (*DeserializedManifest, error) {
+	var deserialized DeserializedManifest
+	deserialized.Manifest = m
+
+	var err error
+	deserialized.canonical, err = json.MarshalIndent(&m, "", "   ")
+	return &deserialized, err
+}
+
+// UnmarshalJSON populates a new Manifest struct from JSON data.
+func (m *DeserializedManifest) UnmarshalJSON(b []byte) error {
+	m.canonical = make([]byte, len(b))
+	// store manifest in canonical
+	copy(m.canonical, b)
+
+	// Unmarshal canonical JSON into Manifest object
+	var manifest Manifest
+	if err := json.Unmarshal(m.canonical, &manifest); err != nil {
+		return err
+	}
+
+	if manifest.MediaType != "" && manifest.MediaType != v1.MediaTypeImageManifest {
+		return fmt.Errorf("if present, mediaType in manifest should be '%s' not '%s'",
+			v1.MediaTypeImageManifest, manifest.MediaType)
+	}
+
+	m.Manifest = manifest
+
+	return nil
+}
+
+// MarshalJSON returns the contents of canonical. If canonical is empty,
+// marshals the inner contents.
+func (m *DeserializedManifest) MarshalJSON() ([]byte, error) {
+	if len(m.canonical) > 0 {
+		return m.canonical, nil
+	}
+
+	return nil, errors.New("JSON representation not initialized in DeserializedManifest")
+}
+
+// Payload returns the raw content of the manifest. The contents can be used to
+// calculate the content identifier.
+func (m DeserializedManifest) Payload() (string, []byte, error) {
+	return v1.MediaTypeImageManifest, m.canonical, nil
+}