Merge pull request #339 from thaJeztah/19.03_backport_fix_containerd_optional_docker_content_digest

[19.03 backport] vendor: containerd to 7c1e88399

vendor.conf

@@ -119,14 +119,14 @@ github.com/googleapis/gax-go                        317e0006254c44a0ac427cc52a0e
 google.golang.org/genproto                          694d95ba50e67b2e363f3483057db5d4910c18f9
 
 # containerd
-github.com/containerd/containerd                    3a3f0aac8819165839a41fee77a4f4ac8b103097
+github.com/containerd/containerd                    7c1e88399ec0b0b077121d9d5ad97e647b11c870
 github.com/containerd/fifo                          a9fb20d87448d386e6d50b1f2e1fa70dcf0de43c
 github.com/containerd/continuity                    aaeac12a7ffcd198ae25440a9dff125c2e2703a7
 github.com/containerd/cgroups                       4994991857f9b0ae8dc439551e8bebdbb4bf66c1
 github.com/containerd/console                       0650fd9eeb50bab4fc99dceb9f2e14cf58f36e7f
 github.com/containerd/go-runc                       7d11b49dc0769f6dbb0d1b19f3d48524d1bad9ad
 github.com/containerd/typeurl                       2a93cfde8c20b23de8eb84a5adbc234ddf7a9e8d
-github.com/containerd/ttrpc                         f02858b1457c5ca3aaec3a0803eb0d59f96e41d6
+github.com/containerd/ttrpc                         699c4e40d1e7416e08bf7019c7ce2e9beced4636
 github.com/gogo/googleapis                          d31c731455cb061f42baff3bda55bad0118b126b # v1.2.0
 
 # cluster

vendor/github.com/containerd/containerd/README.md

@@ -1,4 +1,4 @@
-![containerd banner](https://raw.githubusercontent.com/cncf/artwork/master/containerd/horizontal/color/containerd-horizontal-color.png)
+![containerd banner](https://raw.githubusercontent.com/cncf/artwork/master/projects/containerd/horizontal/color/containerd-horizontal-color.png)
 
 [![GoDoc](https://godoc.org/github.com/containerd/containerd?status.svg)](https://godoc.org/github.com/containerd/containerd)
 [![Build Status](https://travis-ci.org/containerd/containerd.svg?branch=master)](https://travis-ci.org/containerd/containerd)

vendor/github.com/containerd/containerd/images/archive/importer.go

@@ -197,10 +197,7 @@ func onUntarJSON(r io.Reader, j interface{}) error {
 	if err != nil {
 		return err
 	}
-	if err := json.Unmarshal(b, j); err != nil {
-		return err
-	}
-	return nil
+	return json.Unmarshal(b, j)
 }
 
 func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, size int64, ref string) (digest.Digest, error) {

vendor/github.com/containerd/containerd/metadata/content.go

@@ -767,11 +767,7 @@ func writeExpireAt(expire time.Time, bkt *bolt.Bucket) error {
 	if err != nil {
 		return err
 	}
-	if err := bkt.Put(bucketKeyExpireAt, expireAt); err != nil {
-		return err
-	}
-
-	return nil
+	return bkt.Put(bucketKeyExpireAt, expireAt)
 }
 
 func (cs *contentStore) garbageCollect(ctx context.Context) (d time.Duration, err error) {

vendor/github.com/containerd/containerd/mount/mount_linux.go

@@ -111,7 +111,18 @@ func unmount(target string, flags int) error {
 // UnmountAll repeatedly unmounts the given mount point until there
 // are no mounts remaining (EINVAL is returned by mount), which is
 // useful for undoing a stack of mounts on the same mount point.
+// UnmountAll all is noop when the first argument is an empty string.
+// This is done when the containerd client did not specify any rootfs
+// mounts (e.g. because the rootfs is managed outside containerd)
+// UnmountAll is noop when the mount path does not exist.
 func UnmountAll(mount string, flags int) error {
+	if mount == "" {
+		return nil
+	}
+	if _, err := os.Stat(mount); os.IsNotExist(err) {
+		return nil
+	}
+
 	for {
 		if err := unmount(mount, flags); err != nil {
 			// EINVAL is returned if the target is not a

vendor/github.com/containerd/containerd/remotes/docker/handler.go

@@ -88,7 +88,7 @@ func appendDistributionSourceLabel(originLabel, repo string) string {
 	}
 	repos = append(repos, repo)
 
-	// use emtpy string to present duplicate items
+	// use empty string to present duplicate items
 	for i := 1; i < len(repos); i++ {
 		tmp, j := repos[i], i-1
 		for ; j >= 0 && repos[j] >= tmp; j-- {

vendor/github.com/containerd/containerd/remotes/docker/resolver.go

@@ -18,10 +18,10 @@ package docker
 
 import (
 	"context"
+	"io"
 	"net/http"
 	"net/url"
 	"path"
-	"strconv"
 	"strings"
 
 	"github.com/containerd/containerd/errdefs"
@@ -29,6 +29,7 @@ import (
 	"github.com/containerd/containerd/log"
 	"github.com/containerd/containerd/reference"
 	"github.com/containerd/containerd/remotes"
+	"github.com/containerd/containerd/remotes/docker/schema1"
 	"github.com/containerd/containerd/version"
 	digest "github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -150,6 +151,32 @@ func NewResolver(options ResolverOptions) remotes.Resolver {
 	}
 }
 
+func getManifestMediaType(resp *http.Response) string {
+	// Strip encoding data (manifests should always be ascii JSON)
+	contentType := resp.Header.Get("Content-Type")
+	if sp := strings.IndexByte(contentType, ';'); sp != -1 {
+		contentType = contentType[0:sp]
+	}
+
+	// As of Apr 30 2019 the registry.access.redhat.com registry does not specify
+	// the content type of any data but uses schema1 manifests.
+	if contentType == "text/plain" {
+		contentType = images.MediaTypeDockerSchema1Manifest
+	}
+	return contentType
+}
+
+type countingReader struct {
+	reader    io.Reader
+	bytesRead int64
+}
+
+func (r *countingReader) Read(p []byte) (int, error) {
+	n, err := r.reader.Read(p)
+	r.bytesRead += int64(n)
+	return n, err
+}
+
 var _ remotes.Resolver = &dockerResolver{}
 
 func (r *dockerResolver) Resolve(ctx context.Context, ref string) (string, ocispec.Descriptor, error) {
@@ -220,40 +247,56 @@ func (r *dockerResolver) Resolve(ctx context.Context, ref string) (string, ocisp
 			}
 			return "", ocispec.Descriptor{}, errors.Errorf("unexpected status code %v: %v", u, resp.Status)
 		}
+		size := resp.ContentLength
 
 		// this is the only point at which we trust the registry. we use the
 		// content headers to assemble a descriptor for the name. when this becomes
 		// more robust, we mostly get this information from a secure trust store.
 		dgstHeader := digest.Digest(resp.Header.Get("Docker-Content-Digest"))
+		contentType := getManifestMediaType(resp)
 
-		if dgstHeader != "" {
+		if dgstHeader != "" && size != -1 {
 			if err := dgstHeader.Validate(); err != nil {
 				return "", ocispec.Descriptor{}, errors.Wrapf(err, "%q in header not a valid digest", dgstHeader)
 			}
 			dgst = dgstHeader
-		}
-
-		if dgst == "" {
-			return "", ocispec.Descriptor{}, errors.Errorf("could not resolve digest for %v", ref)
-		}
+		} else {
+			log.G(ctx).Debug("no Docker-Content-Digest header, fetching manifest instead")
 
-		var (
-			size       int64
-			sizeHeader = resp.Header.Get("Content-Length")
-		)
-
-		size, err = strconv.ParseInt(sizeHeader, 10, 64)
-		if err != nil {
+			req, err := http.NewRequest(http.MethodGet, u, nil)
+			if err != nil {
+				return "", ocispec.Descriptor{}, err
+			}
+			req.Header = r.headers
 
-			return "", ocispec.Descriptor{}, errors.Wrapf(err, "invalid size header: %q", sizeHeader)
-		}
-		if size < 0 {
-			return "", ocispec.Descriptor{}, errors.Errorf("%q in header not a valid size", sizeHeader)
+			resp, err := fetcher.doRequestWithRetries(ctx, req, nil)
+			if err != nil {
+				return "", ocispec.Descriptor{}, err
+			}
+			defer resp.Body.Close()
+
+			bodyReader := countingReader{reader: resp.Body}
+
+			contentType = getManifestMediaType(resp)
+			if contentType == images.MediaTypeDockerSchema1Manifest {
+				b, err := schema1.ReadStripSignature(&bodyReader)
+				if err != nil {
+					return "", ocispec.Descriptor{}, err
+				}
+
+				dgst = digest.FromBytes(b)
+			} else {
+				dgst, err = digest.FromReader(&bodyReader)
+				if err != nil {
+					return "", ocispec.Descriptor{}, err
+				}
+			}
+			size = bodyReader.bytesRead
 		}
 
 		desc := ocispec.Descriptor{
 			Digest:    dgst,
-			MediaType: resp.Header.Get("Content-Type"), // need to strip disposition?
+			MediaType: contentType,
 			Size:      size,
 		}
 

vendor/github.com/containerd/containerd/remotes/docker/schema1/converter.go

@@ -227,6 +227,17 @@ func (c *Converter) Convert(ctx context.Context, opts ...ConvertOpt) (ocispec.De
 	return desc, nil
 }
 
+// ReadStripSignature reads in a schema1 manifest and returns a byte array
+// with the "signatures" field stripped
+func ReadStripSignature(schema1Blob io.Reader) ([]byte, error) {
+	b, err := ioutil.ReadAll(io.LimitReader(schema1Blob, manifestSizeLimit)) // limit to 8MB
+	if err != nil {
+		return nil, err
+	}
+
+	return stripSignature(b)
+}
+
 func (c *Converter) fetchManifest(ctx context.Context, desc ocispec.Descriptor) error {
 	log.G(ctx).Debug("fetch schema 1")
 
@@ -235,17 +246,12 @@ func (c *Converter) fetchManifest(ctx context.Context, desc ocispec.Descriptor)
 		return err
 	}
 
-	b, err := ioutil.ReadAll(io.LimitReader(rc, manifestSizeLimit)) // limit to 8MB
+	b, err := ReadStripSignature(rc)
 	rc.Close()
 	if err != nil {
 		return err
 	}
 
-	b, err = stripSignature(b)
-	if err != nil {
-		return err
-	}
-
 	var m manifest
 	if err := json.Unmarshal(b, &m); err != nil {
 		return err

vendor/github.com/containerd/containerd/runtime/v1/linux/bundle.go

@@ -65,9 +65,6 @@ func newBundle(id, path, workDir string, spec []byte) (b *bundle, err error) {
 			os.RemoveAll(workDir)
 		}
 	}()
-	if err := os.Mkdir(filepath.Join(path, "rootfs"), 0711); err != nil {
-		return nil, err
-	}
 	err = ioutil.WriteFile(filepath.Join(path, configFilename), spec, 0666)
 	return &bundle{
 		id:      id,

vendor/github.com/containerd/containerd/runtime/v1/linux/proc/deleted_state.go

@@ -22,6 +22,7 @@ import (
 	"context"
 
 	"github.com/containerd/console"
+	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/runtime/proc"
 	google_protobuf "github.com/gogo/protobuf/types"
 	"github.com/pkg/errors"
@@ -55,11 +56,11 @@ func (s *deletedState) Start(ctx context.Context) error {
 }
 
 func (s *deletedState) Delete(ctx context.Context) error {
-	return errors.Errorf("cannot delete a deleted process")
+	return errors.Wrap(errdefs.ErrNotFound, "cannot delete a deleted process")
 }
 
 func (s *deletedState) Kill(ctx context.Context, sig uint32, all bool) error {
-	return errors.Errorf("cannot kill a deleted process")
+	return errors.Wrap(errdefs.ErrNotFound, "cannot kill a deleted process")
 }
 
 func (s *deletedState) SetExited(status int) {

vendor/github.com/containerd/containerd/runtime/v1/linux/proc/io.go

@@ -271,27 +271,35 @@ func newBinaryIO(ctx context.Context, id string, uri *url.URL) (runc.IO, error)
 	)
 	out, err := newPipe()
 	if err != nil {
+		cancel()
 		return nil, err
 	}
 	serr, err := newPipe()
 	if err != nil {
+		cancel()
 		return nil, err
 	}
 	r, w, err := os.Pipe()
 	if err != nil {
+		cancel()
 		return nil, err
 	}
 	cmd.ExtraFiles = append(cmd.ExtraFiles, out.r, serr.r, w)
 	// don't need to register this with the reaper or wait when
 	// running inside a shim
 	if err := cmd.Start(); err != nil {
+		cancel()
 		return nil, err
 	}
 	// close our side of the pipe after start
-	w.Close()
+	if err := w.Close(); err != nil {
+		cancel()
+		return nil, err
+	}
 	// wait for the logging binary to be ready
 	b := make([]byte, 1)
 	if _, err := r.Read(b); err != nil && err != io.EOF {
+		cancel()
 		return nil, err
 	}
 	return &binaryIO{

vendor/github.com/containerd/containerd/runtime/v1/linux/task.go

@@ -87,7 +87,7 @@ func (t *Task) Namespace() string {
 // Delete the task and return the exit status
 func (t *Task) Delete(ctx context.Context) (*runtime.Exit, error) {
 	rsp, err := t.shim.Delete(ctx, empty)
-	if err != nil {
+	if err != nil && !errdefs.IsNotFound(err) {
 		return nil, errdefs.FromGRPC(err)
 	}
 	t.tasks.Delete(ctx, t.id)

vendor/github.com/containerd/containerd/runtime/v1/shim/client/client.go

@@ -219,8 +219,7 @@ func WithConnect(address string, onClose func()) Opt {
 		if err != nil {
 			return nil, nil, err
 		}
-		client := ttrpc.NewClient(conn)
-		client.OnClose(onClose)
+		client := ttrpc.NewClient(conn, ttrpc.WithOnClose(onClose))
 		return shimapi.NewShimClient(client), conn, nil
 	}
 }

vendor/github.com/containerd/containerd/runtime/v1/shim/service.go

@@ -124,6 +124,14 @@ func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (_ *
 		})
 	}
 
+	rootfs := ""
+	if len(mounts) > 0 {
+		rootfs = filepath.Join(r.Bundle, "rootfs")
+		if err := os.Mkdir(rootfs, 0711); err != nil {
+			return nil, err
+		}
+	}
+
 	config := &proc.CreateConfig{
 		ID:               r.ID,
 		Bundle:           r.Bundle,
@@ -137,7 +145,6 @@ func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (_ *
 		ParentCheckpoint: r.ParentCheckpoint,
 		Options:          r.Options,
 	}
-	rootfs := filepath.Join(r.Bundle, "rootfs")
 	defer func() {
 		if err != nil {
 			if err2 := mount.UnmountAll(rootfs, 0); err2 != nil {
@@ -169,6 +176,7 @@ func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (_ *
 		s.config.SystemdCgroup,
 		s.platform,
 		config,
+		rootfs,
 	)
 	if err != nil {
 		return nil, errdefs.ToGRPC(err)
@@ -632,7 +640,7 @@ func getTopic(ctx context.Context, e interface{}) string {
 	return runtime.TaskUnknownTopic
 }
 
-func newInit(ctx context.Context, path, workDir, runtimeRoot, namespace, criu string, systemdCgroup bool, platform rproc.Platform, r *proc.CreateConfig) (*proc.Init, error) {
+func newInit(ctx context.Context, path, workDir, runtimeRoot, namespace, criu string, systemdCgroup bool, platform rproc.Platform, r *proc.CreateConfig, rootfs string) (*proc.Init, error) {
 	var options runctypes.CreateOptions
 	if r.Options != nil {
 		v, err := typeurl.UnmarshalAny(r.Options)
@@ -642,7 +650,6 @@ func newInit(ctx context.Context, path, workDir, runtimeRoot, namespace, criu st
 		options = *v.(*runctypes.CreateOptions)
 	}
 
-	rootfs := filepath.Join(path, "rootfs")
 	runtime := proc.NewRunc(runtimeRoot, path, namespace, r.Runtime, criu, systemdCgroup)
 	p := proc.New(r.ID, runtime, rproc.Stdio{
 		Stdin:    r.Stdin,

vendor/github.com/containerd/containerd/vendor.conf

@@ -20,7 +20,7 @@ github.com/gogo/protobuf v1.2.1
 github.com/gogo/googleapis v1.2.0
 github.com/golang/protobuf v1.2.0
 github.com/opencontainers/runtime-spec 29686dbc5559d93fb1ef402eeda3e35c38d75af4 # v1.0.1-59-g29686db
-github.com/opencontainers/runc 029124da7af7360afa781a0234d1b083550f797c
+github.com/opencontainers/runc v1.0.0-rc8
 github.com/konsorten/go-windows-terminal-sequences v1.0.1
 github.com/sirupsen/logrus v1.4.1
 github.com/urfave/cli 7bc6a0acffa589f415f88aca16cc1de5ffd66f9c
@@ -37,15 +37,15 @@ github.com/Microsoft/go-winio 84b4ab48a50763fe7b3abcef38e5205c12027fac
 github.com/Microsoft/hcsshim 8abdbb8205e4192c68b5f84c31197156f31be517
 google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944
 golang.org/x/text 19e51611da83d6be54ddafce4a4af510cb3e9ea4
-github.com/containerd/ttrpc f02858b1457c5ca3aaec3a0803eb0d59f96e41d6
+github.com/containerd/ttrpc 699c4e40d1e7416e08bf7019c7ce2e9beced4636
 github.com/syndtr/gocapability d98352740cb2c55f81556b63d4a1ec64c5a319c2
 gotest.tools v2.3.0
 github.com/google/go-cmp v0.2.0
 go.etcd.io/bbolt v1.3.2
 
 # cri dependencies
-github.com/containerd/cri 6d353571e64417d80c9478ffaea793714dd539d0 # master
-github.com/containerd/go-cni 40bcf8ec8acd7372be1d77031d585d5d8e561c90
+github.com/containerd/cri 2fc62db8146ce66f27b37306ad5fda34207835f3 # master
+github.com/containerd/go-cni 891c2a41e18144b2d7921f971d6c9789a68046b2
 github.com/containernetworking/cni v0.6.0
 github.com/containernetworking/plugins v0.7.0
 github.com/davecgh/go-spew v1.1.0
@@ -59,7 +59,7 @@ github.com/hashicorp/go-multierror ed905158d87462226a13fe39ddf685ea65f1c11f
 github.com/json-iterator/go 1.1.5
 github.com/modern-go/reflect2 1.0.1
 github.com/modern-go/concurrent 1.0.3
-github.com/opencontainers/selinux v1.2.1
+github.com/opencontainers/selinux v1.2.2
 github.com/seccomp/libseccomp-golang 32f571b70023028bd57d9288c20efbcb237f3ce0
 github.com/tchap/go-patricia v2.2.6
 golang.org/x/crypto 88737f569e3a9c7ab309cdc09a07fe7fc87233c3

vendor/github.com/containerd/ttrpc/client.go

@@ -49,7 +49,15 @@ type Client struct {
 	err       error
 }
 
-func NewClient(conn net.Conn) *Client {
+type ClientOpts func(c *Client)
+
+func WithOnClose(onClose func()) ClientOpts {
+	return func(c *Client) {
+		c.closeFunc = onClose
+	}
+}
+
+func NewClient(conn net.Conn, opts ...ClientOpts) *Client {
 	c := &Client{
 		codec:     codec{},
 		conn:      conn,
@@ -60,6 +68,10 @@ func NewClient(conn net.Conn) *Client {
 		closeFunc: func() {},
 	}
 
+	for _, o := range opts {
+		o(c)
+	}
+
 	go c.run()
 	return c
 }
@@ -141,11 +153,6 @@ func (c *Client) Close() error {
 	return nil
 }
 
-// OnClose allows a close func to be called when the server is closed
-func (c *Client) OnClose(closer func()) {
-	c.closeFunc = closer
-}
-
 type message struct {
 	messageHeader
 	p   []byte
@@ -255,7 +262,7 @@ func (c *Client) recv(resp *Response, msg *message) error {
 	}
 
 	if msg.Type != messageTypeResponse {
-		return errors.New("unkown message type received")
+		return errors.New("unknown message type received")
 	}
 
 	defer c.channel.putmbuf(msg.p)

vendor/github.com/containerd/ttrpc/services.go

@@ -76,7 +76,7 @@ func (s *serviceSet) dispatch(ctx context.Context, serviceName, methodName strin
 		switch v := obj.(type) {
 		case proto.Message:
 			if err := proto.Unmarshal(p, v); err != nil {
-				return status.Errorf(codes.Internal, "ttrpc: error unmarshaling payload: %v", err.Error())
+				return status.Errorf(codes.Internal, "ttrpc: error unmarshalling payload: %v", err.Error())
 			}
 		default:
 			return status.Errorf(codes.Internal, "ttrpc: error unsupported request type: %T", v)