Add Tarsum Calculation during v2 Pull operation

While the v2 pull operation is writing the body of the layer blob to disk
it now computes the tarsum checksum of the archive before extracting it to
the backend storage driver. If the checksum does not match that from the
image manifest an error is raised.

Also adds more debug logging to the pull operation and fixes existing test
cases which were failing. Adds a reverse lookup constructor to the tarsum
package so that you can get a tarsum object using a checksum label.

Docker-DCO-1.1-Signed-off-by: Josh Hawn <josh.hawn@docker.com> (github: jlhawn)

graph/pull.go

@@ -15,6 +15,7 @@ import (
 	log "github.com/Sirupsen/logrus"
 	"github.com/docker/docker/engine"
 	"github.com/docker/docker/image"
+	"github.com/docker/docker/pkg/tarsum"
 	"github.com/docker/docker/registry"
 	"github.com/docker/docker/utils"
 	"github.com/docker/libtrust"
@@ -112,6 +113,8 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status {
 	}
 	defer s.poolRemove("pull", repoInfo.LocalName+":"+tag)
 
+
+	log.Debugf("pulling image from host %q with remote name %q", repoInfo.Index.Name, repoInfo.RemoteName)
 	endpoint, err := repoInfo.GetEndpoint()
 	if err != nil {
 		return job.Error(err)
@@ -127,6 +130,10 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status {
 		logName += ":" + tag
 	}
 
+	// Calling the v2 code path might change the session
+	// endpoint value, so save the original one!
+	originalSession := *r
+
 	if len(repoInfo.Index.Mirrors) == 0 && (repoInfo.Index.Official || endpoint.Version == registry.APIVersion2) {
 		j := job.Eng.Job("trust_update_base")
 		if err = j.Run(); err != nil {
@@ -138,6 +145,7 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status {
 			return job.Errorf("error getting authorization: %s", err)
 		}
 
+		log.Debugf("pulling v2 repository with local name %q", repoInfo.LocalName)
 		if err := s.pullV2Repository(job.Eng, r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel"), auth); err == nil {
 			if err = job.Eng.Job("log", "pull", logName, "").Run(); err != nil {
 				log.Errorf("Error logging event 'pull' for %s: %s", logName, err)
@@ -146,8 +154,13 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status {
 		} else if err != registry.ErrDoesNotExist {
 			log.Errorf("Error from V2 registry: %s", err)
 		}
+
+		log.Debug("image does not exist on v2 registry, falling back to v1")
 	}
 
+	r = &originalSession
+
+	log.Debugf("pulling v1 repository with local name %q", repoInfo.LocalName)
 	if err = s.pullRepository(r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel")); err != nil {
 		return job.Error(err)
 	}
@@ -174,7 +187,7 @@ func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, repoInfo *
 	log.Debugf("Retrieving the tag list")
 	tagsList, err := r.GetRemoteTags(repoData.Endpoints, repoInfo.RemoteName, repoData.Tokens)
 	if err != nil {
-		log.Errorf("%v", err)
+		log.Errorf("unable to get remote tags: %s", err)
 		return err
 	}
 
@@ -535,7 +548,20 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri
 					return err
 				}
 				defer r.Close()
-				io.Copy(tmpFile, utils.ProgressReader(r, int(l), out, sf, false, utils.TruncateID(img.ID), "Downloading"))
+
+				// Wrap the reader with the appropriate TarSum reader.
+				tarSumReader, err := tarsum.NewTarSumForLabel(r, true, sumType)
+				if err != nil {
+					return fmt.Errorf("unable to wrap image blob reader with TarSum: %s", err)
+				}
+
+				io.Copy(tmpFile, utils.ProgressReader(ioutil.NopCloser(tarSumReader), int(l), out, sf, false, utils.TruncateID(img.ID), "Downloading"))
+
+				out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Verifying Checksum", nil))
+
+				if finalChecksum := tarSumReader.Sum(nil); !strings.EqualFold(finalChecksum, sumStr) {
+					return fmt.Errorf("image verification failed: checksum mismatch - expected %q but got %q", sumStr, finalChecksum)
+				}
 
 				out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Download complete", nil))
 

image/image.go

@@ -94,28 +94,29 @@ func StoreImage(img *Image, layerData archive.ArchiveReader, root string) error
 
 	// If layerData is not nil, unpack it into the new layer
 	if layerData != nil {
-		layerDataDecompressed, err := archive.DecompressStream(layerData)
-		if err != nil {
+		// If the image doesn't have a checksum, we should add it. The layer
+		// checksums are verified when they are pulled from a remote, but when
+		// a container is committed it should be added here.
+		if img.Checksum == "" {
+			layerDataDecompressed, err := archive.DecompressStream(layerData)
+			if err != nil {
+				return err
+			}
+			defer layerDataDecompressed.Close()
+
+			if layerTarSum, err = tarsum.NewTarSum(layerDataDecompressed, true, tarsum.VersionDev); err != nil {
+				return err
+			}
+
+			if size, err = driver.ApplyDiff(img.ID, img.Parent, layerTarSum); err != nil {
+				return err
+			}
+
+			img.Checksum = layerTarSum.Sum(nil)
+		} else if size, err = driver.ApplyDiff(img.ID, img.Parent, layerData); err != nil {
 			return err
 		}
 
-		defer layerDataDecompressed.Close()
-
-		if layerTarSum, err = tarsum.NewTarSum(layerDataDecompressed, true, tarsum.VersionDev); err != nil {
-			return err
-		}
-
-		if size, err = driver.ApplyDiff(img.ID, img.Parent, layerTarSum); err != nil {
-			return err
-		}
-
-		checksum := layerTarSum.Sum(nil)
-
-		if img.Checksum != "" && img.Checksum != checksum {
-			log.Warnf("image layer checksum mismatch: computed %q, expected %q", checksum, img.Checksum)
-		}
-
-		img.Checksum = checksum
 	}
 
 	img.Size = size

pkg/tarsum/tarsum.go

@@ -3,8 +3,11 @@ package tarsum
 import (
 	"bytes"
 	"compress/gzip"
+	"crypto"
 	"crypto/sha256"
 	"encoding/hex"
+	"errors"
+	"fmt"
 	"hash"
 	"io"
 	"strings"
@@ -39,6 +42,30 @@ func NewTarSumHash(r io.Reader, dc bool, v Version, tHash THash) (TarSum, error)
 	return ts, err
 }
 
+// Create a new TarSum using the provided TarSum version+hash label.
+func NewTarSumForLabel(r io.Reader, disableCompression bool, label string) (TarSum, error) {
+	parts := strings.SplitN(label, "+", 2)
+	if len(parts) != 2 {
+		return nil, errors.New("tarsum label string should be of the form: {tarsum_version}+{hash_name}")
+	}
+
+	versionName, hashName := parts[0], parts[1]
+
+	version, ok := tarSumVersionsByName[versionName]
+	if !ok {
+		return nil, fmt.Errorf("unknown TarSum version name: %q", versionName)
+	}
+
+	hashConfig, ok := standardHashConfigs[hashName]
+	if !ok {
+		return nil, fmt.Errorf("unknown TarSum hash name: %q", hashName)
+	}
+
+	tHash := NewTHash(hashConfig.name, hashConfig.hash.New)
+
+	return NewTarSumHash(r, disableCompression, version, tHash)
+}
+
 // TarSum is the generic interface for calculating fixed time
 // checksums of a tar archive
 type TarSum interface {
@@ -89,6 +116,18 @@ func NewTHash(name string, h func() hash.Hash) THash {
 	return simpleTHash{n: name, h: h}
 }
 
+type tHashConfig struct {
+	name string
+	hash crypto.Hash
+}
+
+var (
+	standardHashConfigs = map[string]tHashConfig{
+		"sha256": {name: "sha256", hash: crypto.SHA256},
+		"sha512": {name: "sha512", hash: crypto.SHA512},
+	}
+)
+
 // TarSum default is "sha256"
 var DefaultTHash = NewTHash("sha256", sha256.New)
 

pkg/tarsum/versioning.go

@@ -31,11 +31,18 @@ func GetVersions() []Version {
 	return v
 }
 
-var tarSumVersions = map[Version]string{
-	Version0:   "tarsum",
-	Version1:   "tarsum.v1",
-	VersionDev: "tarsum.dev",
-}
+var (
+	tarSumVersions = map[Version]string{
+		Version0:   "tarsum",
+		Version1:   "tarsum.v1",
+		VersionDev: "tarsum.dev",
+	}
+	tarSumVersionsByName = map[string]Version{
+		"tarsum":     Version0,
+		"tarsum.v1":  Version1,
+		"tarsum.dev": VersionDev,
+	}
+)
 
 func (tsv Version) String() string {
 	return tarSumVersions[tsv]

registry/endpoint.go

@@ -47,16 +47,23 @@ func NewEndpoint(index *IndexInfo) (*Endpoint, error) {
 	if err != nil {
 		return nil, err
 	}
+	if err := validateEndpoint(endpoint); err != nil {
+		return nil, err
+	}
 
+	return endpoint, nil
+}
+
+func validateEndpoint(endpoint *Endpoint) error {
 	log.Debugf("pinging registry endpoint %s", endpoint)
 
 	// Try HTTPS ping to registry
 	endpoint.URL.Scheme = "https"
 	if _, err := endpoint.Ping(); err != nil {
-		if index.Secure {
+		if endpoint.IsSecure {
 			// If registry is secure and HTTPS failed, show user the error and tell them about `--insecure-registry`
 			// in case that's what they need. DO NOT accept unknown CA certificates, and DO NOT fallback to HTTP.
-			return nil, fmt.Errorf("invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host)
+			return fmt.Errorf("invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host)
 		}
 
 		// If registry is insecure and HTTPS failed, fallback to HTTP.
@@ -65,13 +72,13 @@ func NewEndpoint(index *IndexInfo) (*Endpoint, error) {
 
 		var err2 error
 		if _, err2 = endpoint.Ping(); err2 == nil {
-			return endpoint, nil
+			return nil
 		}
 
-		return nil, fmt.Errorf("invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2)
+		return fmt.Errorf("invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2)
 	}
 
-	return endpoint, nil
+	return nil
 }
 
 func newEndpoint(address string, secure bool) (*Endpoint, error) {

registry/session_v2.go

@@ -30,8 +30,12 @@ func (r *Session) GetV2Authorization(imageName string, readOnly bool) (auth *Req
 	}
 
 	var registry *Endpoint
-	if r.indexEndpoint.URL.Host == IndexServerURL.Host {
-		registry, err = NewEndpoint(REGISTRYSERVER, nil)
+	if r.indexEndpoint.String() == IndexServerAddress() {
+		registry, err = newEndpoint(REGISTRYSERVER, true)
+		if err != nil {
+			return
+		}
+		err = validateEndpoint(registry)
 		if err != nil {
 			return
 		}