Merge pull request #17093 from endophage/update_notary_gotuf

updating notary and gotuf with latest bugfixes

hack/vendor.sh

@@ -41,8 +41,8 @@ clone git github.com/boltdb/bolt v1.0
 clone git github.com/docker/distribution 20c4b7a1805a52753dfd593ee1cc35558722a0ce # docker/1.9 branch
 clone git github.com/vbatts/tar-split v0.9.10
 
-clone git github.com/docker/notary ac05822d7d71ef077df3fc24f506672282a1feea
-clone git github.com/endophage/gotuf 9bcdad0308e34a49f38448b8ad436ad8860825ce
+clone git github.com/docker/notary 089d8450d8928aa1c58fd03f09cabbde9bcb4590
+clone git github.com/endophage/gotuf 876c31a61bc4aa0dae09bb8ef3946dc26dd04924
 clone git github.com/jfrazelle/go 6e461eb70cb4187b41a84e9a567d7137bdbe0f16
 clone git github.com/agl/ed25519 d2b94fd789ea21d12fac1a4443dd3a3f79cda72c
 

vendor/src/github.com/docker/notary/client/changelist/change.go

@@ -1,5 +1,9 @@
 package changelist
 
+import (
+	"github.com/endophage/gotuf/data"
+)
+
 // Scopes for TufChanges are simply the TUF roles.
 // Unfortunately because of targets delegations, we can only
 // cover the base roles.
@@ -10,6 +14,17 @@ const (
 	ScopeTimestamp = "timestamp"
 )
 
+// Types for TufChanges are namespaced by the Role they
+// are relevant for. The Root and Targets roles are the
+// only ones for which user action can cause a change, as
+// all changes in Snapshot and Timestamp are programatically
+// generated base on Root and Targets changes.
+const (
+	TypeRootRole          = "role"
+	TypeTargetsTarget     = "target"
+	TypeTargetsDelegation = "delegation"
+)
+
 // TufChange represents a change to a TUF repo
 type TufChange struct {
 	// Abbreviated because Go doesn't permit a field and method of the same name
@@ -20,6 +35,13 @@ type TufChange struct {
 	Data       []byte `json:"data"`
 }
 
+// TufRootData represents a modification of the keys associated
+// with a role that appears in the root.json
+type TufRootData struct {
+	Keys     []data.TUFKey `json:"keys"`
+	RoleName string        `json:"role"`
+}
+
 // NewTufChange initializes a tufChange object
 func NewTufChange(action string, role, changeType, changePath string, content []byte) *TufChange {
 	return &TufChange{

vendor/src/github.com/docker/notary/client/client.go

@@ -245,6 +245,7 @@ func (r *NotaryRepository) AddTarget(target *Target) error {
 	if err != nil {
 		return err
 	}
+	defer cl.Close()
 	logrus.Debugf("Adding target \"%s\" with sha256 \"%x\" and size %d bytes.\n", target.Name, target.Hashes["sha256"], target.Length)
 
 	meta := data.FileMeta{Length: target.Length, Hashes: target.Hashes}
@@ -258,7 +259,7 @@ func (r *NotaryRepository) AddTarget(target *Target) error {
 	if err != nil {
 		return err
 	}
-	return cl.Close()
+	return nil
 }
 
 // RemoveTarget creates a new changelist entry to remove a target from the repository
@@ -326,6 +327,17 @@ func (r *NotaryRepository) GetTargetByName(name string) (*Target, error) {
 	return &Target{Name: name, Hashes: meta.Hashes, Length: meta.Length}, nil
 }
 
+// GetChangelist returns the list of the repository's unpublished changes
+func (r *NotaryRepository) GetChangelist() (changelist.Changelist, error) {
+	changelistDir := filepath.Join(r.tufRepoPath, "changelist")
+	cl, err := changelist.NewFileChangelist(changelistDir)
+	if err != nil {
+		logrus.Debug("Error initializing changelist")
+		return nil, err
+	}
+	return cl, nil
+}
+
 // Publish pushes the local changes in signed material to the remote notary-server
 // Conceptually it performs an operation similar to a `git rebase`
 func (r *NotaryRepository) Publish() error {
@@ -371,11 +383,8 @@ func (r *NotaryRepository) Publish() error {
 			return err
 		}
 	}
-	// load the changelist for this repo
-	changelistDir := filepath.Join(r.tufRepoPath, "changelist")
-	cl, err := changelist.NewFileChangelist(changelistDir)
+	cl, err := r.GetChangelist()
 	if err != nil {
-		logrus.Debug("Error initializing changelist")
 		return err
 	}
 	// apply the changelist to the repo
@@ -445,7 +454,7 @@ func (r *NotaryRepository) Publish() error {
 		// This is not a critical problem when only a single host is pushing
 		// but will cause weird behaviour if changelist cleanup is failing
 		// and there are multiple hosts writing to the repo.
-		logrus.Warn("Unable to clear changelist. You may want to manually delete the folder ", changelistDir)
+		logrus.Warn("Unable to clear changelist. You may want to manually delete the folder ", filepath.Join(r.tufRepoPath, "changelist"))
 	}
 	return nil
 }
@@ -596,3 +605,55 @@ func (r *NotaryRepository) bootstrapClient() (*tufclient.Client, error) {
 		r.fileStore,
 	), nil
 }
+
+// RotateKeys removes all existing keys associated with role and adds
+// the keys specified by keyIDs to the role. These changes are staged
+// in a changelist until publish is called.
+func (r *NotaryRepository) RotateKeys() error {
+	for _, role := range []string{"targets", "snapshot"} {
+		key, err := r.cryptoService.Create(role, data.ECDSAKey)
+		if err != nil {
+			return err
+		}
+		err = r.rootFileKeyChange(role, changelist.ActionCreate, key)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (r *NotaryRepository) rootFileKeyChange(role, action string, key data.PublicKey) error {
+	cl, err := changelist.NewFileChangelist(filepath.Join(r.tufRepoPath, "changelist"))
+	if err != nil {
+		return err
+	}
+	defer cl.Close()
+
+	k, ok := key.(*data.TUFKey)
+	if !ok {
+		return errors.New("Invalid key type found during rotation.")
+	}
+
+	meta := changelist.TufRootData{
+		RoleName: role,
+		Keys:     []data.TUFKey{*k},
+	}
+	metaJSON, err := json.Marshal(meta)
+	if err != nil {
+		return err
+	}
+
+	c := changelist.NewTufChange(
+		action,
+		changelist.ScopeRoot,
+		changelist.TypeRootRole,
+		role,
+		metaJSON,
+	)
+	err = cl.Add(c)
+	if err != nil {
+		return err
+	}
+	return nil
+}

vendor/src/github.com/docker/notary/client/helpers.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/notary/client/changelist"
-	"github.com/endophage/gotuf"
+	tuf "github.com/endophage/gotuf"
 	"github.com/endophage/gotuf/data"
 	"github.com/endophage/gotuf/keys"
 	"github.com/endophage/gotuf/store"
@@ -38,14 +38,16 @@ func applyChangelist(repo *tuf.TufRepo, cl changelist.Changelist) error {
 		}
 		switch c.Scope() {
 		case changelist.ScopeTargets:
-			err := applyTargetsChange(repo, c)
-			if err != nil {
-				return err
-			}
+			err = applyTargetsChange(repo, c)
+		case changelist.ScopeRoot:
+			err = applyRootChange(repo, c)
 		default:
 			logrus.Debug("scope not supported: ", c.Scope())
 		}
 		index++
+		if err != nil {
+			return err
+		}
 	}
 	logrus.Debugf("applied %d change(s)", index)
 	return nil
@@ -75,6 +77,40 @@ func applyTargetsChange(repo *tuf.TufRepo, c changelist.Change) error {
 	return nil
 }
 
+func applyRootChange(repo *tuf.TufRepo, c changelist.Change) error {
+	var err error
+	switch c.Type() {
+	case changelist.TypeRootRole:
+		err = applyRootRoleChange(repo, c)
+	default:
+		logrus.Debug("type of root change not yet supported: ", c.Type())
+	}
+	return err // might be nil
+}
+
+func applyRootRoleChange(repo *tuf.TufRepo, c changelist.Change) error {
+	switch c.Action() {
+	case changelist.ActionCreate:
+		// replaces all keys for a role
+		d := &changelist.TufRootData{}
+		err := json.Unmarshal(c.Content(), d)
+		if err != nil {
+			return err
+		}
+		k := []data.PublicKey{}
+		for _, key := range d.Keys {
+			k = append(k, data.NewPublicKey(key.Algorithm(), key.Public()))
+		}
+		err = repo.ReplaceBaseKeys(d.RoleName, k...)
+		if err != nil {
+			return err
+		}
+	default:
+		logrus.Debug("action not yet supported for root: ", c.Action())
+	}
+	return nil
+}
+
 func nearExpiry(r *data.SignedRoot) bool {
 	plus6mo := time.Now().AddDate(0, 6, 0)
 	return r.Signed.Expires.Before(plus6mo)

vendor/src/github.com/endophage/gotuf/client/client.go

@@ -175,19 +175,7 @@ func (c *Client) downloadRoot() error {
 	var s *data.Signed
 	var raw []byte
 	if download {
-		logrus.Debug("downloading new root")
-		raw, err = c.remote.GetMeta(role, size)
-		if err != nil {
-			return err
-		}
-		hash := sha256.Sum256(raw)
-		if expectedSha256 != nil && !bytes.Equal(hash[:], expectedSha256) {
-			// if we don't have an expected sha256, we're going to trust the root
-			// based purely on signature and expiry time validation
-			return fmt.Errorf("Remote root sha256 did not match snapshot root sha256: %#x vs. %#x", hash, []byte(expectedSha256))
-		}
-		s = &data.Signed{}
-		err = json.Unmarshal(raw, s)
+		raw, s, err = c.downloadSigned(role, size, expectedSha256)
 		if err != nil {
 			return err
 		}
@@ -247,6 +235,8 @@ func (c Client) verifyRoot(role string, s *data.Signed, minVersion int) error {
 }
 
 // downloadTimestamp is responsible for downloading the timestamp.json
+// Timestamps are special in that we ALWAYS attempt to download and only
+// use cache if the download fails (and the cache is still valid).
 func (c *Client) downloadTimestamp() error {
 	logrus.Debug("downloadTimestamp")
 	role := data.RoleName("timestamp")
@@ -271,7 +261,6 @@ func (c *Client) downloadTimestamp() error {
 	}
 	// unlike root, targets and snapshot, always try and download timestamps
 	// from remote, only using the cache one if we couldn't reach remote.
-	logrus.Debug("Downloading timestamp")
 	raw, err := c.remote.GetMeta(role, maxSize)
 	var s *data.Signed
 	if err != nil || len(raw) == 0 {
@@ -286,6 +275,7 @@ func (c *Client) downloadTimestamp() error {
 			}
 			return err
 		}
+		logrus.Debug("using cached timestamp")
 		s = old
 	} else {
 		download = true
@@ -351,17 +341,7 @@ func (c *Client) downloadSnapshot() error {
 	}
 	var s *data.Signed
 	if download {
-		logrus.Debug("downloading new snapshot")
-		raw, err = c.remote.GetMeta(role, size)
-		if err != nil {
-			return err
-		}
-		genHash := sha256.Sum256(raw)
-		if !bytes.Equal(genHash[:], expectedSha256) {
-			return fmt.Errorf("Retrieved snapshot did not verify against hash in timestamp.")
-		}
-		s = &data.Signed{}
-		err = json.Unmarshal(raw, s)
+		raw, s, err = c.downloadSigned(role, size, expectedSha256)
 		if err != nil {
 			return err
 		}
@@ -390,8 +370,7 @@ func (c *Client) downloadSnapshot() error {
 }
 
 // downloadTargets is responsible for downloading any targets file
-// including delegates roles. It will download the whole tree of
-// delegated roles below the given one
+// including delegates roles.
 func (c *Client) downloadTargets(role string) error {
 	role = data.RoleName(role) // this will really only do something for base targets role
 	snap := c.local.Snapshot.Signed
@@ -418,6 +397,24 @@ func (c *Client) downloadTargets(role string) error {
 	return nil
 }
 
+func (c *Client) downloadSigned(role string, size int64, expectedSha256 []byte) ([]byte, *data.Signed, error) {
+	logrus.Debugf("downloading new %s", role)
+	raw, err := c.remote.GetMeta(role, size)
+	if err != nil {
+		return nil, nil, err
+	}
+	genHash := sha256.Sum256(raw)
+	if !bytes.Equal(genHash[:], expectedSha256) {
+		return nil, nil, ErrChecksumMismatch{role: role}
+	}
+	s := &data.Signed{}
+	err = json.Unmarshal(raw, s)
+	if err != nil {
+		return nil, nil, err
+	}
+	return raw, s, nil
+}
+
 func (c Client) GetTargetsFile(role string, keyIDs []string, snapshotMeta data.Files, consistent bool, threshold int) (*data.Signed, error) {
 	// require role exists in snapshots
 	roleMeta, ok := snapshotMeta[role]
@@ -454,23 +451,17 @@ func (c Client) GetTargetsFile(role string, keyIDs []string, snapshotMeta data.F
 		} else {
 			download = true
 		}
-
 	}
 
+	size := snapshotMeta[role].Length
 	var s *data.Signed
 	if download {
 		rolePath, err := c.RoleTargetsPath(role, hex.EncodeToString(expectedSha256), consistent)
 		if err != nil {
 			return nil, err
 		}
-		raw, err = c.remote.GetMeta(rolePath, snapshotMeta[role].Length)
-		if err != nil {
-			return nil, err
-		}
-		s = &data.Signed{}
-		err = json.Unmarshal(raw, s)
+		raw, s, err = c.downloadSigned(rolePath, size, expectedSha256)
 		if err != nil {
-			logrus.Error("Error unmarshalling targets file:", err)
 			return nil, err
 		}
 	} else {

vendor/src/github.com/endophage/gotuf/client/errors.go

@@ -10,6 +10,14 @@ var (
 	ErrInsufficientKeys = errors.New("tuf: insufficient keys to meet threshold")
 )
 
+type ErrChecksumMismatch struct {
+	role string
+}
+
+func (e ErrChecksumMismatch) Error() string {
+	return fmt.Sprintf("tuf: checksum for %s did not match", e.role)
+}
+
 type ErrMissingRemoteMetadata struct {
 	Name string
 }

vendor/src/github.com/endophage/gotuf/data/keys.go

@@ -71,7 +71,7 @@ func (k TUFKey) Public() []byte {
 	return k.Value.Public
 }
 
-func (k *TUFKey) Private() []byte {
+func (k TUFKey) Private() []byte {
 	return k.Value.Private
 }
 

vendor/src/github.com/endophage/gotuf/data/roles.go

@@ -24,7 +24,7 @@ var ValidRoles = map[string]string{
 
 func SetValidRoles(rs map[string]string) {
 	// iterate ValidRoles
-	for k, _ := range ValidRoles {
+	for k := range ValidRoles {
 		if v, ok := rs[k]; ok {
 			ValidRoles[k] = v
 		}
@@ -88,6 +88,7 @@ type Role struct {
 	Name             string   `json:"name"`
 	Paths            []string `json:"paths,omitempty"`
 	PathHashPrefixes []string `json:"path_hash_prefixes,omitempty"`
+	Email            string   `json:"email,omitempty"`
 }
 
 func NewRole(name string, threshold int, keyIDs, paths, pathHashPrefixes []string) (*Role, error) {

vendor/src/github.com/endophage/gotuf/store/httpstore.go

@@ -90,6 +90,7 @@ func (s HTTPStore) GetMeta(name string, size int64) ([]byte, error) {
 	if resp.StatusCode == http.StatusNotFound {
 		return nil, ErrMetaNotFound{}
 	} else if resp.StatusCode != http.StatusOK {
+		logrus.Debugf("received HTTP status %d when requesting %s.", resp.StatusCode, name)
 		return nil, ErrServerUnavailable{code: resp.StatusCode}
 	}
 	if resp.ContentLength > size {

vendor/src/github.com/endophage/gotuf/tuf.go

@@ -71,24 +71,46 @@ func NewTufRepo(keysDB *keys.KeyDB, cryptoService signed.CryptoService) *TufRepo
 }
 
 // AddBaseKeys is used to add keys to the role in root.json
-func (tr *TufRepo) AddBaseKeys(role string, keys ...*data.TUFKey) error {
+func (tr *TufRepo) AddBaseKeys(role string, keys ...data.PublicKey) error {
 	if tr.Root == nil {
 		return ErrNotLoaded{role: "root"}
 	}
+	ids := []string{}
 	for _, k := range keys {
 		// Store only the public portion
-		pubKey := *k
-		pubKey.Value.Private = nil
-		tr.Root.Signed.Keys[pubKey.ID()] = &pubKey
-		tr.keysDB.AddKey(&pubKey)
+		pubKey := data.NewPrivateKey(k.Algorithm(), k.Public(), nil)
+		tr.Root.Signed.Keys[pubKey.ID()] = pubKey
+		tr.keysDB.AddKey(k)
 		tr.Root.Signed.Roles[role].KeyIDs = append(tr.Root.Signed.Roles[role].KeyIDs, pubKey.ID())
+		ids = append(ids, pubKey.ID())
+	}
+	r, err := data.NewRole(
+		role,
+		tr.Root.Signed.Roles[role].Threshold,
+		ids,
+		nil,
+		nil,
+	)
+	if err != nil {
+		return err
 	}
+	tr.keysDB.AddRole(r)
 	tr.Root.Dirty = true
 	return nil
 
 }
 
-// RemoveKeys is used to remove keys from the roles in root.json
+// ReplaceBaseKeys is used to replace all keys for the given role with the new keys
+func (tr *TufRepo) ReplaceBaseKeys(role string, keys ...data.PublicKey) error {
+	r := tr.keysDB.GetRole(role)
+	err := tr.RemoveBaseKeys(role, r.KeyIDs...)
+	if err != nil {
+		return err
+	}
+	return tr.AddBaseKeys(role, keys...)
+}
+
+// RemoveBaseKeys is used to remove keys from the roles in root.json
 func (tr *TufRepo) RemoveBaseKeys(role string, keyIDs ...string) error {
 	if tr.Root == nil {
 		return ErrNotLoaded{role: "root"}
@@ -119,7 +141,7 @@ func (tr *TufRepo) RemoveBaseKeys(role string, keyIDs ...string) error {
 	}
 
 	// remove keys no longer in use by any roles
-	for k, _ := range toDelete {
+	for k := range toDelete {
 		delete(tr.Root.Signed.Keys, k)
 	}
 	tr.Root.Dirty = true