Add image store
The image store abstracts image handling. It keeps track of the available images, and makes it possible to delete existing images or register new ones. The image store holds references to the underlying layers for each image. The image/v1 package provides compatibility functions for interoperating with older (non-content-addressable) image structures. Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
This commit is contained in:
Tonis Tiigi
Aaron Lehmann
parent
7de380c5c6
commit
01ba0a935b
23 changed files with 2029 additions and 144 deletions
|
|
@ -1 +0,0 @@
|
|||
sha256:f2722a8ec6926e02fa9f2674072cbc2a25cf0f449f27350f613cd843b02c9105
|
||||
|
|
@ -1 +0,0 @@
|
|||
{"architecture":"amd64","config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":null,"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"fb1f7270da9519308361b99dc8e0d30f12c24dfd28537c2337ece995ac853a16","container_config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":["/bin/sh","-c","#(nop) ADD file:11998b2a4d664a75cd0c3f4e4cb1837434e0f997ba157a0ac1d3c68a07aa2f4f in /"],"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2015-09-08T21:30:30.807853054Z","docker_version":"1.9.0-dev","layer_id":"sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a","os":"linux","parent_id":"sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02"}
|
||||
|
|
@ -1 +0,0 @@
|
|||
sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a
|
||||
|
|
@ -1 +0,0 @@
|
|||
sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02
|
||||
|
|
@ -1 +0,0 @@
|
|||
{"id":"8dfb96b5d09e6cf6f376d81f1e2770ee5ede309f9bd9e079688c9782649ab326","parent":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","created":"2015-09-08T21:30:30.807853054Z","container":"fb1f7270da9519308361b99dc8e0d30f12c24dfd28537c2337ece995ac853a16","container_config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":["/bin/sh","-c","#(nop) ADD file:11998b2a4d664a75cd0c3f4e4cb1837434e0f997ba157a0ac1d3c68a07aa2f4f in /"],"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"docker_version":"1.9.0-dev","config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":null,"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"architecture":"amd64","os":"linux"}
|
||||
|
|
@ -1 +0,0 @@
|
|||
sha256:fd6ebfedda8ea140a9380767e15bd32c6e899303cfe34bc4580c931f2f816f89
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
{"architecture":"amd64","config":{"Hostname":"03797203757d","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":null,"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"OnBuild":[],"Labels":{}},"container":"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253","container_config":{"Hostname":"03797203757d","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":["/bin/sh","-c","#(nop) ENTRYPOINT [\"/go/bin/dnsdock\"]"],"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"OnBuild":[],"Labels":{}},"created":"2015-08-19T16:49:11.368300679Z","docker_version":"1.6.2","layer_id":"sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a","os":"linux","parent_id":"sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02"}
|
||||
|
||||
|
|
@ -1 +0,0 @@
|
|||
sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a
|
||||
|
|
@ -1 +0,0 @@
|
|||
sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02
|
||||
|
|
@ -1 +0,0 @@
|
|||
{"id":"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9","parent":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","created":"2015-08-19T16:49:11.368300679Z","container":"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253","container_config":{"Hostname":"03797203757d","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":["/bin/sh","-c","#(nop) ENTRYPOINT [\"/go/bin/dnsdock\"]"],"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"NetworkDisabled":false,"MacAddress":"","OnBuild":[],"Labels":{}},"docker_version":"1.6.2","config":{"Hostname":"03797203757d","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":null,"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"NetworkDisabled":false,"MacAddress":"","OnBuild":[],"Labels":{}},"architecture":"amd64","os":"linux","Size":0}
|
||||
192
image/fs.go
Normal file
192
image/fs.go
Normal file
|
|
@ -0,0 +1,192 @@
|
|||
package image
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"github.com/docker/distribution/digest"
|
||||
)
|
||||
|
||||
// IDWalkFunc is function called by StoreBackend.Walk
|
||||
type IDWalkFunc func(id ID) error
|
||||
|
||||
// StoreBackend provides interface for image.Store persistence
|
||||
type StoreBackend interface {
|
||||
Walk(f IDWalkFunc) error
|
||||
Get(id ID) ([]byte, error)
|
||||
Set(data []byte) (ID, error)
|
||||
Delete(id ID) error
|
||||
SetMetadata(id ID, key string, data []byte) error
|
||||
GetMetadata(id ID, key string) ([]byte, error)
|
||||
DeleteMetadata(id ID, key string) error
|
||||
}
|
||||
|
||||
// fs implements StoreBackend using the filesystem.
|
||||
type fs struct {
|
||||
sync.RWMutex
|
||||
root string
|
||||
}
|
||||
|
||||
const (
|
||||
contentDirName = "content"
|
||||
metadataDirName = "metadata"
|
||||
)
|
||||
|
||||
// NewFSStoreBackend returns new filesystem based backend for image.Store
|
||||
func NewFSStoreBackend(root string) (StoreBackend, error) {
|
||||
return newFSStore(root)
|
||||
}
|
||||
|
||||
func newFSStore(root string) (*fs, error) {
|
||||
s := &fs{
|
||||
root: root,
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(root, contentDirName, string(digest.Canonical)), 0700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(root, metadataDirName, string(digest.Canonical)), 0700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *fs) contentFile(id ID) string {
|
||||
dgst := digest.Digest(id)
|
||||
return filepath.Join(s.root, contentDirName, string(dgst.Algorithm()), dgst.Hex())
|
||||
}
|
||||
|
||||
func (s *fs) metadataDir(id ID) string {
|
||||
dgst := digest.Digest(id)
|
||||
return filepath.Join(s.root, metadataDirName, string(dgst.Algorithm()), dgst.Hex())
|
||||
}
|
||||
|
||||
// Walk calls the supplied callback for each image ID in the storage backend.
|
||||
func (s *fs) Walk(f IDWalkFunc) error {
|
||||
// Only Canonical digest (sha256) is currently supported
|
||||
s.RLock()
|
||||
dir, err := ioutil.ReadDir(filepath.Join(s.root, contentDirName, string(digest.Canonical)))
|
||||
s.RUnlock()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, v := range dir {
|
||||
dgst := digest.NewDigestFromHex(string(digest.Canonical), v.Name())
|
||||
if err := dgst.Validate(); err != nil {
|
||||
logrus.Debugf("Skipping invalid digest %s: %s", dgst, err)
|
||||
continue
|
||||
}
|
||||
if err := f(ID(dgst)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get returns the content stored under a given ID.
|
||||
func (s *fs) Get(id ID) ([]byte, error) {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
return s.get(id)
|
||||
}
|
||||
|
||||
func (s *fs) get(id ID) ([]byte, error) {
|
||||
content, err := ioutil.ReadFile(s.contentFile(id))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// todo: maybe optional
|
||||
validated, err := digest.FromBytes(content)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ID(validated) != id {
|
||||
return nil, fmt.Errorf("failed to verify image: %v", id)
|
||||
}
|
||||
|
||||
return content, nil
|
||||
}
|
||||
|
||||
// Set stores content under a given ID.
|
||||
func (s *fs) Set(data []byte) (ID, error) {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
if len(data) == 0 {
|
||||
return "", fmt.Errorf("Invalid empty data")
|
||||
}
|
||||
|
||||
dgst, err := digest.FromBytes(data)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
id := ID(dgst)
|
||||
filePath := s.contentFile(id)
|
||||
tempFilePath := s.contentFile(id) + ".tmp"
|
||||
if err := ioutil.WriteFile(tempFilePath, data, 0600); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.Rename(tempFilePath, filePath); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// Delete removes content and metadata files associated with the ID.
|
||||
func (s *fs) Delete(id ID) error {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
if err := os.RemoveAll(s.metadataDir(id)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Remove(s.contentFile(id)); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetMetadata sets metadata for a given ID. It fails if there's no base file.
|
||||
func (s *fs) SetMetadata(id ID, key string, data []byte) error {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
if _, err := s.get(id); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
baseDir := filepath.Join(s.metadataDir(id))
|
||||
if err := os.MkdirAll(baseDir, 0700); err != nil {
|
||||
return err
|
||||
}
|
||||
filePath := filepath.Join(s.metadataDir(id), key)
|
||||
tempFilePath := filePath + ".tmp"
|
||||
if err := ioutil.WriteFile(tempFilePath, data, 0600); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tempFilePath, filePath)
|
||||
}
|
||||
|
||||
// GetMetadata returns metadata for a given ID.
|
||||
func (s *fs) GetMetadata(id ID, key string) ([]byte, error) {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
if _, err := s.get(id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ioutil.ReadFile(filepath.Join(s.metadataDir(id), key))
|
||||
}
|
||||
|
||||
// DeleteMetadata removes the metadata associated with an ID.
|
||||
func (s *fs) DeleteMetadata(id ID, key string) error {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
return os.RemoveAll(filepath.Join(s.metadataDir(id), key))
|
||||
}
|
||||
391
image/fs_test.go
Normal file
391
image/fs_test.go
Normal file
|
|
@ -0,0 +1,391 @@
|
|||
package image
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/docker/distribution/digest"
|
||||
)
|
||||
|
||||
func TestFSGetSet(t *testing.T) {
|
||||
tmpdir, err := ioutil.TempDir("", "images-fs-store")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpdir)
|
||||
fs, err := NewFSStoreBackend(tmpdir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
testGetSet(t, fs)
|
||||
}
|
||||
|
||||
func TestFSGetInvalidData(t *testing.T) {
|
||||
tmpdir, err := ioutil.TempDir("", "images-fs-store")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpdir)
|
||||
fs, err := NewFSStoreBackend(tmpdir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
id, err := fs.Set([]byte("foobar"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dgst := digest.Digest(id)
|
||||
|
||||
if err := ioutil.WriteFile(filepath.Join(tmpdir, contentDirName, string(dgst.Algorithm()), dgst.Hex()), []byte("foobar2"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = fs.Get(id)
|
||||
if err == nil {
|
||||
t.Fatal("Expected get to fail after data modification.")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFSInvalidSet(t *testing.T) {
|
||||
tmpdir, err := ioutil.TempDir("", "images-fs-store")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpdir)
|
||||
fs, err := NewFSStoreBackend(tmpdir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
id, err := digest.FromBytes([]byte("foobar"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = os.Mkdir(filepath.Join(tmpdir, contentDirName, string(id.Algorithm()), id.Hex()), 0700)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = fs.Set([]byte("foobar"))
|
||||
if err == nil {
|
||||
t.Fatal("Expecting error from invalid filesystem data.")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFSInvalidRoot(t *testing.T) {
|
||||
tmpdir, err := ioutil.TempDir("", "images-fs-store")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpdir)
|
||||
|
||||
tcases := []struct {
|
||||
root, invalidFile string
|
||||
}{
|
||||
{"root", "root"},
|
||||
{"root", "root/content"},
|
||||
{"root", "root/metadata"},
|
||||
}
|
||||
|
||||
for _, tc := range tcases {
|
||||
root := filepath.Join(tmpdir, tc.root)
|
||||
filePath := filepath.Join(tmpdir, tc.invalidFile)
|
||||
err := os.MkdirAll(filepath.Dir(filePath), 0700)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := os.Create(filePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.Close()
|
||||
|
||||
_, err = NewFSStoreBackend(root)
|
||||
if err == nil {
|
||||
t.Fatalf("Expected error from root %q and invlid file %q", tc.root, tc.invalidFile)
|
||||
}
|
||||
|
||||
os.RemoveAll(root)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func testMetadataGetSet(t *testing.T, store StoreBackend) {
|
||||
id, err := store.Set([]byte("foo"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id2, err := store.Set([]byte("bar"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
tcases := []struct {
|
||||
id ID
|
||||
key string
|
||||
value []byte
|
||||
}{
|
||||
{id, "tkey", []byte("tval1")},
|
||||
{id, "tkey2", []byte("tval2")},
|
||||
{id2, "tkey", []byte("tval3")},
|
||||
}
|
||||
|
||||
for _, tc := range tcases {
|
||||
err = store.SetMetadata(tc.id, tc.key, tc.value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
actual, err := store.GetMetadata(tc.id, tc.key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if bytes.Compare(actual, tc.value) != 0 {
|
||||
t.Fatalf("Metadata expected %q, got %q", tc.value, actual)
|
||||
}
|
||||
}
|
||||
|
||||
_, err = store.GetMetadata(id2, "tkey2")
|
||||
if err == nil {
|
||||
t.Fatal("Expected error for getting metadata for unknown key")
|
||||
}
|
||||
|
||||
id3, err := digest.FromBytes([]byte("baz"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = store.SetMetadata(ID(id3), "tkey", []byte("tval"))
|
||||
if err == nil {
|
||||
t.Fatal("Expected error for setting metadata for unknown ID.")
|
||||
}
|
||||
|
||||
_, err = store.GetMetadata(ID(id3), "tkey")
|
||||
if err == nil {
|
||||
t.Fatal("Expected error for getting metadata for unknown ID.")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFSMetadataGetSet(t *testing.T) {
|
||||
tmpdir, err := ioutil.TempDir("", "images-fs-store")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpdir)
|
||||
fs, err := NewFSStoreBackend(tmpdir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
testMetadataGetSet(t, fs)
|
||||
}
|
||||
|
||||
func TestFSDelete(t *testing.T) {
|
||||
tmpdir, err := ioutil.TempDir("", "images-fs-store")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpdir)
|
||||
fs, err := NewFSStoreBackend(tmpdir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
testDelete(t, fs)
|
||||
}
|
||||
|
||||
func TestFSWalker(t *testing.T) {
|
||||
tmpdir, err := ioutil.TempDir("", "images-fs-store")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpdir)
|
||||
fs, err := NewFSStoreBackend(tmpdir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
testWalker(t, fs)
|
||||
}
|
||||
|
||||
func TestFSInvalidWalker(t *testing.T) {
|
||||
tmpdir, err := ioutil.TempDir("", "images-fs-store")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpdir)
|
||||
fs, err := NewFSStoreBackend(tmpdir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
fooID, err := fs.Set([]byte("foo"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := ioutil.WriteFile(filepath.Join(tmpdir, contentDirName, "sha256/foobar"), []byte("foobar"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
n := 0
|
||||
err = fs.Walk(func(id ID) error {
|
||||
if id != fooID {
|
||||
t.Fatalf("Invalid walker ID %q, expected %q", id, fooID)
|
||||
}
|
||||
n++
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Invalid data should not have caused walker error, got %v", err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("Expected 1 walk initialization, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func testGetSet(t *testing.T, store StoreBackend) {
|
||||
type tcase struct {
|
||||
input []byte
|
||||
expected ID
|
||||
}
|
||||
tcases := []tcase{
|
||||
{[]byte("foobar"), ID("sha256:c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2")},
|
||||
}
|
||||
|
||||
randomInput := make([]byte, 8*1024)
|
||||
_, err := rand.Read(randomInput)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// skipping use of digest pkg because its used by the imlementation
|
||||
h := sha256.New()
|
||||
_, err = h.Write(randomInput)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tcases = append(tcases, tcase{
|
||||
input: randomInput,
|
||||
expected: ID("sha256:" + hex.EncodeToString(h.Sum(nil))),
|
||||
})
|
||||
|
||||
for _, tc := range tcases {
|
||||
id, err := store.Set([]byte(tc.input))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if id != tc.expected {
|
||||
t.Fatalf("Expected ID %q, got %q", tc.expected, id)
|
||||
}
|
||||
}
|
||||
|
||||
for _, emptyData := range [][]byte{nil, {}} {
|
||||
_, err := store.Set(emptyData)
|
||||
if err == nil {
|
||||
t.Fatal("Expected error for nil input.")
|
||||
}
|
||||
}
|
||||
|
||||
for _, tc := range tcases {
|
||||
data, err := store.Get(tc.expected)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if bytes.Compare(data, tc.input) != 0 {
|
||||
t.Fatalf("Expected data %q, got %q", tc.input, data)
|
||||
}
|
||||
}
|
||||
|
||||
for _, key := range []ID{"foobar:abc", "sha256:abc", "sha256:c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2a"} {
|
||||
_, err := store.Get(key)
|
||||
if err == nil {
|
||||
t.Fatalf("Expected error for ID %q.", key)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func testDelete(t *testing.T, store StoreBackend) {
|
||||
id, err := store.Set([]byte("foo"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id2, err := store.Set([]byte("bar"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = store.Delete(id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = store.Get(id)
|
||||
if err == nil {
|
||||
t.Fatalf("Expected getting deleted item %q to fail", id)
|
||||
}
|
||||
_, err = store.Get(id2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = store.Delete(id2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = store.Get(id2)
|
||||
if err == nil {
|
||||
t.Fatalf("Expected getting deleted item %q to fail", id2)
|
||||
}
|
||||
}
|
||||
|
||||
func testWalker(t *testing.T, store StoreBackend) {
|
||||
id, err := store.Set([]byte("foo"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id2, err := store.Set([]byte("bar"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
tcases := make(map[ID]struct{})
|
||||
tcases[id] = struct{}{}
|
||||
tcases[id2] = struct{}{}
|
||||
n := 0
|
||||
err = store.Walk(func(id ID) error {
|
||||
delete(tcases, id)
|
||||
n++
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if n != 2 {
|
||||
t.Fatalf("Expected 2 walk initializations, got %d", n)
|
||||
}
|
||||
if len(tcases) != 0 {
|
||||
t.Fatalf("Expected empty unwalked set, got %+v", tcases)
|
||||
}
|
||||
|
||||
// stop on error
|
||||
tcases = make(map[ID]struct{})
|
||||
tcases[id] = struct{}{}
|
||||
err = store.Walk(func(id ID) error {
|
||||
return errors.New("")
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("Exected error from walker.")
|
||||
}
|
||||
}
|
||||
163
image/image.go
163
image/image.go
|
|
@ -2,36 +2,23 @@ package image
|
|||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"errors"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"github.com/docker/distribution/digest"
|
||||
derr "github.com/docker/docker/errors"
|
||||
"github.com/docker/docker/pkg/version"
|
||||
"github.com/docker/docker/runconfig"
|
||||
)
|
||||
|
||||
var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`)
|
||||
// ID is the content-addressable ID of an image.
|
||||
type ID digest.Digest
|
||||
|
||||
// noFallbackMinVersion is the minimum version for which v1compatibility
|
||||
// information will not be marshaled through the Image struct to remove
|
||||
// blank fields.
|
||||
var noFallbackMinVersion = version.Version("1.8.3")
|
||||
|
||||
// Descriptor provides the information necessary to register an image in
|
||||
// the graph.
|
||||
type Descriptor interface {
|
||||
ID() string
|
||||
Parent() string
|
||||
MarshalConfig() ([]byte, error)
|
||||
func (id ID) String() string {
|
||||
return digest.Digest(id).String()
|
||||
}
|
||||
|
||||
// Image stores the image configuration.
|
||||
// All fields in this struct must be marked `omitempty` to keep getting
|
||||
// predictable hashes from the old `v1Compatibility` configuration.
|
||||
type Image struct {
|
||||
// V1Image stores the V1 image configuration.
|
||||
type V1Image struct {
|
||||
// ID a unique 64 character identifier of the image
|
||||
ID string `json:"id,omitempty"`
|
||||
// Parent id of the image
|
||||
|
|
@ -55,95 +42,87 @@ type Image struct {
|
|||
// OS is the operating system used to build and run the image
|
||||
OS string `json:"os,omitempty"`
|
||||
// Size is the total size of the image including all layers it is composed of
|
||||
Size int64 `json:",omitempty"` // capitalized for backwards compatibility
|
||||
// ParentID specifies the strong, content address of the parent configuration.
|
||||
ParentID digest.Digest `json:"parent_id,omitempty"`
|
||||
// LayerID provides the content address of the associated layer.
|
||||
LayerID digest.Digest `json:"layer_id,omitempty"`
|
||||
Size int64 `json:",omitempty"`
|
||||
}
|
||||
|
||||
// NewImgJSON creates an Image configuration from json.
|
||||
func NewImgJSON(src []byte) (*Image, error) {
|
||||
ret := &Image{}
|
||||
// Image stores the image configuration
|
||||
type Image struct {
|
||||
V1Image
|
||||
Parent ID `json:"parent,omitempty"`
|
||||
RootFS *RootFS `json:"rootfs,omitempty"`
|
||||
History []History `json:"history,omitempty"`
|
||||
|
||||
// FIXME: Is there a cleaner way to "purify" the input json?
|
||||
if err := json.Unmarshal(src, ret); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ret, nil
|
||||
// rawJSON caches the immutable JSON associated with this image.
|
||||
rawJSON []byte
|
||||
|
||||
// computedID is the ID computed from the hash of the image config.
|
||||
// Not to be confused with the legacy V1 ID in V1Image.
|
||||
computedID ID
|
||||
}
|
||||
|
||||
// ValidateID checks whether an ID string is a valid image ID.
|
||||
func ValidateID(id string) error {
|
||||
if ok := validHex.MatchString(id); !ok {
|
||||
return derr.ErrorCodeInvalidImageID.WithArgs(id)
|
||||
}
|
||||
return nil
|
||||
// RawJSON returns the immutable JSON associated with the image.
|
||||
func (img *Image) RawJSON() []byte {
|
||||
return img.rawJSON
|
||||
}
|
||||
|
||||
// MakeImageConfig returns immutable configuration JSON for image based on the
|
||||
// v1Compatibility object, layer digest and parent StrongID. SHA256() of this
|
||||
// config is the new image ID (strongID).
|
||||
func MakeImageConfig(v1Compatibility []byte, layerID, parentID digest.Digest) ([]byte, error) {
|
||||
// ID returns the image's content-addressable ID.
|
||||
func (img *Image) ID() ID {
|
||||
return img.computedID
|
||||
}
|
||||
|
||||
// Detect images created after 1.8.3
|
||||
img, err := NewImgJSON(v1Compatibility)
|
||||
// MarshalJSON serializes the image to JSON. It sorts the top-level keys so
|
||||
// that JSON that's been manipulated by a push/pull cycle with a legacy
|
||||
// registry won't end up with a different key order.
|
||||
func (img *Image) MarshalJSON() ([]byte, error) {
|
||||
type MarshalImage Image
|
||||
|
||||
pass1, err := json.Marshal(MarshalImage(*img))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
useFallback := version.Version(img.DockerVersion).LessThan(noFallbackMinVersion)
|
||||
|
||||
if useFallback {
|
||||
// Fallback for pre-1.8.3. Calculate base config based on Image struct
|
||||
// so that fields with default values added by Docker will use same ID
|
||||
logrus.Debugf("Using fallback hash for %v", layerID)
|
||||
|
||||
v1Compatibility, err = json.Marshal(img)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
var c map[string]*json.RawMessage
|
||||
if err := json.Unmarshal(v1Compatibility, &c); err != nil {
|
||||
if err := json.Unmarshal(pass1, &c); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := layerID.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("invalid layerID: %v", err)
|
||||
}
|
||||
|
||||
c["layer_id"] = rawJSON(layerID)
|
||||
|
||||
if parentID != "" {
|
||||
if err := parentID.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("invalid parentID %v", err)
|
||||
}
|
||||
c["parent_id"] = rawJSON(parentID)
|
||||
}
|
||||
|
||||
delete(c, "id")
|
||||
delete(c, "parent")
|
||||
delete(c, "Size") // Size is calculated from data on disk and is inconsitent
|
||||
|
||||
return json.Marshal(c)
|
||||
}
|
||||
|
||||
// StrongID returns image ID for the config JSON.
|
||||
func StrongID(configJSON []byte) (digest.Digest, error) {
|
||||
digester := digest.Canonical.New()
|
||||
if _, err := digester.Hash().Write(configJSON); err != nil {
|
||||
return "", err
|
||||
}
|
||||
dgst := digester.Digest()
|
||||
logrus.Debugf("H(%v) = %v", string(configJSON), dgst)
|
||||
return dgst, nil
|
||||
// History stores build commands that were used to create an image
|
||||
type History struct {
|
||||
// Created timestamp for build point
|
||||
Created time.Time `json:"created"`
|
||||
// Author of the build point
|
||||
Author string `json:"author,omitempty"`
|
||||
// CreatedBy keeps the Dockerfile command used while building image.
|
||||
CreatedBy string `json:"created_by,omitempty"`
|
||||
// Comment is custom mesage set by the user when creating the image.
|
||||
Comment string `json:"comment,omitempty"`
|
||||
// EmptyLayer is set to true if this history item did not generate a
|
||||
// layer. Otherwise, the history item is associated with the next
|
||||
// layer in the RootFS section.
|
||||
EmptyLayer bool `json:"empty_layer,omitempty"`
|
||||
}
|
||||
|
||||
func rawJSON(value interface{}) *json.RawMessage {
|
||||
jsonval, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return (*json.RawMessage)(&jsonval)
|
||||
// Exporter provides interface for exporting and importing images
|
||||
type Exporter interface {
|
||||
Load(io.ReadCloser, io.Writer) error
|
||||
// TODO: Load(net.Context, io.ReadCloser, <- chan StatusMessage) error
|
||||
Save([]string, io.Writer) error
|
||||
}
|
||||
|
||||
// NewFromJSON creates an Image configuration from json.
|
||||
func NewFromJSON(src []byte) (*Image, error) {
|
||||
img := &Image{}
|
||||
|
||||
if err := json.Unmarshal(src, img); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if img.RootFS == nil {
|
||||
return nil, errors.New("Invalid image JSON, no RootFS key.")
|
||||
}
|
||||
|
||||
img.rawJSON = src
|
||||
|
||||
return img, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,55 +1,59 @@
|
|||
package image
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io/ioutil"
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/docker/distribution/digest"
|
||||
)
|
||||
|
||||
var fixtures = []string{
|
||||
"fixtures/pre1.9",
|
||||
"fixtures/post1.9",
|
||||
}
|
||||
const sampleImageJSON = `{
|
||||
"architecture": "amd64",
|
||||
"os": "linux",
|
||||
"config": {},
|
||||
"rootfs": {
|
||||
"type": "layers",
|
||||
"diff_ids": []
|
||||
}
|
||||
}`
|
||||
|
||||
func loadFixtureFile(t *testing.T, path string) []byte {
|
||||
fileData, err := ioutil.ReadFile(path)
|
||||
func TestJSON(t *testing.T) {
|
||||
img, err := NewFromJSON([]byte(sampleImageJSON))
|
||||
if err != nil {
|
||||
t.Fatalf("error opening %s: %v", path, err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return bytes.TrimSpace(fileData)
|
||||
}
|
||||
|
||||
// TestMakeImageConfig makes sure that MakeImageConfig returns the expected
|
||||
// canonical JSON for a reference Image.
|
||||
func TestMakeImageConfig(t *testing.T) {
|
||||
for _, fixture := range fixtures {
|
||||
v1Compatibility := loadFixtureFile(t, fixture+"/v1compatibility")
|
||||
expectedConfig := loadFixtureFile(t, fixture+"/expected_config")
|
||||
layerID := digest.Digest(loadFixtureFile(t, fixture+"/layer_id"))
|
||||
parentID := digest.Digest(loadFixtureFile(t, fixture+"/parent_id"))
|
||||
|
||||
json, err := MakeImageConfig(v1Compatibility, layerID, parentID)
|
||||
if err != nil {
|
||||
t.Fatalf("MakeImageConfig on %s returned error: %v", fixture, err)
|
||||
}
|
||||
if !bytes.Equal(json, expectedConfig) {
|
||||
t.Fatalf("did not get expected JSON for %s\nexpected: %s\ngot: %s", fixture, expectedConfig, json)
|
||||
}
|
||||
rawJSON := img.RawJSON()
|
||||
if string(rawJSON) != sampleImageJSON {
|
||||
t.Fatalf("Raw JSON of config didn't match: expected %+v, got %v", sampleImageJSON, rawJSON)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGetStrongID makes sure that GetConfigJSON returns the expected
|
||||
// hash for a reference Image.
|
||||
func TestGetStrongID(t *testing.T) {
|
||||
for _, fixture := range fixtures {
|
||||
expectedConfig := loadFixtureFile(t, fixture+"/expected_config")
|
||||
expectedComputedID := digest.Digest(loadFixtureFile(t, fixture+"/expected_computed_id"))
|
||||
|
||||
if id, err := StrongID(expectedConfig); err != nil || id != expectedComputedID {
|
||||
t.Fatalf("did not get expected ID for %s\nexpected: %s\ngot: %s\nerror: %v", fixture, expectedComputedID, id, err)
|
||||
}
|
||||
func TestInvalidJSON(t *testing.T) {
|
||||
_, err := NewFromJSON([]byte("{}"))
|
||||
if err == nil {
|
||||
t.Fatal("Expected JSON parse error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalKeyOrder(t *testing.T) {
|
||||
b, err := json.Marshal(&Image{
|
||||
V1Image: V1Image{
|
||||
Comment: "a",
|
||||
Author: "b",
|
||||
Architecture: "c",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
expectedOrder := []string{"architecture", "author", "comment"}
|
||||
var indexes []int
|
||||
for _, k := range expectedOrder {
|
||||
indexes = append(indexes, strings.Index(string(b), k))
|
||||
}
|
||||
|
||||
if !sort.IntsAreSorted(indexes) {
|
||||
t.Fatal("invalid key order in JSON: ", string(b))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
8
image/rootfs.go
Normal file
8
image/rootfs.go
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
package image
|
||||
|
||||
import "github.com/docker/docker/layer"
|
||||
|
||||
// Append appends a new diffID to rootfs
|
||||
func (r *RootFS) Append(id layer.DiffID) {
|
||||
r.DiffIDs = append(r.DiffIDs, id)
|
||||
}
|
||||
23
image/rootfs_unix.go
Normal file
23
image/rootfs_unix.go
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
// +build !windows
|
||||
|
||||
package image
|
||||
|
||||
import "github.com/docker/docker/layer"
|
||||
|
||||
// RootFS describes images root filesystem
|
||||
// This is currently a placeholder that only supports layers. In the future
|
||||
// this can be made into a interface that supports different implementaions.
|
||||
type RootFS struct {
|
||||
Type string `json:"type"`
|
||||
DiffIDs []layer.DiffID `json:"diff_ids,omitempty"`
|
||||
}
|
||||
|
||||
// ChainID returns the ChainID for the top layer in RootFS.
|
||||
func (r *RootFS) ChainID() layer.ChainID {
|
||||
return layer.CreateChainID(r.DiffIDs)
|
||||
}
|
||||
|
||||
// NewRootFS returns empty RootFS struct
|
||||
func NewRootFS() *RootFS {
|
||||
return &RootFS{Type: "layers"}
|
||||
}
|
||||
37
image/rootfs_windows.go
Normal file
37
image/rootfs_windows.go
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
// +build windows
|
||||
|
||||
package image
|
||||
|
||||
import (
|
||||
"crypto/sha512"
|
||||
"fmt"
|
||||
|
||||
"github.com/docker/distribution/digest"
|
||||
"github.com/docker/docker/layer"
|
||||
)
|
||||
|
||||
// RootFS describes images root filesystem
|
||||
// This is currently a placeholder that only supports layers. In the future
|
||||
// this can be made into a interface that supports different implementaions.
|
||||
type RootFS struct {
|
||||
Type string `json:"type"`
|
||||
DiffIDs []layer.DiffID `json:"diff_ids,omitempty"`
|
||||
BaseLayer string `json:"base_layer,omitempty"`
|
||||
}
|
||||
|
||||
// BaseLayerID returns the 64 byte hex ID for the baselayer name.
|
||||
func (r *RootFS) BaseLayerID() string {
|
||||
baseID := sha512.Sum384([]byte(r.BaseLayer))
|
||||
return fmt.Sprintf("%x", baseID[:32])
|
||||
}
|
||||
|
||||
// ChainID returns the ChainID for the top layer in RootFS.
|
||||
func (r *RootFS) ChainID() layer.ChainID {
|
||||
baseDiffID, _ := digest.FromBytes([]byte(r.BaseLayerID())) // can never error
|
||||
return layer.CreateChainID(append([]layer.DiffID{layer.DiffID(baseDiffID)}, r.DiffIDs...))
|
||||
}
|
||||
|
||||
// NewRootFS returns empty RootFS struct
|
||||
func NewRootFS() *RootFS {
|
||||
return &RootFS{Type: "layers+base"}
|
||||
}
|
||||
286
image/store.go
Normal file
286
image/store.go
Normal file
|
|
@ -0,0 +1,286 @@
|
|||
package image
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"github.com/docker/distribution/digest"
|
||||
"github.com/docker/docker/layer"
|
||||
)
|
||||
|
||||
// Store is an interface for creating and accessing images
|
||||
type Store interface {
|
||||
Create(config []byte) (ID, error)
|
||||
Get(id ID) (*Image, error)
|
||||
Delete(id ID) ([]layer.Metadata, error)
|
||||
Search(partialID string) (ID, error)
|
||||
SetParent(id ID, parent ID) error
|
||||
GetParent(id ID) (ID, error)
|
||||
Children(id ID) []ID
|
||||
Map() map[ID]*Image
|
||||
Heads() map[ID]*Image
|
||||
}
|
||||
|
||||
// LayerGetReleaser is a minimal interface for getting and releasing images.
|
||||
type LayerGetReleaser interface {
|
||||
Get(layer.ChainID) (layer.Layer, error)
|
||||
Release(layer.Layer) ([]layer.Metadata, error)
|
||||
}
|
||||
|
||||
type imageMeta struct {
|
||||
layer layer.Layer
|
||||
children map[ID]struct{}
|
||||
}
|
||||
|
||||
type store struct {
|
||||
sync.Mutex
|
||||
ls LayerGetReleaser
|
||||
images map[ID]*imageMeta
|
||||
fs StoreBackend
|
||||
digestSet *digest.Set
|
||||
}
|
||||
|
||||
// NewImageStore returns new store object for given layer store
|
||||
func NewImageStore(fs StoreBackend, ls LayerGetReleaser) (Store, error) {
|
||||
is := &store{
|
||||
ls: ls,
|
||||
images: make(map[ID]*imageMeta),
|
||||
fs: fs,
|
||||
digestSet: digest.NewSet(),
|
||||
}
|
||||
|
||||
// load all current images and retain layers
|
||||
if err := is.restore(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return is, nil
|
||||
}
|
||||
|
||||
func (is *store) restore() error {
|
||||
err := is.fs.Walk(func(id ID) error {
|
||||
img, err := is.Get(id)
|
||||
if err != nil {
|
||||
logrus.Errorf("invalid image %v, %v", id, err)
|
||||
return nil
|
||||
}
|
||||
var l layer.Layer
|
||||
if chainID := img.RootFS.ChainID(); chainID != "" {
|
||||
l, err = is.ls.Get(chainID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := is.digestSet.Add(digest.Digest(id)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
imageMeta := &imageMeta{
|
||||
layer: l,
|
||||
children: make(map[ID]struct{}),
|
||||
}
|
||||
|
||||
is.images[ID(id)] = imageMeta
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Second pass to fill in children maps
|
||||
for id := range is.images {
|
||||
if parent, err := is.GetParent(id); err == nil {
|
||||
if parentMeta := is.images[parent]; parentMeta != nil {
|
||||
parentMeta.children[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (is *store) Create(config []byte) (ID, error) {
|
||||
var img Image
|
||||
err := json.Unmarshal(config, &img)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Must reject any config that references diffIDs from the history
|
||||
// which aren't among the rootfs layers.
|
||||
rootFSLayers := make(map[layer.DiffID]struct{})
|
||||
for _, diffID := range img.RootFS.DiffIDs {
|
||||
rootFSLayers[diffID] = struct{}{}
|
||||
}
|
||||
|
||||
layerCounter := 0
|
||||
for _, h := range img.History {
|
||||
if !h.EmptyLayer {
|
||||
layerCounter++
|
||||
}
|
||||
}
|
||||
if layerCounter > len(img.RootFS.DiffIDs) {
|
||||
return "", errors.New("too many non-empty layers in History section")
|
||||
}
|
||||
|
||||
dgst, err := is.fs.Set(config)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
imageID := ID(dgst)
|
||||
|
||||
is.Lock()
|
||||
defer is.Unlock()
|
||||
|
||||
if _, exists := is.images[imageID]; exists {
|
||||
return imageID, nil
|
||||
}
|
||||
|
||||
layerID := img.RootFS.ChainID()
|
||||
|
||||
var l layer.Layer
|
||||
if layerID != "" {
|
||||
l, err = is.ls.Get(layerID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
imageMeta := &imageMeta{
|
||||
layer: l,
|
||||
children: make(map[ID]struct{}),
|
||||
}
|
||||
|
||||
is.images[imageID] = imageMeta
|
||||
if err := is.digestSet.Add(digest.Digest(imageID)); err != nil {
|
||||
delete(is.images, imageID)
|
||||
return "", err
|
||||
}
|
||||
|
||||
return imageID, nil
|
||||
}
|
||||
|
||||
func (is *store) Search(term string) (ID, error) {
|
||||
is.Lock()
|
||||
defer is.Unlock()
|
||||
|
||||
dgst, err := is.digestSet.Lookup(term)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return ID(dgst), nil
|
||||
}
|
||||
|
||||
func (is *store) Get(id ID) (*Image, error) {
|
||||
// todo: Check if image is in images
|
||||
// todo: Detect manual insertions and start using them
|
||||
config, err := is.fs.Get(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
img, err := NewFromJSON(config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
img.computedID = id
|
||||
|
||||
img.Parent, err = is.GetParent(id)
|
||||
if err != nil {
|
||||
img.Parent = ""
|
||||
}
|
||||
|
||||
return img, nil
|
||||
}
|
||||
|
||||
func (is *store) Delete(id ID) ([]layer.Metadata, error) {
|
||||
is.Lock()
|
||||
defer is.Unlock()
|
||||
|
||||
imageMeta := is.images[id]
|
||||
if imageMeta == nil {
|
||||
return nil, fmt.Errorf("unrecognized image ID %s", id.String())
|
||||
}
|
||||
for id := range imageMeta.children {
|
||||
is.fs.DeleteMetadata(id, "parent")
|
||||
}
|
||||
if parent, err := is.GetParent(id); err == nil && is.images[parent] != nil {
|
||||
delete(is.images[parent].children, id)
|
||||
}
|
||||
|
||||
delete(is.images, id)
|
||||
is.fs.Delete(id)
|
||||
|
||||
if imageMeta.layer != nil {
|
||||
return is.ls.Release(imageMeta.layer)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (is *store) SetParent(id, parent ID) error {
|
||||
is.Lock()
|
||||
defer is.Unlock()
|
||||
parentMeta := is.images[parent]
|
||||
if parentMeta == nil {
|
||||
return fmt.Errorf("unknown parent image ID %s", parent.String())
|
||||
}
|
||||
parentMeta.children[id] = struct{}{}
|
||||
return is.fs.SetMetadata(id, "parent", []byte(parent))
|
||||
}
|
||||
|
||||
func (is *store) GetParent(id ID) (ID, error) {
|
||||
d, err := is.fs.GetMetadata(id, "parent")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return ID(d), nil // todo: validate?
|
||||
}
|
||||
|
||||
func (is *store) Children(id ID) []ID {
|
||||
is.Lock()
|
||||
defer is.Unlock()
|
||||
|
||||
return is.children(id)
|
||||
}
|
||||
|
||||
func (is *store) children(id ID) []ID {
|
||||
var ids []ID
|
||||
if is.images[id] != nil {
|
||||
for id := range is.images[id].children {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func (is *store) Heads() map[ID]*Image {
|
||||
return is.imagesMap(false)
|
||||
}
|
||||
|
||||
func (is *store) Map() map[ID]*Image {
|
||||
return is.imagesMap(true)
|
||||
}
|
||||
|
||||
func (is *store) imagesMap(all bool) map[ID]*Image {
|
||||
is.Lock()
|
||||
defer is.Unlock()
|
||||
|
||||
images := make(map[ID]*Image)
|
||||
|
||||
for id := range is.images {
|
||||
if !all && len(is.children(id)) > 0 {
|
||||
continue
|
||||
}
|
||||
img, err := is.Get(id)
|
||||
if err != nil {
|
||||
logrus.Errorf("invalid image access: %q, error: %q", id, err)
|
||||
continue
|
||||
}
|
||||
images[id] = img
|
||||
}
|
||||
return images
|
||||
}
|
||||
205
image/store_test.go
Normal file
205
image/store_test.go
Normal file
|
|
@ -0,0 +1,205 @@
|
|||
package image
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/docker/distribution/digest"
|
||||
"github.com/docker/docker/layer"
|
||||
)
|
||||
|
||||
func TestRestore(t *testing.T) {
|
||||
tmpdir, err := ioutil.TempDir("", "images-fs-store")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpdir)
|
||||
fs, err := NewFSStoreBackend(tmpdir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
id1, err := fs.Set([]byte(`{"comment": "abc", "rootfs": {"type": "layers"}}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = fs.Set([]byte(`invalid`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id2, err := fs.Set([]byte(`{"comment": "def", "rootfs": {"type": "layers", "diff_ids": ["2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"]}}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = fs.SetMetadata(id2, "parent", []byte(id1))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
is, err := NewImageStore(fs, &mockLayerGetReleaser{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
imgs := is.Map()
|
||||
if actual, expected := len(imgs), 2; actual != expected {
|
||||
t.Fatalf("invalid images length, expected 2, got %q", len(imgs))
|
||||
}
|
||||
|
||||
img1, err := is.Get(ID(id1))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if actual, expected := img1.computedID, ID(id1); actual != expected {
|
||||
t.Fatalf("invalid image ID: expected %q, got %q", expected, actual)
|
||||
}
|
||||
|
||||
if actual, expected := img1.computedID.String(), string(id1); actual != expected {
|
||||
t.Fatalf("invalid image ID string: expected %q, got %q", expected, actual)
|
||||
}
|
||||
|
||||
img2, err := is.Get(ID(id2))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if actual, expected := img1.Comment, "abc"; actual != expected {
|
||||
t.Fatalf("invalid comment for image1: expected %q, got %q", expected, actual)
|
||||
}
|
||||
|
||||
if actual, expected := img2.Comment, "def"; actual != expected {
|
||||
t.Fatalf("invalid comment for image2: expected %q, got %q", expected, actual)
|
||||
}
|
||||
|
||||
p, err := is.GetParent(ID(id1))
|
||||
if err == nil {
|
||||
t.Fatal("expected error for getting parent")
|
||||
}
|
||||
|
||||
p, err = is.GetParent(ID(id2))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if actual, expected := p, ID(id1); actual != expected {
|
||||
t.Fatalf("invalid parent: expected %q, got %q", expected, actual)
|
||||
}
|
||||
|
||||
children := is.Children(ID(id1))
|
||||
if len(children) != 1 {
|
||||
t.Fatalf("invalid children length: %q", len(children))
|
||||
}
|
||||
if actual, expected := children[0], ID(id2); actual != expected {
|
||||
t.Fatalf("invalid child for id1: expected %q, got %q", expected, actual)
|
||||
}
|
||||
|
||||
heads := is.Heads()
|
||||
if actual, expected := len(heads), 1; actual != expected {
|
||||
t.Fatalf("invalid images length: expected %q, got %q", expected, actual)
|
||||
}
|
||||
|
||||
sid1, err := is.Search(string(id1)[:10])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if actual, expected := sid1, ID(id1); actual != expected {
|
||||
t.Fatalf("searched ID mismatch: expected %q, got %q", expected, actual)
|
||||
}
|
||||
|
||||
sid1, err = is.Search(digest.Digest(id1).Hex()[:6])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if actual, expected := sid1, ID(id1); actual != expected {
|
||||
t.Fatalf("searched ID mismatch: expected %q, got %q", expected, actual)
|
||||
}
|
||||
|
||||
invalidPattern := digest.Digest(id1).Hex()[1:6]
|
||||
_, err = is.Search(invalidPattern)
|
||||
if err == nil {
|
||||
t.Fatalf("expected search for %q to fail", invalidPattern)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestAddDelete(t *testing.T) {
|
||||
tmpdir, err := ioutil.TempDir("", "images-fs-store")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpdir)
|
||||
fs, err := NewFSStoreBackend(tmpdir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
is, err := NewImageStore(fs, &mockLayerGetReleaser{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
id1, err := is.Create([]byte(`{"comment": "abc", "rootfs": {"type": "layers", "diff_ids": ["2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"]}}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if actual, expected := id1, ID("sha256:8d25a9c45df515f9d0fe8e4a6b1c64dd3b965a84790ddbcc7954bb9bc89eb993"); actual != expected {
|
||||
t.Fatalf("create ID mismatch: expected %q, got %q", expected, actual)
|
||||
}
|
||||
|
||||
img, err := is.Get(id1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if actual, expected := img.Comment, "abc"; actual != expected {
|
||||
t.Fatalf("invalid comment in image: expected %q, got %q", expected, actual)
|
||||
}
|
||||
|
||||
id2, err := is.Create([]byte(`{"comment": "def", "rootfs": {"type": "layers", "diff_ids": ["2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"]}}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = is.SetParent(id2, id1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
pid1, err := is.GetParent(id2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if actual, expected := pid1, id1; actual != expected {
|
||||
t.Fatalf("invalid parent for image: expected %q, got %q", expected, actual)
|
||||
}
|
||||
|
||||
_, err = is.Delete(id1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = is.Get(id1)
|
||||
if err == nil {
|
||||
t.Fatalf("expected get for deleted image %q to fail", id1)
|
||||
}
|
||||
_, err = is.Get(id2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pid1, err = is.GetParent(id2)
|
||||
if err == nil {
|
||||
t.Fatalf("expected parent check for image %q to fail, got %q", id2, pid1)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
type mockLayerGetReleaser struct{}
|
||||
|
||||
func (ls *mockLayerGetReleaser) Get(layer.ChainID) (layer.Layer, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (ls *mockLayerGetReleaser) Release(layer.Layer) ([]layer.Metadata, error) {
|
||||
return nil, nil
|
||||
}
|
||||
284
image/tarexport/load.go
Normal file
284
image/tarexport/load.go
Normal file
|
|
@ -0,0 +1,284 @@
|
|||
package tarexport
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"github.com/docker/distribution/reference"
|
||||
"github.com/docker/docker/image"
|
||||
"github.com/docker/docker/image/v1"
|
||||
"github.com/docker/docker/layer"
|
||||
"github.com/docker/docker/pkg/archive"
|
||||
"github.com/docker/docker/pkg/chrootarchive"
|
||||
"github.com/docker/docker/pkg/symlink"
|
||||
)
|
||||
|
||||
func (l *tarexporter) Load(inTar io.ReadCloser, outStream io.Writer) error {
|
||||
tmpDir, err := ioutil.TempDir("", "docker-import-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
if err := chrootarchive.Untar(inTar, tmpDir, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
// read manifest, if no file then load in legacy mode
|
||||
manifestPath, err := safePath(tmpDir, manifestFileName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
manifestFile, err := os.Open(manifestPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return l.legacyLoad(tmpDir, outStream)
|
||||
}
|
||||
return manifestFile.Close()
|
||||
}
|
||||
defer manifestFile.Close()
|
||||
|
||||
var manifest []manifestItem
|
||||
if err := json.NewDecoder(manifestFile).Decode(&manifest); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, m := range manifest {
|
||||
configPath, err := safePath(tmpDir, m.Config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
config, err := ioutil.ReadFile(configPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
img, err := image.NewFromJSON(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var rootFS image.RootFS
|
||||
rootFS = *img.RootFS
|
||||
rootFS.DiffIDs = nil
|
||||
|
||||
if expected, actual := len(m.Layers), len(img.RootFS.DiffIDs); expected != actual {
|
||||
return fmt.Errorf("invalid manifest, layers length mismatch: expected %q, got %q", expected, actual)
|
||||
}
|
||||
|
||||
for i, diffID := range img.RootFS.DiffIDs {
|
||||
layerPath, err := safePath(tmpDir, m.Layers[i])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newLayer, err := l.loadLayer(layerPath, rootFS)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer layer.ReleaseAndLog(l.ls, newLayer)
|
||||
if expected, actual := diffID, newLayer.DiffID(); expected != actual {
|
||||
return fmt.Errorf("invalid diffID for layer %d: expected %q, got %q", i, expected, actual)
|
||||
}
|
||||
rootFS.Append(diffID)
|
||||
}
|
||||
|
||||
imgID, err := l.is.Create(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, repoTag := range m.RepoTags {
|
||||
named, err := reference.ParseNamed(repoTag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ref, ok := named.(reference.NamedTagged)
|
||||
if !ok {
|
||||
return fmt.Errorf("invalid tag %q", repoTag)
|
||||
}
|
||||
l.setLoadedTag(ref, imgID, outStream)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *tarexporter) loadLayer(filename string, rootFS image.RootFS) (layer.Layer, error) {
|
||||
rawTar, err := os.Open(filename)
|
||||
if err != nil {
|
||||
logrus.Debugf("Error reading embedded tar: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
inflatedLayerData, err := archive.DecompressStream(rawTar)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer rawTar.Close()
|
||||
defer inflatedLayerData.Close()
|
||||
|
||||
return l.ls.Register(inflatedLayerData, rootFS.ChainID())
|
||||
}
|
||||
|
||||
func (l *tarexporter) setLoadedTag(ref reference.NamedTagged, imgID image.ID, outStream io.Writer) error {
|
||||
if prevID, err := l.ts.Get(ref); err == nil && prevID != imgID {
|
||||
fmt.Fprintf(outStream, "The image %s already exists, renaming the old one with ID %s to empty string\n", ref.String(), string(prevID)) // todo: this message is wrong in case of multiple tags
|
||||
}
|
||||
|
||||
if err := l.ts.Add(ref, imgID, true); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *tarexporter) legacyLoad(tmpDir string, outStream io.Writer) error {
|
||||
legacyLoadedMap := make(map[string]image.ID)
|
||||
|
||||
dirs, err := ioutil.ReadDir(tmpDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// every dir represents an image
|
||||
for _, d := range dirs {
|
||||
if d.IsDir() {
|
||||
if err := l.legacyLoadImage(d.Name(), tmpDir, legacyLoadedMap); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// load tags from repositories file
|
||||
repositoriesPath, err := safePath(tmpDir, legacyRepositoriesFileName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
repositoriesFile, err := os.Open(repositoriesPath)
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return repositoriesFile.Close()
|
||||
}
|
||||
defer repositoriesFile.Close()
|
||||
|
||||
repositories := make(map[string]map[string]string)
|
||||
if err := json.NewDecoder(repositoriesFile).Decode(&repositories); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for name, tagMap := range repositories {
|
||||
for tag, oldID := range tagMap {
|
||||
imgID, ok := legacyLoadedMap[oldID]
|
||||
if !ok {
|
||||
return fmt.Errorf("invalid target ID: %v", oldID)
|
||||
}
|
||||
named, err := reference.WithName(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ref, err := reference.WithTag(named, tag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
l.setLoadedTag(ref, imgID, outStream)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *tarexporter) legacyLoadImage(oldID, sourceDir string, loadedMap map[string]image.ID) error {
|
||||
if _, loaded := loadedMap[oldID]; loaded {
|
||||
return nil
|
||||
}
|
||||
configPath, err := safePath(sourceDir, filepath.Join(oldID, legacyConfigFileName))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
imageJSON, err := ioutil.ReadFile(configPath)
|
||||
if err != nil {
|
||||
logrus.Debugf("Error reading json: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
var img struct{ Parent string }
|
||||
if err := json.Unmarshal(imageJSON, &img); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var parentID image.ID
|
||||
if img.Parent != "" {
|
||||
for {
|
||||
var loaded bool
|
||||
if parentID, loaded = loadedMap[img.Parent]; !loaded {
|
||||
if err := l.legacyLoadImage(img.Parent, sourceDir, loadedMap); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// todo: try to connect with migrate code
|
||||
rootFS := image.NewRootFS()
|
||||
var history []image.History
|
||||
|
||||
if parentID != "" {
|
||||
parentImg, err := l.is.Get(parentID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rootFS = parentImg.RootFS
|
||||
history = parentImg.History
|
||||
}
|
||||
|
||||
layerPath, err := safePath(sourceDir, filepath.Join(oldID, legacyLayerFileName))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newLayer, err := l.loadLayer(layerPath, *rootFS)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rootFS.Append(newLayer.DiffID())
|
||||
|
||||
h, err := v1.HistoryFromConfig(imageJSON, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
history = append(history, h)
|
||||
|
||||
config, err := v1.MakeConfigFromV1Config(imageJSON, rootFS, history)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
imgID, err := l.is.Create(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
metadata, err := l.ls.Release(newLayer)
|
||||
layer.LogReleaseMetadata(metadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if parentID != "" {
|
||||
if err := l.is.SetParent(imgID, parentID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
loadedMap[oldID] = imgID
|
||||
return nil
|
||||
}
|
||||
|
||||
func safePath(base, path string) (string, error) {
|
||||
return symlink.FollowSymlinkInScope(filepath.Join(base, path), base)
|
||||
}
|
||||
303
image/tarexport/save.go
Normal file
303
image/tarexport/save.go
Normal file
|
|
@ -0,0 +1,303 @@
|
|||
package tarexport
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/docker/distribution/digest"
|
||||
"github.com/docker/distribution/reference"
|
||||
"github.com/docker/docker/image"
|
||||
"github.com/docker/docker/image/v1"
|
||||
"github.com/docker/docker/layer"
|
||||
"github.com/docker/docker/pkg/archive"
|
||||
"github.com/docker/docker/registry"
|
||||
"github.com/docker/docker/tag"
|
||||
)
|
||||
|
||||
type imageDescriptor struct {
|
||||
refs []reference.NamedTagged
|
||||
layers []string
|
||||
}
|
||||
|
||||
type saveSession struct {
|
||||
*tarexporter
|
||||
outDir string
|
||||
images map[image.ID]*imageDescriptor
|
||||
savedLayers map[string]struct{}
|
||||
}
|
||||
|
||||
func (l *tarexporter) Save(names []string, outStream io.Writer) error {
|
||||
images, err := l.parseNames(names)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return (&saveSession{tarexporter: l, images: images}).save(outStream)
|
||||
}
|
||||
|
||||
func (l *tarexporter) parseNames(names []string) (map[image.ID]*imageDescriptor, error) {
|
||||
imgDescr := make(map[image.ID]*imageDescriptor)
|
||||
|
||||
addAssoc := func(id image.ID, ref reference.Named) {
|
||||
if _, ok := imgDescr[id]; !ok {
|
||||
imgDescr[id] = &imageDescriptor{}
|
||||
}
|
||||
|
||||
if ref != nil {
|
||||
var tagged reference.NamedTagged
|
||||
if _, ok := ref.(reference.Digested); ok {
|
||||
return
|
||||
}
|
||||
var ok bool
|
||||
if tagged, ok = ref.(reference.NamedTagged); !ok {
|
||||
var err error
|
||||
if tagged, err = reference.WithTag(ref, tag.DefaultTag); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
for _, t := range imgDescr[id].refs {
|
||||
if tagged.String() == t.String() {
|
||||
return
|
||||
}
|
||||
}
|
||||
imgDescr[id].refs = append(imgDescr[id].refs, tagged)
|
||||
}
|
||||
}
|
||||
|
||||
for _, name := range names {
|
||||
ref, err := reference.ParseNamed(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ref = registry.NormalizeLocalReference(ref)
|
||||
if ref.Name() == string(digest.Canonical) {
|
||||
imgID, err := l.is.Search(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
addAssoc(imgID, nil)
|
||||
continue
|
||||
}
|
||||
if _, ok := ref.(reference.Digested); !ok {
|
||||
if _, ok := ref.(reference.NamedTagged); !ok {
|
||||
assocs := l.ts.ReferencesByName(ref)
|
||||
for _, assoc := range assocs {
|
||||
addAssoc(assoc.ImageID, assoc.Ref)
|
||||
}
|
||||
if len(assocs) == 0 {
|
||||
imgID, err := l.is.Search(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
addAssoc(imgID, nil)
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
var imgID image.ID
|
||||
if imgID, err = l.ts.Get(ref); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
addAssoc(imgID, ref)
|
||||
|
||||
}
|
||||
return imgDescr, nil
|
||||
}
|
||||
|
||||
func (s *saveSession) save(outStream io.Writer) error {
|
||||
s.savedLayers = make(map[string]struct{})
|
||||
|
||||
// get image json
|
||||
tempDir, err := ioutil.TempDir("", "docker-export-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
s.outDir = tempDir
|
||||
reposLegacy := make(map[string]map[string]string)
|
||||
|
||||
var manifest []manifestItem
|
||||
|
||||
for id, imageDescr := range s.images {
|
||||
if err = s.saveImage(id); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var repoTags []string
|
||||
var layers []string
|
||||
|
||||
for _, ref := range imageDescr.refs {
|
||||
if _, ok := reposLegacy[ref.Name()]; !ok {
|
||||
reposLegacy[ref.Name()] = make(map[string]string)
|
||||
}
|
||||
reposLegacy[ref.Name()][ref.Tag()] = imageDescr.layers[len(imageDescr.layers)-1]
|
||||
repoTags = append(repoTags, ref.String())
|
||||
}
|
||||
|
||||
for _, l := range imageDescr.layers {
|
||||
layers = append(layers, filepath.Join(l, legacyLayerFileName))
|
||||
}
|
||||
|
||||
manifest = append(manifest, manifestItem{
|
||||
Config: digest.Digest(id).Hex() + ".json",
|
||||
RepoTags: repoTags,
|
||||
Layers: layers,
|
||||
})
|
||||
}
|
||||
|
||||
if len(reposLegacy) > 0 {
|
||||
reposFile := filepath.Join(tempDir, legacyRepositoriesFileName)
|
||||
f, err := os.OpenFile(reposFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if err := json.NewEncoder(f).Encode(reposLegacy); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Chtimes(reposFile, time.Unix(0, 0), time.Unix(0, 0)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
manifestFileName := filepath.Join(tempDir, manifestFileName)
|
||||
f, err := os.OpenFile(manifestFileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if err := json.NewEncoder(f).Encode(manifest); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Chtimes(manifestFileName, time.Unix(0, 0), time.Unix(0, 0)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fs, err := archive.Tar(tempDir, archive.Uncompressed)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer fs.Close()
|
||||
|
||||
if _, err := io.Copy(outStream, fs); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *saveSession) saveImage(id image.ID) error {
|
||||
img, err := s.is.Get(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(img.RootFS.DiffIDs) == 0 {
|
||||
return fmt.Errorf("empty export - not implemented")
|
||||
}
|
||||
|
||||
var parent digest.Digest
|
||||
var layers []string
|
||||
for i := range img.RootFS.DiffIDs {
|
||||
v1Img := image.V1Image{}
|
||||
if i == len(img.RootFS.DiffIDs)-1 {
|
||||
v1Img = img.V1Image
|
||||
}
|
||||
rootFS := *img.RootFS
|
||||
rootFS.DiffIDs = rootFS.DiffIDs[:i+1]
|
||||
v1ID, err := v1.CreateID(v1Img, rootFS.ChainID(), parent)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
v1Img.ID = v1ID.Hex()
|
||||
if parent != "" {
|
||||
v1Img.Parent = parent.Hex()
|
||||
}
|
||||
|
||||
if err := s.saveLayer(rootFS.ChainID(), v1Img, img.Created); err != nil {
|
||||
return err
|
||||
}
|
||||
layers = append(layers, v1Img.ID)
|
||||
parent = v1ID
|
||||
}
|
||||
|
||||
configFile := filepath.Join(s.outDir, digest.Digest(id).Hex()+".json")
|
||||
if err := ioutil.WriteFile(configFile, img.RawJSON(), 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Chtimes(configFile, img.Created, img.Created); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.images[id].layers = layers
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *saveSession) saveLayer(id layer.ChainID, legacyImg image.V1Image, createdTime time.Time) error {
|
||||
if _, exists := s.savedLayers[legacyImg.ID]; exists {
|
||||
return nil
|
||||
}
|
||||
|
||||
outDir := filepath.Join(s.outDir, legacyImg.ID)
|
||||
if err := os.Mkdir(outDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// todo: why is this version file here?
|
||||
if err := ioutil.WriteFile(filepath.Join(outDir, legacyVersionFileName), []byte("1.0"), 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
imageConfig, err := json.Marshal(legacyImg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := ioutil.WriteFile(filepath.Join(outDir, legacyConfigFileName), imageConfig, 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// serialize filesystem
|
||||
tarFile, err := os.Create(filepath.Join(outDir, legacyLayerFileName))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tarFile.Close()
|
||||
|
||||
l, err := s.ls.Get(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer layer.ReleaseAndLog(s.ls, l)
|
||||
|
||||
arch, err := l.TarStream()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(tarFile, arch); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, fname := range []string{"", legacyVersionFileName, legacyConfigFileName, legacyLayerFileName} {
|
||||
// todo: maybe save layer created timestamp?
|
||||
if err := os.Chtimes(filepath.Join(outDir, fname), createdTime, createdTime); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
s.savedLayers[legacyImg.ID] = struct{}{}
|
||||
return nil
|
||||
}
|
||||
36
image/tarexport/tarexport.go
Normal file
36
image/tarexport/tarexport.go
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
package tarexport
|
||||
|
||||
import (
|
||||
"github.com/docker/docker/image"
|
||||
"github.com/docker/docker/layer"
|
||||
"github.com/docker/docker/tag"
|
||||
)
|
||||
|
||||
const (
|
||||
manifestFileName = "manifest.json"
|
||||
legacyLayerFileName = "layer.tar"
|
||||
legacyConfigFileName = "json"
|
||||
legacyVersionFileName = "VERSION"
|
||||
legacyRepositoriesFileName = "repositories"
|
||||
)
|
||||
|
||||
type manifestItem struct {
|
||||
Config string
|
||||
RepoTags []string
|
||||
Layers []string
|
||||
}
|
||||
|
||||
type tarexporter struct {
|
||||
is image.Store
|
||||
ls layer.Store
|
||||
ts tag.Store
|
||||
}
|
||||
|
||||
// NewTarExporter returns new ImageExporter for tar packages
|
||||
func NewTarExporter(is image.Store, ls layer.Store, ts tag.Store) image.Exporter {
|
||||
return &tarexporter{
|
||||
is: is,
|
||||
ls: ls,
|
||||
ts: ts,
|
||||
}
|
||||
}
|
||||
148
image/v1/imagev1.go
Normal file
148
image/v1/imagev1.go
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
package v1
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"github.com/docker/distribution/digest"
|
||||
"github.com/docker/docker/image"
|
||||
"github.com/docker/docker/layer"
|
||||
"github.com/docker/docker/pkg/version"
|
||||
)
|
||||
|
||||
var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`)
|
||||
|
||||
// noFallbackMinVersion is the minimum version for which v1compatibility
|
||||
// information will not be marshaled through the Image struct to remove
|
||||
// blank fields.
|
||||
var noFallbackMinVersion = version.Version("1.8.3")
|
||||
|
||||
// HistoryFromConfig creates a History struct from v1 configuration JSON
|
||||
func HistoryFromConfig(imageJSON []byte, emptyLayer bool) (image.History, error) {
|
||||
h := image.History{}
|
||||
var v1Image image.V1Image
|
||||
if err := json.Unmarshal(imageJSON, &v1Image); err != nil {
|
||||
return h, err
|
||||
}
|
||||
|
||||
return image.History{
|
||||
Author: v1Image.Author,
|
||||
Created: v1Image.Created,
|
||||
CreatedBy: strings.Join(v1Image.ContainerConfig.Cmd.Slice(), " "),
|
||||
Comment: v1Image.Comment,
|
||||
EmptyLayer: emptyLayer,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CreateID creates an ID from v1 image, layerID and parent ID.
|
||||
// Used for backwards compatibility with old clients.
|
||||
func CreateID(v1Image image.V1Image, layerID layer.ChainID, parent digest.Digest) (digest.Digest, error) {
|
||||
v1Image.ID = ""
|
||||
v1JSON, err := json.Marshal(v1Image)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var config map[string]*json.RawMessage
|
||||
if err := json.Unmarshal(v1JSON, &config); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// FIXME: note that this is slightly incompatible with RootFS logic
|
||||
config["layer_id"] = rawJSON(layerID)
|
||||
if parent != "" {
|
||||
config["parent"] = rawJSON(parent)
|
||||
}
|
||||
|
||||
configJSON, err := json.Marshal(config)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
logrus.Debugf("CreateV1ID %s", configJSON)
|
||||
|
||||
return digest.FromBytes(configJSON)
|
||||
}
|
||||
|
||||
// MakeConfigFromV1Config creates an image config from the legacy V1 config format.
|
||||
func MakeConfigFromV1Config(imageJSON []byte, rootfs *image.RootFS, history []image.History) ([]byte, error) {
|
||||
var dver struct {
|
||||
DockerVersion string `json:"docker_version"`
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(imageJSON, &dver); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
useFallback := version.Version(dver.DockerVersion).LessThan(noFallbackMinVersion)
|
||||
|
||||
if useFallback {
|
||||
var v1Image image.V1Image
|
||||
err := json.Unmarshal(imageJSON, &v1Image)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
imageJSON, err = json.Marshal(v1Image)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
var c map[string]*json.RawMessage
|
||||
if err := json.Unmarshal(imageJSON, &c); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
delete(c, "id")
|
||||
delete(c, "parent")
|
||||
delete(c, "Size") // Size is calculated from data on disk and is inconsitent
|
||||
delete(c, "parent_id")
|
||||
delete(c, "layer_id")
|
||||
delete(c, "throwaway")
|
||||
|
||||
c["rootfs"] = rawJSON(rootfs)
|
||||
c["history"] = rawJSON(history)
|
||||
|
||||
return json.Marshal(c)
|
||||
}
|
||||
|
||||
// MakeV1ConfigFromConfig creates an legacy V1 image config from an Image struct
|
||||
func MakeV1ConfigFromConfig(img *image.Image, v1ID, parentV1ID string, throwaway bool) ([]byte, error) {
|
||||
// Top-level v1compatibility string should be a modified version of the
|
||||
// image config.
|
||||
var configAsMap map[string]*json.RawMessage
|
||||
if err := json.Unmarshal(img.RawJSON(), &configAsMap); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Delete fields that didn't exist in old manifest
|
||||
delete(configAsMap, "rootfs")
|
||||
delete(configAsMap, "history")
|
||||
configAsMap["id"] = rawJSON(v1ID)
|
||||
if parentV1ID != "" {
|
||||
configAsMap["parent"] = rawJSON(parentV1ID)
|
||||
}
|
||||
if throwaway {
|
||||
configAsMap["throwaway"] = rawJSON(true)
|
||||
}
|
||||
|
||||
return json.Marshal(configAsMap)
|
||||
}
|
||||
|
||||
func rawJSON(value interface{}) *json.RawMessage {
|
||||
jsonval, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return (*json.RawMessage)(&jsonval)
|
||||
}
|
||||
|
||||
// ValidateID checks whether an ID string is a valid image ID.
|
||||
func ValidateID(id string) error {
|
||||
if ok := validHex.MatchString(id); !ok {
|
||||
return fmt.Errorf("image ID '%s' is invalid ", id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Loading…
Reference in a new issue