Add layer store

Layer store manages read-only and read-write layers on a union file system.
Read only layers are always referenced by content addresses.
Read-write layer identifiers are handled by the caller but upon registering
its difference, the committed read-only layer will be referenced by content
hash.

Signed-off-by: Derek McGowan <derek@mcgstyle.net> (github: dmcgowan)
Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

layer/empty.go

@@ -0,0 +1,47 @@
+package layer
+
+import (
+	"archive/tar"
+	"bytes"
+	"io"
+)
+
+// DigestSHA256EmptyTar is the canonical sha256 digest of empty tar file -
+// (1024 NULL bytes)
+const DigestSHA256EmptyTar = DiffID("sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef")
+
+type emptyLayer struct{}
+
+// EmptyLayer is a layer that corresponds to empty tar.
+var EmptyLayer = &emptyLayer{}
+
+func (el *emptyLayer) TarStream() (io.Reader, error) {
+	buf := new(bytes.Buffer)
+	tarWriter := tar.NewWriter(buf)
+	tarWriter.Close()
+	return buf, nil
+}
+
+func (el *emptyLayer) ChainID() ChainID {
+	return ChainID(DigestSHA256EmptyTar)
+}
+
+func (el *emptyLayer) DiffID() DiffID {
+	return DigestSHA256EmptyTar
+}
+
+func (el *emptyLayer) Parent() Layer {
+	return nil
+}
+
+func (el *emptyLayer) Size() (size int64, err error) {
+	return 0, nil
+}
+
+func (el *emptyLayer) DiffSize() (size int64, err error) {
+	return 0, nil
+}
+
+func (el *emptyLayer) Metadata() (map[string]string, error) {
+	return make(map[string]string), nil
+}

layer/empty_test.go

@@ -0,0 +1,46 @@
+package layer
+
+import (
+	"io"
+	"testing"
+
+	"github.com/docker/distribution/digest"
+)
+
+func TestEmptyLayer(t *testing.T) {
+	if EmptyLayer.ChainID() != ChainID(DigestSHA256EmptyTar) {
+		t.Fatal("wrong ID for empty layer")
+	}
+
+	if EmptyLayer.DiffID() != DigestSHA256EmptyTar {
+		t.Fatal("wrong DiffID for empty layer")
+	}
+
+	if EmptyLayer.Parent() != nil {
+		t.Fatal("expected no parent for empty layer")
+	}
+
+	if size, err := EmptyLayer.Size(); err != nil || size != 0 {
+		t.Fatal("expected zero size for empty layer")
+	}
+
+	if diffSize, err := EmptyLayer.DiffSize(); err != nil || diffSize != 0 {
+		t.Fatal("expected zero diffsize for empty layer")
+	}
+
+	tarStream, err := EmptyLayer.TarStream()
+	if err != nil {
+		t.Fatalf("error streaming tar for empty layer: %v", err)
+	}
+
+	digester := digest.Canonical.New()
+	_, err = io.Copy(digester.Hash(), tarStream)
+
+	if err != nil {
+		t.Fatalf("error hashing empty tar layer: %v", err)
+	}
+
+	if digester.Digest() != digest.Digest(DigestSHA256EmptyTar) {
+		t.Fatal("empty layer tar stream hashes to wrong value")
+	}
+}

layer/filestore.go

@@ -0,0 +1,318 @@
+package layer
+
+import (
+	"compress/gzip"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/docker/pkg/ioutils"
+)
+
+var (
+	stringIDRegexp      = regexp.MustCompile(`^[a-f0-9]{64}(-init)?$`)
+	supportedAlgorithms = []digest.Algorithm{
+		digest.SHA256,
+		// digest.SHA384, // Currently not used
+		// digest.SHA512, // Currently not used
+	}
+)
+
+type fileMetadataStore struct {
+	root string
+}
+
+type fileMetadataTransaction struct {
+	store *fileMetadataStore
+	root  string
+}
+
+// NewFSMetadataStore returns an instance of a metadata store
+// which is backed by files on disk using the provided root
+// as the root of metadata files.
+func NewFSMetadataStore(root string) (MetadataStore, error) {
+	if err := os.MkdirAll(root, 0700); err != nil {
+		return nil, err
+	}
+	return &fileMetadataStore{
+		root: root,
+	}, nil
+}
+
+func (fms *fileMetadataStore) getLayerDirectory(layer ChainID) string {
+	dgst := digest.Digest(layer)
+	return filepath.Join(fms.root, string(dgst.Algorithm()), dgst.Hex())
+}
+
+func (fms *fileMetadataStore) getLayerFilename(layer ChainID, filename string) string {
+	return filepath.Join(fms.getLayerDirectory(layer), filename)
+}
+
+func (fms *fileMetadataStore) getMountDirectory(mount string) string {
+	return filepath.Join(fms.root, "mounts", mount)
+}
+
+func (fms *fileMetadataStore) getMountFilename(mount, filename string) string {
+	return filepath.Join(fms.getMountDirectory(mount), filename)
+}
+
+func (fms *fileMetadataStore) StartTransaction() (MetadataTransaction, error) {
+	tmpDir := filepath.Join(fms.root, "tmp")
+	if err := os.MkdirAll(tmpDir, 0755); err != nil {
+		return nil, err
+	}
+
+	td, err := ioutil.TempDir(tmpDir, "layer-")
+	if err != nil {
+		return nil, err
+	}
+	// Create a new tempdir
+	return &fileMetadataTransaction{
+		store: fms,
+		root:  td,
+	}, nil
+}
+
+func (fm *fileMetadataTransaction) SetSize(size int64) error {
+	content := fmt.Sprintf("%d", size)
+	return ioutil.WriteFile(filepath.Join(fm.root, "size"), []byte(content), 0644)
+}
+
+func (fm *fileMetadataTransaction) SetParent(parent ChainID) error {
+	return ioutil.WriteFile(filepath.Join(fm.root, "parent"), []byte(digest.Digest(parent).String()), 0644)
+}
+
+func (fm *fileMetadataTransaction) SetDiffID(diff DiffID) error {
+	return ioutil.WriteFile(filepath.Join(fm.root, "diff"), []byte(digest.Digest(diff).String()), 0644)
+}
+
+func (fm *fileMetadataTransaction) SetCacheID(cacheID string) error {
+	return ioutil.WriteFile(filepath.Join(fm.root, "cache-id"), []byte(cacheID), 0644)
+}
+
+func (fm *fileMetadataTransaction) TarSplitWriter() (io.WriteCloser, error) {
+	f, err := os.OpenFile(filepath.Join(fm.root, "tar-split.json.gz"), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return nil, err
+	}
+
+	fz := gzip.NewWriter(f)
+
+	return ioutils.NewWriteCloserWrapper(fz, func() error {
+		fz.Close()
+		return f.Close()
+	}), nil
+}
+
+func (fm *fileMetadataTransaction) Commit(layer ChainID) error {
+	finalDir := fm.store.getLayerDirectory(layer)
+	if err := os.MkdirAll(filepath.Dir(finalDir), 0755); err != nil {
+		return err
+	}
+	return os.Rename(fm.root, finalDir)
+}
+
+func (fm *fileMetadataTransaction) Cancel() error {
+	return os.RemoveAll(fm.root)
+}
+
+func (fm *fileMetadataTransaction) String() string {
+	return fm.root
+}
+
+func (fms *fileMetadataStore) GetSize(layer ChainID) (int64, error) {
+	content, err := ioutil.ReadFile(fms.getLayerFilename(layer, "size"))
+	if err != nil {
+		return 0, err
+	}
+
+	size, err := strconv.ParseInt(string(content), 10, 64)
+	if err != nil {
+		return 0, err
+	}
+
+	return size, nil
+}
+
+func (fms *fileMetadataStore) GetParent(layer ChainID) (ChainID, error) {
+	content, err := ioutil.ReadFile(fms.getLayerFilename(layer, "parent"))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return "", nil
+		}
+		return "", err
+	}
+
+	dgst, err := digest.ParseDigest(string(content))
+	if err != nil {
+		return "", err
+	}
+
+	return ChainID(dgst), nil
+}
+
+func (fms *fileMetadataStore) GetDiffID(layer ChainID) (DiffID, error) {
+	content, err := ioutil.ReadFile(fms.getLayerFilename(layer, "diff"))
+	if err != nil {
+		return "", err
+	}
+
+	dgst, err := digest.ParseDigest(string(content))
+	if err != nil {
+		return "", err
+	}
+
+	return DiffID(dgst), nil
+}
+
+func (fms *fileMetadataStore) GetCacheID(layer ChainID) (string, error) {
+	content, err := ioutil.ReadFile(fms.getLayerFilename(layer, "cache-id"))
+	if err != nil {
+		return "", err
+	}
+
+	if !stringIDRegexp.MatchString(string(content)) {
+		return "", errors.New("invalid cache id value")
+	}
+
+	return string(content), nil
+}
+
+func (fms *fileMetadataStore) TarSplitReader(layer ChainID) (io.ReadCloser, error) {
+	fz, err := os.Open(fms.getLayerFilename(layer, "tar-split.json.gz"))
+	if err != nil {
+		return nil, err
+	}
+	f, err := gzip.NewReader(fz)
+	if err != nil {
+		return nil, err
+	}
+
+	return ioutils.NewReadCloserWrapper(f, func() error {
+		f.Close()
+		return fz.Close()
+	}), nil
+}
+
+func (fms *fileMetadataStore) SetMountID(mount string, mountID string) error {
+	if err := os.MkdirAll(fms.getMountDirectory(mount), 0755); err != nil {
+		return err
+	}
+	return ioutil.WriteFile(fms.getMountFilename(mount, "mount-id"), []byte(mountID), 0644)
+}
+
+func (fms *fileMetadataStore) SetInitID(mount string, init string) error {
+	if err := os.MkdirAll(fms.getMountDirectory(mount), 0755); err != nil {
+		return err
+	}
+	return ioutil.WriteFile(fms.getMountFilename(mount, "init-id"), []byte(init), 0644)
+}
+
+func (fms *fileMetadataStore) SetMountParent(mount string, parent ChainID) error {
+	if err := os.MkdirAll(fms.getMountDirectory(mount), 0755); err != nil {
+		return err
+	}
+	return ioutil.WriteFile(fms.getMountFilename(mount, "parent"), []byte(digest.Digest(parent).String()), 0644)
+}
+
+func (fms *fileMetadataStore) GetMountID(mount string) (string, error) {
+	content, err := ioutil.ReadFile(fms.getMountFilename(mount, "mount-id"))
+	if err != nil {
+		return "", err
+	}
+
+	if !stringIDRegexp.MatchString(string(content)) {
+		return "", errors.New("invalid mount id value")
+	}
+
+	return string(content), nil
+}
+
+func (fms *fileMetadataStore) GetInitID(mount string) (string, error) {
+	content, err := ioutil.ReadFile(fms.getMountFilename(mount, "init-id"))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return "", nil
+		}
+		return "", err
+	}
+
+	if !stringIDRegexp.MatchString(string(content)) {
+		return "", errors.New("invalid init id value")
+	}
+
+	return string(content), nil
+}
+
+func (fms *fileMetadataStore) GetMountParent(mount string) (ChainID, error) {
+	content, err := ioutil.ReadFile(fms.getMountFilename(mount, "parent"))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return "", nil
+		}
+		return "", err
+	}
+
+	dgst, err := digest.ParseDigest(string(content))
+	if err != nil {
+		return "", err
+	}
+
+	return ChainID(dgst), nil
+}
+
+func (fms *fileMetadataStore) List() ([]ChainID, []string, error) {
+	var ids []ChainID
+	for _, algorithm := range supportedAlgorithms {
+		fileInfos, err := ioutil.ReadDir(filepath.Join(fms.root, string(algorithm)))
+		if err != nil {
+			if os.IsNotExist(err) {
+				continue
+			}
+			return nil, nil, err
+		}
+
+		for _, fi := range fileInfos {
+			if fi.IsDir() && fi.Name() != "mounts" {
+				dgst := digest.NewDigestFromHex(string(algorithm), fi.Name())
+				if err := dgst.Validate(); err != nil {
+					logrus.Debugf("Ignoring invalid digest %s:%s", algorithm, fi.Name())
+				} else {
+					ids = append(ids, ChainID(dgst))
+				}
+			}
+		}
+	}
+
+	fileInfos, err := ioutil.ReadDir(filepath.Join(fms.root, "mounts"))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return ids, []string{}, nil
+		}
+		return nil, nil, err
+	}
+
+	var mounts []string
+	for _, fi := range fileInfos {
+		if fi.IsDir() {
+			mounts = append(mounts, fi.Name())
+		}
+	}
+
+	return ids, mounts, nil
+}
+
+func (fms *fileMetadataStore) Remove(layer ChainID) error {
+	return os.RemoveAll(fms.getLayerDirectory(layer))
+}
+
+func (fms *fileMetadataStore) RemoveMount(mount string) error {
+	return os.RemoveAll(fms.getMountDirectory(mount))
+}

layer/filestore_test.go

@@ -0,0 +1,119 @@
+package layer
+
+import (
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"testing"
+
+	"github.com/docker/distribution/digest"
+)
+
+func randomLayerID(seed int64) ChainID {
+	r := rand.New(rand.NewSource(seed))
+	dgst, err := digest.FromBytes([]byte(fmt.Sprintf("%d", r.Int63())))
+	if err != nil {
+		panic(err)
+	}
+
+	return ChainID(dgst)
+}
+
+func newFileMetadataStore(t *testing.T) (*fileMetadataStore, string, func()) {
+	td, err := ioutil.TempDir("", "layers-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	fms, err := NewFSMetadataStore(td)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return fms.(*fileMetadataStore), td, func() {
+		if err := os.RemoveAll(td); err != nil {
+			t.Logf("Failed to cleanup %q: %s", td, err)
+		}
+	}
+}
+
+func assertNotDirectoryError(t *testing.T, err error) {
+	perr, ok := err.(*os.PathError)
+	if !ok {
+		t.Fatalf("Unexpected error %#v, expected path error", err)
+	}
+
+	if perr.Err != syscall.ENOTDIR {
+		t.Fatalf("Unexpected error %s, expected %s", perr.Err, syscall.ENOTDIR)
+	}
+}
+
+func assertPermissionError(t *testing.T, err error) {
+	perr, ok := err.(*os.PathError)
+	if !ok {
+		t.Fatalf("Unexpected error %#v, expected path error", err)
+	}
+
+	if perr.Err != syscall.EACCES {
+		t.Fatalf("Unexpected error %s, expected %s", perr.Err, syscall.EACCES)
+	}
+}
+
+func TestCommitFailure(t *testing.T) {
+	fms, td, cleanup := newFileMetadataStore(t)
+	defer cleanup()
+
+	if err := ioutil.WriteFile(filepath.Join(td, "sha256"), []byte("was here first!"), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	tx, err := fms.StartTransaction()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := tx.SetSize(0); err != nil {
+		t.Fatal(err)
+	}
+
+	err = tx.Commit(randomLayerID(5))
+	if err == nil {
+		t.Fatalf("Expected error committing with invalid layer parent directory")
+	}
+	assertNotDirectoryError(t, err)
+}
+
+func TestStartTransactionFailure(t *testing.T) {
+	fms, td, cleanup := newFileMetadataStore(t)
+	defer cleanup()
+
+	if err := ioutil.WriteFile(filepath.Join(td, "tmp"), []byte("was here first!"), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err := fms.StartTransaction()
+	if err == nil {
+		t.Fatalf("Expected error starting transaction with invalid layer parent directory")
+	}
+	assertNotDirectoryError(t, err)
+
+	if err := os.Remove(filepath.Join(td, "tmp")); err != nil {
+		t.Fatal(err)
+	}
+
+	tx, err := fms.StartTransaction()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if expected := filepath.Join(td, "tmp"); strings.HasPrefix(expected, tx.String()) {
+		t.Fatalf("Unexpected transaction string %q, expected prefix %q", tx.String(), expected)
+	}
+
+	if err := tx.Cancel(); err != nil {
+		t.Fatal(err)
+	}
+}

layer/layer.go

@@ -0,0 +1,238 @@
+// Package layer is package for managing read only
+// and read-write mounts on the union file system
+// driver. Read-only mounts are refenced using a
+// content hash and are protected from mutation in
+// the exposed interface. The tar format is used
+// to create read only layers and export both
+// read only and writable layers. The exported
+// tar data for a read only layer should match
+// the tar used to create the layer.
+package layer
+
+import (
+	"errors"
+	"io"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/docker/pkg/archive"
+)
+
+var (
+	// ErrLayerDoesNotExist is used when an operation is
+	// attempted on a layer which does not exist.
+	ErrLayerDoesNotExist = errors.New("layer does not exist")
+
+	// ErrLayerNotRetained is used when a release is
+	// attempted on a layer which is not retained.
+	ErrLayerNotRetained = errors.New("layer not retained")
+
+	// ErrMountDoesNotExist is used when an operation is
+	// attempted on a mount layer which does not exist.
+	ErrMountDoesNotExist = errors.New("mount does not exist")
+
+	// ErrActiveMount is used when an operation on a
+	// mount is attempted but the layer is still
+	// mounted and the operation cannot be performed.
+	ErrActiveMount = errors.New("mount still active")
+
+	// ErrNotMounted is used when requesting an active
+	// mount but the layer is not mounted.
+	ErrNotMounted = errors.New("not mounted")
+
+	// ErrMaxDepthExceeded is used when a layer is attempted
+	// to be created which would result in a layer depth
+	// greater than the 125 max.
+	ErrMaxDepthExceeded = errors.New("max depth exceeded")
+)
+
+// ChainID is the content-addressable ID of a layer.
+type ChainID digest.Digest
+
+// String returns a string rendition of a layer ID
+func (id ChainID) String() string {
+	return string(id)
+}
+
+// DiffID is the hash of an individual layer tar.
+type DiffID digest.Digest
+
+// String returns a string rendition of a layer DiffID
+func (diffID DiffID) String() string {
+	return string(diffID)
+}
+
+// TarStreamer represents an object which may
+// have its contents exported as a tar stream.
+type TarStreamer interface {
+	// TarStream returns a tar archive stream
+	// for the contents of a layer.
+	TarStream() (io.Reader, error)
+}
+
+// Layer represents a read only layer
+type Layer interface {
+	TarStreamer
+
+	// ChainID returns the content hash of the entire layer chain. The hash
+	// chain is made up of DiffID of top layer and all of its parents.
+	ChainID() ChainID
+
+	// DiffID returns the content hash of the layer
+	// tar stream used to create this layer.
+	DiffID() DiffID
+
+	// Parent returns the next layer in the layer chain.
+	Parent() Layer
+
+	// Size returns the size of the entire layer chain. The size
+	// is calculated from the total size of all files in the layers.
+	Size() (int64, error)
+
+	// DiffSize returns the size difference of the top layer
+	// from parent layer.
+	DiffSize() (int64, error)
+
+	// Metadata returns the low level storage metadata associated
+	// with layer.
+	Metadata() (map[string]string, error)
+}
+
+// RWLayer represents a layer which is
+// read and writable
+type RWLayer interface {
+	TarStreamer
+
+	// Path returns the filesystem path to the writable
+	// layer.
+	Path() (string, error)
+
+	// Parent returns the layer which the writable
+	// layer was created from.
+	Parent() Layer
+
+	// Size represents the size of the writable layer
+	// as calculated by the total size of the files
+	// changed in the mutable layer.
+	Size() (int64, error)
+}
+
+// Metadata holds information about a
+// read only layer
+type Metadata struct {
+	// ChainID is the content hash of the layer
+	ChainID ChainID
+
+	// DiffID is the hash of the tar data used to
+	// create the layer
+	DiffID DiffID
+
+	// Size is the size of the layer and all parents
+	Size int64
+
+	// DiffSize is the size of the top layer
+	DiffSize int64
+}
+
+// MountInit is a function to initialize a
+// writable mount. Changes made here will
+// not be included in the Tar stream of the
+// RWLayer.
+type MountInit func(root string) error
+
+// Store represents a backend for managing both
+// read-only and read-write layers.
+type Store interface {
+	Register(io.Reader, ChainID) (Layer, error)
+	Get(ChainID) (Layer, error)
+	Release(Layer) ([]Metadata, error)
+
+	Mount(id string, parent ChainID, label string, init MountInit) (RWLayer, error)
+	Unmount(id string) error
+	DeleteMount(id string) ([]Metadata, error)
+	Changes(id string) ([]archive.Change, error)
+}
+
+// MetadataTransaction represents functions for setting layer metadata
+// with a single transaction.
+type MetadataTransaction interface {
+	SetSize(int64) error
+	SetParent(parent ChainID) error
+	SetDiffID(DiffID) error
+	SetCacheID(string) error
+	TarSplitWriter() (io.WriteCloser, error)
+
+	Commit(ChainID) error
+	Cancel() error
+	String() string
+}
+
+// MetadataStore represents a backend for persisting
+// metadata about layers and providing the metadata
+// for restoring a Store.
+type MetadataStore interface {
+	// StartTransaction starts an update for new metadata
+	// which will be used to represent an ID on commit.
+	StartTransaction() (MetadataTransaction, error)
+
+	GetSize(ChainID) (int64, error)
+	GetParent(ChainID) (ChainID, error)
+	GetDiffID(ChainID) (DiffID, error)
+	GetCacheID(ChainID) (string, error)
+	TarSplitReader(ChainID) (io.ReadCloser, error)
+
+	SetMountID(string, string) error
+	SetInitID(string, string) error
+	SetMountParent(string, ChainID) error
+
+	GetMountID(string) (string, error)
+	GetInitID(string) (string, error)
+	GetMountParent(string) (ChainID, error)
+
+	// List returns the full list of referened
+	// read-only and read-write layers
+	List() ([]ChainID, []string, error)
+
+	Remove(ChainID) error
+	RemoveMount(string) error
+}
+
+// CreateChainID returns ID for a layerDigest slice
+func CreateChainID(dgsts []DiffID) ChainID {
+	return createChainIDFromParent("", dgsts...)
+}
+
+func createChainIDFromParent(parent ChainID, dgsts ...DiffID) ChainID {
+	if len(dgsts) == 0 {
+		return parent
+	}
+	if parent == "" {
+		return createChainIDFromParent(ChainID(dgsts[0]), dgsts[1:]...)
+	}
+	// H = "H(n-1) SHA256(n)"
+	dgst, err := digest.FromBytes([]byte(string(parent) + " " + string(dgsts[0])))
+	if err != nil {
+		// Digest calculation is not expected to throw an error,
+		// any error at this point is a program error
+		panic(err)
+	}
+	return createChainIDFromParent(ChainID(dgst), dgsts[1:]...)
+}
+
+// ReleaseAndLog releases the provided layer from the given layer
+// store, logging any error and release metadata
+func ReleaseAndLog(ls Store, l Layer) {
+	metadata, err := ls.Release(l)
+	if err != nil {
+		logrus.Errorf("Error releasing layer %s: %v", l.ChainID(), err)
+	}
+	LogReleaseMetadata(metadata)
+}
+
+// LogReleaseMetadata logs a metadata array, use this to
+// ensure consistent logging for release metadata
+func LogReleaseMetadata(metadatas []Metadata) {
+	for _, metadata := range metadatas {
+		logrus.Infof("Layer %s cleaned up", metadata.ChainID)
+	}
+}

layer/layer_store.go

@@ -0,0 +1,649 @@
+package layer
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"runtime"
+	"sync"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/docker/daemon/graphdriver"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/stringid"
+	"github.com/vbatts/tar-split/tar/asm"
+	"github.com/vbatts/tar-split/tar/storage"
+)
+
+// maxLayerDepth represents the maximum number of
+// layers which can be chained together. 125 was
+// chosen to account for the 127 max in some
+// graphdrivers plus the 2 additional layers
+// used to create a rwlayer.
+const maxLayerDepth = 125
+
+type layerStore struct {
+	store  MetadataStore
+	driver graphdriver.Driver
+
+	layerMap map[ChainID]*roLayer
+	layerL   sync.Mutex
+
+	mounts map[string]*mountedLayer
+	mountL sync.Mutex
+}
+
+// NewStore creates a new Store instance using
+// the provided metadata store and graph driver.
+// The metadata store will be used to restore
+// the Store.
+func NewStore(store MetadataStore, driver graphdriver.Driver) (Store, error) {
+	ls := &layerStore{
+		store:    store,
+		driver:   driver,
+		layerMap: map[ChainID]*roLayer{},
+		mounts:   map[string]*mountedLayer{},
+	}
+
+	ids, mounts, err := store.List()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, id := range ids {
+		l, err := ls.loadLayer(id)
+		if err != nil {
+			logrus.Debugf("Failed to load layer %s: %s", id, err)
+		}
+		if l.parent != nil {
+			l.parent.referenceCount++
+		}
+	}
+
+	for _, mount := range mounts {
+		if err := ls.loadMount(mount); err != nil {
+			logrus.Debugf("Failed to load mount %s: %s", mount, err)
+		}
+	}
+
+	return ls, nil
+}
+
+func (ls *layerStore) loadLayer(layer ChainID) (*roLayer, error) {
+	cl, ok := ls.layerMap[layer]
+	if ok {
+		return cl, nil
+	}
+
+	diff, err := ls.store.GetDiffID(layer)
+	if err != nil {
+		return nil, err
+	}
+
+	size, err := ls.store.GetSize(layer)
+	if err != nil {
+		return nil, err
+	}
+
+	cacheID, err := ls.store.GetCacheID(layer)
+	if err != nil {
+		return nil, err
+	}
+
+	parent, err := ls.store.GetParent(layer)
+	if err != nil {
+		return nil, err
+	}
+
+	cl = &roLayer{
+		chainID:    layer,
+		diffID:     diff,
+		size:       size,
+		cacheID:    cacheID,
+		layerStore: ls,
+		references: map[Layer]struct{}{},
+	}
+
+	if parent != "" {
+		p, err := ls.loadLayer(parent)
+		if err != nil {
+			return nil, err
+		}
+		cl.parent = p
+	}
+
+	ls.layerMap[cl.chainID] = cl
+
+	return cl, nil
+}
+
+func (ls *layerStore) loadMount(mount string) error {
+	if _, ok := ls.mounts[mount]; ok {
+		return nil
+	}
+
+	mountID, err := ls.store.GetMountID(mount)
+	if err != nil {
+		return err
+	}
+
+	initID, err := ls.store.GetInitID(mount)
+	if err != nil {
+		return err
+	}
+
+	parent, err := ls.store.GetMountParent(mount)
+	if err != nil {
+		return err
+	}
+
+	ml := &mountedLayer{
+		name:       mount,
+		mountID:    mountID,
+		initID:     initID,
+		layerStore: ls,
+	}
+
+	if parent != "" {
+		p, err := ls.loadLayer(parent)
+		if err != nil {
+			return err
+		}
+		ml.parent = p
+
+		p.referenceCount++
+	}
+
+	ls.mounts[ml.name] = ml
+
+	return nil
+}
+
+func (ls *layerStore) applyTar(tx MetadataTransaction, ts io.Reader, parent string, layer *roLayer) error {
+	digester := digest.Canonical.New()
+	tr := io.TeeReader(ts, digester.Hash())
+
+	tsw, err := tx.TarSplitWriter()
+	if err != nil {
+		return err
+	}
+	metaPacker := storage.NewJSONPacker(tsw)
+	defer tsw.Close()
+
+	// we're passing nil here for the file putter, because the ApplyDiff will
+	// handle the extraction of the archive
+	rdr, err := asm.NewInputTarStream(tr, metaPacker, nil)
+	if err != nil {
+		return err
+	}
+
+	applySize, err := ls.driver.ApplyDiff(layer.cacheID, parent, archive.Reader(rdr))
+	if err != nil {
+		return err
+	}
+
+	// Discard trailing data but ensure metadata is picked up to reconstruct stream
+	io.Copy(ioutil.Discard, rdr) // ignore error as reader may be closed
+
+	layer.size = applySize
+	layer.diffID = DiffID(digester.Digest())
+
+	logrus.Debugf("Applied tar %s to %s, size: %d", layer.diffID, layer.cacheID, applySize)
+
+	return nil
+}
+
+func (ls *layerStore) Register(ts io.Reader, parent ChainID) (Layer, error) {
+	// err is used to hold the error which will always trigger
+	// cleanup of creates sources but may not be an error returned
+	// to the caller (already exists).
+	var err error
+	var pid string
+	var p *roLayer
+	if string(parent) != "" {
+		p = ls.get(parent)
+		if p == nil {
+			return nil, ErrLayerDoesNotExist
+		}
+		pid = p.cacheID
+		// Release parent chain if error
+		defer func() {
+			if err != nil {
+				ls.layerL.Lock()
+				ls.releaseLayer(p)
+				ls.layerL.Unlock()
+			}
+		}()
+		if p.depth() >= maxLayerDepth {
+			err = ErrMaxDepthExceeded
+			return nil, err
+		}
+	}
+
+	// Create new roLayer
+	layer := &roLayer{
+		parent:         p,
+		cacheID:        stringid.GenerateRandomID(),
+		referenceCount: 1,
+		layerStore:     ls,
+		references:     map[Layer]struct{}{},
+	}
+
+	if err = ls.driver.Create(layer.cacheID, pid, ""); err != nil {
+		return nil, err
+	}
+
+	tx, err := ls.store.StartTransaction()
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		if err != nil {
+			logrus.Debugf("Cleaning up layer %s: %v", layer.cacheID, err)
+			if err := ls.driver.Remove(layer.cacheID); err != nil {
+				logrus.Errorf("Error cleaning up cache layer %s: %v", layer.cacheID, err)
+			}
+			if err := tx.Cancel(); err != nil {
+				logrus.Errorf("Error canceling metadata transaction %q: %s", tx.String(), err)
+			}
+		}
+	}()
+
+	if err = ls.applyTar(tx, ts, pid, layer); err != nil {
+		return nil, err
+	}
+
+	if layer.parent == nil {
+		layer.chainID = ChainID(layer.diffID)
+	} else {
+		layer.chainID = createChainIDFromParent(layer.parent.chainID, layer.diffID)
+	}
+
+	if err = storeLayer(tx, layer); err != nil {
+		return nil, err
+	}
+
+	ls.layerL.Lock()
+	defer ls.layerL.Unlock()
+
+	if existingLayer := ls.getAndRetainLayer(layer.chainID); existingLayer != nil {
+		// Set error for cleanup, but do not return the error
+		err = errors.New("layer already exists")
+		return existingLayer.getReference(), nil
+	}
+
+	if err = tx.Commit(layer.chainID); err != nil {
+		return nil, err
+	}
+
+	ls.layerMap[layer.chainID] = layer
+
+	return layer.getReference(), nil
+}
+
+func (ls *layerStore) get(l ChainID) *roLayer {
+	ls.layerL.Lock()
+	defer ls.layerL.Unlock()
+
+	layer, ok := ls.layerMap[l]
+	if !ok {
+		return nil
+	}
+
+	layer.referenceCount++
+
+	return layer
+}
+
+func (ls *layerStore) Get(l ChainID) (Layer, error) {
+	layer := ls.get(l)
+	if layer == nil {
+		return nil, ErrLayerDoesNotExist
+	}
+
+	return layer.getReference(), nil
+}
+
+func (ls *layerStore) deleteLayer(layer *roLayer, metadata *Metadata) error {
+	err := ls.driver.Remove(layer.cacheID)
+	if err != nil {
+		return err
+	}
+
+	err = ls.store.Remove(layer.chainID)
+	if err != nil {
+		return err
+	}
+	metadata.DiffID = layer.diffID
+	metadata.ChainID = layer.chainID
+	metadata.Size, err = layer.Size()
+	if err != nil {
+		return err
+	}
+	metadata.DiffSize = layer.size
+
+	return nil
+}
+
+func (ls *layerStore) releaseLayers(l *roLayer, removed *[]Metadata, depth int) error {
+	if l.referenceCount == 0 {
+		panic("layer not retained")
+	}
+	l.referenceCount--
+	if l.referenceCount != 0 {
+		return nil
+	}
+
+	if len(*removed) == 0 && depth > 0 {
+		panic("cannot remove layer with child")
+	}
+	if l.hasReferences() {
+		panic("cannot delete referenced layer")
+	}
+	var metadata Metadata
+	if err := ls.deleteLayer(l, &metadata); err != nil {
+		return err
+	}
+
+	delete(ls.layerMap, l.chainID)
+	*removed = append(*removed, metadata)
+
+	if l.parent != nil {
+		if err := ls.releaseLayers(l.parent, removed, depth+1); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (ls *layerStore) releaseLayer(layer *roLayer) ([]Metadata, error) {
+	removed := []Metadata{}
+	err := ls.releaseLayers(layer, &removed, 0)
+	return removed, err
+}
+
+func (ls *layerStore) Release(l Layer) ([]Metadata, error) {
+	ls.layerL.Lock()
+	defer ls.layerL.Unlock()
+	layer, ok := ls.layerMap[l.ChainID()]
+	if !ok {
+		return []Metadata{}, nil
+	}
+	if !layer.hasReference(l) {
+		return nil, ErrLayerNotRetained
+	}
+
+	layer.deleteReference(l)
+
+	return ls.releaseLayer(layer)
+}
+
+func (ls *layerStore) mount(m *mountedLayer, mountLabel string) error {
+	dir, err := ls.driver.Get(m.mountID, mountLabel)
+	if err != nil {
+		return err
+	}
+	m.path = dir
+	m.activityCount++
+
+	return nil
+}
+
+func (ls *layerStore) saveMount(mount *mountedLayer) error {
+	if err := ls.store.SetMountID(mount.name, mount.mountID); err != nil {
+		return err
+	}
+
+	if mount.initID != "" {
+		if err := ls.store.SetInitID(mount.name, mount.initID); err != nil {
+			return err
+		}
+	}
+
+	if mount.parent != nil {
+		if err := ls.store.SetMountParent(mount.name, mount.parent.chainID); err != nil {
+			return err
+		}
+	}
+
+	ls.mounts[mount.name] = mount
+
+	return nil
+}
+
+func (ls *layerStore) getAndRetainLayer(layer ChainID) *roLayer {
+	l, ok := ls.layerMap[layer]
+	if !ok {
+		return nil
+	}
+
+	l.referenceCount++
+
+	return l
+}
+
+func (ls *layerStore) initMount(graphID, parent, mountLabel string, initFunc MountInit) (string, error) {
+	// Use "<graph-id>-init" to maintain compatibility with graph drivers
+	// which are expecting this layer with this special name. If all
+	// graph drivers can be updated to not rely on knowin about this layer
+	// then the initID should be randomly generated.
+	initID := fmt.Sprintf("%s-init", graphID)
+
+	if err := ls.driver.Create(initID, parent, mountLabel); err != nil {
+
+	}
+	p, err := ls.driver.Get(initID, "")
+	if err != nil {
+		return "", err
+	}
+
+	if err := initFunc(p); err != nil {
+		ls.driver.Put(initID)
+		return "", err
+	}
+
+	if err := ls.driver.Put(initID); err != nil {
+		return "", err
+	}
+
+	return initID, nil
+}
+
+func (ls *layerStore) Mount(name string, parent ChainID, mountLabel string, initFunc MountInit) (l RWLayer, err error) {
+	ls.mountL.Lock()
+	defer ls.mountL.Unlock()
+	m, ok := ls.mounts[name]
+	if ok {
+		// Check if has path
+		if err := ls.mount(m, mountLabel); err != nil {
+			return nil, err
+		}
+		return m, nil
+	}
+
+	var pid string
+	var p *roLayer
+	if string(parent) != "" {
+		ls.layerL.Lock()
+		p = ls.getAndRetainLayer(parent)
+		ls.layerL.Unlock()
+		if p == nil {
+			return nil, ErrLayerDoesNotExist
+		}
+		pid = p.cacheID
+
+		// Release parent chain if error
+		defer func() {
+			if err != nil {
+				ls.layerL.Lock()
+				ls.releaseLayer(p)
+				ls.layerL.Unlock()
+			}
+		}()
+	}
+
+	mountID := name
+	if runtime.GOOS != "windows" {
+		// windows has issues if container ID doesn't match mount ID
+		mountID = stringid.GenerateRandomID()
+	}
+
+	m = &mountedLayer{
+		name:       name,
+		parent:     p,
+		mountID:    mountID,
+		layerStore: ls,
+	}
+
+	if initFunc != nil {
+		pid, err = ls.initMount(m.mountID, pid, mountLabel, initFunc)
+		if err != nil {
+			return nil, err
+		}
+		m.initID = pid
+	}
+
+	if err = ls.driver.Create(m.mountID, pid, ""); err != nil {
+		return nil, err
+	}
+
+	if err = ls.saveMount(m); err != nil {
+		return nil, err
+	}
+
+	if err = ls.mount(m, mountLabel); err != nil {
+		return nil, err
+	}
+
+	return m, nil
+}
+
+func (ls *layerStore) Unmount(name string) error {
+	ls.mountL.Lock()
+	defer ls.mountL.Unlock()
+
+	m := ls.mounts[name]
+	if m == nil {
+		return ErrMountDoesNotExist
+	}
+
+	m.activityCount--
+
+	if err := ls.driver.Put(m.mountID); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (ls *layerStore) DeleteMount(name string) ([]Metadata, error) {
+	ls.mountL.Lock()
+	defer ls.mountL.Unlock()
+
+	m := ls.mounts[name]
+	if m == nil {
+		return nil, ErrMountDoesNotExist
+	}
+	if m.activityCount > 0 {
+		return nil, ErrActiveMount
+	}
+
+	delete(ls.mounts, name)
+
+	if err := ls.driver.Remove(m.mountID); err != nil {
+		logrus.Errorf("Error removing mounted layer %s: %s", m.name, err)
+		return nil, err
+	}
+
+	if m.initID != "" {
+		if err := ls.driver.Remove(m.initID); err != nil {
+			logrus.Errorf("Error removing init layer %s: %s", m.name, err)
+			return nil, err
+		}
+	}
+
+	if err := ls.store.RemoveMount(m.name); err != nil {
+		logrus.Errorf("Error removing mount metadata: %s: %s", m.name, err)
+		return nil, err
+	}
+
+	ls.layerL.Lock()
+	defer ls.layerL.Unlock()
+	if m.parent != nil {
+		return ls.releaseLayer(m.parent)
+	}
+
+	return []Metadata{}, nil
+}
+
+func (ls *layerStore) Changes(name string) ([]archive.Change, error) {
+	ls.mountL.Lock()
+	m := ls.mounts[name]
+	ls.mountL.Unlock()
+	if m == nil {
+		return nil, ErrMountDoesNotExist
+	}
+	pid := m.initID
+	if pid == "" && m.parent != nil {
+		pid = m.parent.cacheID
+	}
+	return ls.driver.Changes(m.mountID, pid)
+}
+
+func (ls *layerStore) assembleTar(graphID string, metadata io.ReadCloser, size *int64) (io.Reader, error) {
+	type diffPathDriver interface {
+		DiffPath(string) (string, func() error, error)
+	}
+
+	diffDriver, ok := ls.driver.(diffPathDriver)
+	if !ok {
+		diffDriver = &naiveDiffPathDriver{ls.driver}
+	}
+
+	// get our relative path to the container
+	fsPath, releasePath, err := diffDriver.DiffPath(graphID)
+	if err != nil {
+		metadata.Close()
+		return nil, err
+	}
+
+	pR, pW := io.Pipe()
+	// this will need to be in a goroutine, as we are returning the stream of a
+	// tar archive, but can not close the metadata reader early (when this
+	// function returns)...
+	go func() {
+		defer releasePath()
+		defer metadata.Close()
+
+		metaUnpacker := storage.NewJSONUnpacker(metadata)
+		upackerCounter := &unpackSizeCounter{metaUnpacker, size}
+		fileGetter := storage.NewPathFileGetter(fsPath)
+		logrus.Debugf("Assembling tar data for %s from %s", graphID, fsPath)
+		ots := asm.NewOutputTarStream(fileGetter, upackerCounter)
+		defer ots.Close()
+		if _, err := io.Copy(pW, ots); err != nil {
+			pW.CloseWithError(err)
+			return
+		}
+		pW.Close()
+	}()
+	return pR, nil
+}
+
+type naiveDiffPathDriver struct {
+	graphdriver.Driver
+}
+
+func (n *naiveDiffPathDriver) DiffPath(id string) (string, func() error, error) {
+	p, err := n.Driver.Get(id, "")
+	if err != nil {
+		return "", nil, err
+	}
+	return p, func() error {
+		return n.Driver.Put(id)
+	}, nil
+}

layer/layer_test.go

@@ -0,0 +1,725 @@
+package layer
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/docker/distribution/digest"
+	"github.com/docker/docker/daemon/graphdriver"
+	"github.com/docker/docker/daemon/graphdriver/vfs"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/idtools"
+	"github.com/docker/docker/pkg/stringid"
+)
+
+func init() {
+	graphdriver.ApplyUncompressedLayer = archive.UnpackLayer
+	vfs.CopyWithTar = archive.CopyWithTar
+}
+
+func newVFSGraphDriver(td string) (graphdriver.Driver, error) {
+	uidMap := []idtools.IDMap{
+		{
+			ContainerID: 0,
+			HostID:      os.Getuid(),
+			Size:        1,
+		},
+	}
+	gidMap := []idtools.IDMap{
+		{
+			ContainerID: 0,
+			HostID:      os.Getgid(),
+			Size:        1,
+		},
+	}
+
+	return graphdriver.GetDriver("vfs", td, nil, uidMap, gidMap)
+}
+
+func newTestGraphDriver(t *testing.T) (graphdriver.Driver, func()) {
+	td, err := ioutil.TempDir("", "graph-")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	driver, err := newVFSGraphDriver(td)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return driver, func() {
+		os.RemoveAll(td)
+	}
+}
+
+func newTestStore(t *testing.T) (Store, func()) {
+	td, err := ioutil.TempDir("", "layerstore-")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	graph, graphcleanup := newTestGraphDriver(t)
+	fms, err := NewFSMetadataStore(td)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ls, err := NewStore(fms, graph)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return ls, func() {
+		graphcleanup()
+		os.RemoveAll(td)
+	}
+}
+
+type layerInit func(root string) error
+
+func createLayer(ls Store, parent ChainID, layerFunc layerInit) (Layer, error) {
+	containerID := stringid.GenerateRandomID()
+	mount, err := ls.Mount(containerID, parent, "", nil)
+	if err != nil {
+		return nil, err
+	}
+
+	path, err := mount.Path()
+	if err != nil {
+		return nil, err
+	}
+
+	if err := layerFunc(path); err != nil {
+		return nil, err
+	}
+
+	ts, err := mount.TarStream()
+	if err != nil {
+		return nil, err
+	}
+
+	layer, err := ls.Register(ts, parent)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := ls.Unmount(containerID); err != nil {
+		return nil, err
+	}
+
+	if _, err := ls.DeleteMount(containerID); err != nil {
+		return nil, err
+	}
+
+	return layer, nil
+}
+
+type FileApplier interface {
+	ApplyFile(root string) error
+}
+
+type testFile struct {
+	name       string
+	content    []byte
+	permission os.FileMode
+}
+
+func newTestFile(name string, content []byte, perm os.FileMode) FileApplier {
+	return &testFile{
+		name:       name,
+		content:    content,
+		permission: perm,
+	}
+}
+
+func (tf *testFile) ApplyFile(root string) error {
+	fullPath := filepath.Join(root, tf.name)
+	if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil {
+		return err
+	}
+	// Check if already exists
+	if stat, err := os.Stat(fullPath); err == nil && stat.Mode().Perm() != tf.permission {
+		if err := os.Chmod(fullPath, tf.permission); err != nil {
+			return err
+		}
+	}
+	if err := ioutil.WriteFile(fullPath, tf.content, tf.permission); err != nil {
+		return err
+	}
+	return nil
+}
+
+func initWithFiles(files ...FileApplier) layerInit {
+	return func(root string) error {
+		for _, f := range files {
+			if err := f.ApplyFile(root); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+}
+
+func getCachedLayer(l Layer) *roLayer {
+	if rl, ok := l.(*referencedCacheLayer); ok {
+		return rl.roLayer
+	}
+	return l.(*roLayer)
+}
+
+func createMetadata(layers ...Layer) []Metadata {
+	metadata := make([]Metadata, len(layers))
+	for i := range layers {
+		size, err := layers[i].Size()
+		if err != nil {
+			panic(err)
+		}
+
+		metadata[i].ChainID = layers[i].ChainID()
+		metadata[i].DiffID = layers[i].DiffID()
+		metadata[i].Size = size
+		metadata[i].DiffSize = getCachedLayer(layers[i]).size
+	}
+
+	return metadata
+}
+
+func assertMetadata(t *testing.T, metadata, expectedMetadata []Metadata) {
+	if len(metadata) != len(expectedMetadata) {
+		t.Fatalf("Unexpected number of deletes %d, expected %d", len(metadata), len(expectedMetadata))
+	}
+
+	for i := range metadata {
+		if metadata[i] != expectedMetadata[i] {
+			t.Errorf("Unexpected metadata\n\tExpected: %#v\n\tActual: %#v", expectedMetadata[i], metadata[i])
+		}
+	}
+	if t.Failed() {
+		t.FailNow()
+	}
+}
+
+func releaseAndCheckDeleted(t *testing.T, ls Store, layer Layer, removed ...Layer) {
+	layerCount := len(ls.(*layerStore).layerMap)
+	expectedMetadata := createMetadata(removed...)
+	metadata, err := ls.Release(layer)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assertMetadata(t, metadata, expectedMetadata)
+
+	if expected := layerCount - len(removed); len(ls.(*layerStore).layerMap) != expected {
+		t.Fatalf("Unexpected number of layers %d, expected %d", len(ls.(*layerStore).layerMap), expected)
+	}
+}
+
+func cacheID(l Layer) string {
+	return getCachedLayer(l).cacheID
+}
+
+func assertLayerEqual(t *testing.T, l1, l2 Layer) {
+	if l1.ChainID() != l2.ChainID() {
+		t.Fatalf("Mismatched ID: %s vs %s", l1.ChainID(), l2.ChainID())
+	}
+	if l1.DiffID() != l2.DiffID() {
+		t.Fatalf("Mismatched DiffID: %s vs %s", l1.DiffID(), l2.DiffID())
+	}
+
+	size1, err := l1.Size()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	size2, err := l2.Size()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if size1 != size2 {
+		t.Fatalf("Mismatched size: %d vs %d", size1, size2)
+	}
+
+	if cacheID(l1) != cacheID(l2) {
+		t.Fatalf("Mismatched cache id: %s vs %s", cacheID(l1), cacheID(l2))
+	}
+
+	p1 := l1.Parent()
+	p2 := l2.Parent()
+	if p1 != nil && p2 != nil {
+		assertLayerEqual(t, p1, p2)
+	} else if p1 != nil || p2 != nil {
+		t.Fatalf("Mismatched parents: %v vs %v", p1, p2)
+	}
+}
+
+func TestMountAndRegister(t *testing.T) {
+	ls, cleanup := newTestStore(t)
+	defer cleanup()
+
+	li := initWithFiles(newTestFile("testfile.txt", []byte("some test data"), 0644))
+	layer, err := createLayer(ls, "", li)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	size, _ := layer.Size()
+	t.Logf("Layer size: %d", size)
+
+	mount2, err := ls.Mount("new-test-mount", layer.ChainID(), "", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	path2, err := mount2.Path()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	b, err := ioutil.ReadFile(filepath.Join(path2, "testfile.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if expected := "some test data"; string(b) != expected {
+		t.Fatalf("Wrong file data, expected %q, got %q", expected, string(b))
+	}
+
+	if err := ls.Unmount("new-test-mount"); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := ls.DeleteMount("new-test-mount"); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestLayerRelease(t *testing.T) {
+	ls, cleanup := newTestStore(t)
+	defer cleanup()
+
+	layer1, err := createLayer(ls, "", initWithFiles(newTestFile("layer1.txt", []byte("layer 1 file"), 0644)))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	layer2, err := createLayer(ls, layer1.ChainID(), initWithFiles(newTestFile("layer2.txt", []byte("layer 2 file"), 0644)))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := ls.Release(layer1); err != nil {
+		t.Fatal(err)
+	}
+
+	layer3a, err := createLayer(ls, layer2.ChainID(), initWithFiles(newTestFile("layer3.txt", []byte("layer 3a file"), 0644)))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	layer3b, err := createLayer(ls, layer2.ChainID(), initWithFiles(newTestFile("layer3.txt", []byte("layer 3b file"), 0644)))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := ls.Release(layer2); err != nil {
+		t.Fatal(err)
+	}
+
+	t.Logf("Layer1:  %s", layer1.ChainID())
+	t.Logf("Layer2:  %s", layer2.ChainID())
+	t.Logf("Layer3a: %s", layer3a.ChainID())
+	t.Logf("Layer3b: %s", layer3b.ChainID())
+
+	if expected := 4; len(ls.(*layerStore).layerMap) != expected {
+		t.Fatalf("Unexpected number of layers %d, expected %d", len(ls.(*layerStore).layerMap), expected)
+	}
+
+	releaseAndCheckDeleted(t, ls, layer3b, layer3b)
+	releaseAndCheckDeleted(t, ls, layer3a, layer3a, layer2, layer1)
+}
+
+func TestStoreRestore(t *testing.T) {
+	ls, cleanup := newTestStore(t)
+	defer cleanup()
+
+	layer1, err := createLayer(ls, "", initWithFiles(newTestFile("layer1.txt", []byte("layer 1 file"), 0644)))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	layer2, err := createLayer(ls, layer1.ChainID(), initWithFiles(newTestFile("layer2.txt", []byte("layer 2 file"), 0644)))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := ls.Release(layer1); err != nil {
+		t.Fatal(err)
+	}
+
+	layer3, err := createLayer(ls, layer2.ChainID(), initWithFiles(newTestFile("layer3.txt", []byte("layer 3 file"), 0644)))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := ls.Release(layer2); err != nil {
+		t.Fatal(err)
+	}
+
+	m, err := ls.Mount("some-mount_name", layer3.ChainID(), "", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	path, err := m.Path()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := ioutil.WriteFile(filepath.Join(path, "testfile.txt"), []byte("nothing here"), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := ls.Unmount("some-mount_name"); err != nil {
+		t.Fatal(err)
+	}
+
+	ls2, err := NewStore(ls.(*layerStore).store, ls.(*layerStore).driver)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	layer3b, err := ls2.Get(layer3.ChainID())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assertLayerEqual(t, layer3b, layer3)
+
+	// Mount again with same name, should already be loaded
+	m2, err := ls2.Mount("some-mount_name", layer3b.ChainID(), "", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	path2, err := m2.Path()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	b, err := ioutil.ReadFile(filepath.Join(path2, "testfile.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if expected := "nothing here"; string(b) != expected {
+		t.Fatalf("Unexpected content %q, expected %q", string(b), expected)
+	}
+
+	if err := ls2.Unmount("some-mount_name"); err != nil {
+		t.Fatal(err)
+	}
+
+	if metadata, err := ls2.DeleteMount("some-mount_name"); err != nil {
+		t.Fatal(err)
+	} else if len(metadata) != 0 {
+		t.Fatalf("Unexpectedly deleted layers: %#v", metadata)
+	}
+
+	releaseAndCheckDeleted(t, ls2, layer3b, layer3, layer2, layer1)
+}
+
+func TestTarStreamStability(t *testing.T) {
+	ls, cleanup := newTestStore(t)
+	defer cleanup()
+
+	files1 := []FileApplier{
+		newTestFile("/etc/hosts", []byte("mydomain 10.0.0.1"), 0644),
+		newTestFile("/etc/profile", []byte("PATH=/usr/bin"), 0644),
+	}
+	addedFile := newTestFile("/etc/shadow", []byte("root:::::::"), 0644)
+	files2 := []FileApplier{
+		newTestFile("/etc/hosts", []byte("mydomain 10.0.0.2"), 0644),
+		newTestFile("/etc/profile", []byte("PATH=/usr/bin"), 0664),
+		newTestFile("/root/.bashrc", []byte("PATH=/usr/sbin:/usr/bin"), 0644),
+	}
+
+	tar1, err := tarFromFiles(files1...)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tar2, err := tarFromFiles(files2...)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	layer1, err := ls.Register(bytes.NewReader(tar1), "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// hack layer to add file
+	p, err := ls.(*layerStore).driver.Get(layer1.(*referencedCacheLayer).cacheID, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := addedFile.ApplyFile(p); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := ls.(*layerStore).driver.Put(layer1.(*referencedCacheLayer).cacheID); err != nil {
+		t.Fatal(err)
+	}
+
+	layer2, err := ls.Register(bytes.NewReader(tar2), layer1.ChainID())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	id1 := layer1.ChainID()
+	t.Logf("Layer 1: %s", layer1.ChainID())
+	t.Logf("Layer 2: %s", layer2.ChainID())
+
+	if _, err := ls.Release(layer1); err != nil {
+		t.Fatal(err)
+	}
+
+	assertLayerDiff(t, tar2, layer2)
+
+	layer1b, err := ls.Get(id1)
+	if err != nil {
+		t.Logf("Content of layer map: %#v", ls.(*layerStore).layerMap)
+		t.Fatal(err)
+	}
+
+	if _, err := ls.Release(layer2); err != nil {
+		t.Fatal(err)
+	}
+
+	assertLayerDiff(t, tar1, layer1b)
+
+	if _, err := ls.Release(layer1b); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func assertLayerDiff(t *testing.T, expected []byte, layer Layer) {
+	expectedDigest, err := digest.FromBytes(expected)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if digest.Digest(layer.DiffID()) != expectedDigest {
+		t.Fatalf("Mismatched diff id for %s, got %s, expected %s", layer.ChainID(), layer.DiffID(), expected)
+	}
+
+	ts, err := layer.TarStream()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	actual, err := ioutil.ReadAll(ts)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(actual) != len(expected) {
+		logByteDiff(t, actual, expected)
+		t.Fatalf("Mismatched tar stream size for %s, got %d, expected %d", layer.ChainID(), len(actual), len(expected))
+	}
+
+	actualDigest, err := digest.FromBytes(actual)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if actualDigest != expectedDigest {
+		logByteDiff(t, actual, expected)
+		t.Fatalf("Wrong digest of tar stream, got %s, expected %s", actualDigest, expectedDigest)
+	}
+}
+
+const maxByteLog = 4 * 1024
+
+func logByteDiff(t *testing.T, actual, expected []byte) {
+	d1, d2 := byteDiff(actual, expected)
+	if len(d1) == 0 && len(d2) == 0 {
+		return
+	}
+
+	prefix := len(actual) - len(d1)
+	if len(d1) > maxByteLog || len(d2) > maxByteLog {
+		t.Logf("Byte diff after %d matching bytes", prefix)
+	} else {
+		t.Logf("Byte diff after %d matching bytes\nActual bytes after prefix:\n%x\nExpected bytes after prefix:\n%x", prefix, d1, d2)
+	}
+}
+
+// byteDiff returns the differing bytes after the matching prefix
+func byteDiff(b1, b2 []byte) ([]byte, []byte) {
+	i := 0
+	for i < len(b1) && i < len(b2) {
+		if b1[i] != b2[i] {
+			break
+		}
+		i++
+	}
+
+	return b1[i:], b2[i:]
+}
+
+func tarFromFiles(files ...FileApplier) ([]byte, error) {
+	td, err := ioutil.TempDir("", "tar-")
+	if err != nil {
+		return nil, err
+	}
+	defer os.RemoveAll(td)
+
+	for _, f := range files {
+		if err := f.ApplyFile(td); err != nil {
+			return nil, err
+		}
+	}
+
+	r, err := archive.Tar(td, archive.Uncompressed)
+	if err != nil {
+		return nil, err
+	}
+
+	buf := bytes.NewBuffer(nil)
+	if _, err := io.Copy(buf, r); err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
+// assertReferences asserts that all the references are to the same
+// image and represent the full set of references to that image.
+func assertReferences(t *testing.T, references ...Layer) {
+	if len(references) == 0 {
+		return
+	}
+	base := references[0].(*referencedCacheLayer).roLayer
+	seenReferences := map[Layer]struct{}{
+		references[0]: {},
+	}
+	for i := 1; i < len(references); i++ {
+		other := references[i].(*referencedCacheLayer).roLayer
+		if base != other {
+			t.Fatalf("Unexpected referenced cache layer %s, expecting %s", other.ChainID(), base.ChainID())
+		}
+		if _, ok := base.references[references[i]]; !ok {
+			t.Fatalf("Reference not part of reference list: %v", references[i])
+		}
+		if _, ok := seenReferences[references[i]]; ok {
+			t.Fatalf("Duplicated reference %v", references[i])
+		}
+	}
+	if rc := len(base.references); rc != len(references) {
+		t.Fatalf("Unexpected number of references %d, expecting %d", rc, len(references))
+	}
+}
+
+func TestRegisterExistingLayer(t *testing.T) {
+	ls, cleanup := newTestStore(t)
+	defer cleanup()
+
+	baseFiles := []FileApplier{
+		newTestFile("/etc/profile", []byte("# Base configuration"), 0644),
+	}
+
+	layerFiles := []FileApplier{
+		newTestFile("/root/.bashrc", []byte("# Root configuration"), 0644),
+	}
+
+	li := initWithFiles(baseFiles...)
+	layer1, err := createLayer(ls, "", li)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tar1, err := tarFromFiles(layerFiles...)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	layer2a, err := ls.Register(bytes.NewReader(tar1), layer1.ChainID())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	layer2b, err := ls.Register(bytes.NewReader(tar1), layer1.ChainID())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assertReferences(t, layer2a, layer2b)
+}
+
+func graphDiffSize(ls Store, l Layer) (int64, error) {
+	cl := getCachedLayer(l)
+	var parent string
+	if cl.parent != nil {
+		parent = cl.parent.cacheID
+	}
+	return ls.(*layerStore).driver.DiffSize(cl.cacheID, parent)
+}
+
+func TestLayerSize(t *testing.T) {
+	ls, cleanup := newTestStore(t)
+	defer cleanup()
+
+	content1 := []byte("Base contents")
+	content2 := []byte("Added contents")
+
+	layer1, err := createLayer(ls, "", initWithFiles(newTestFile("file1", content1, 0644)))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	layer2, err := createLayer(ls, layer1.ChainID(), initWithFiles(newTestFile("file2", content2, 0644)))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	layer1DiffSize, err := graphDiffSize(ls, layer1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if int(layer1DiffSize) != len(content1) {
+		t.Fatalf("Unexpected diff size %d, expected %d", layer1DiffSize, len(content1))
+	}
+
+	layer1Size, err := layer1.Size()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if expected := len(content1); int(layer1Size) != expected {
+		t.Fatalf("Unexpected size %d, expected %d", layer1Size, expected)
+	}
+
+	layer2DiffSize, err := graphDiffSize(ls, layer2)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if int(layer2DiffSize) != len(content2) {
+		t.Fatalf("Unexpected diff size %d, expected %d", layer2DiffSize, len(content2))
+	}
+
+	layer2Size, err := layer2.Size()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if expected := len(content1) + len(content2); int(layer2Size) != expected {
+		t.Fatalf("Unexpected size %d, expected %d", layer2Size, expected)
+	}
+}

layer/layer_windows.go

@@ -0,0 +1,109 @@
+package layer
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution/digest"
+)
+
+// GetLayerPath returns the path to a layer
+func GetLayerPath(s Store, layer ChainID) (string, error) {
+	ls, ok := s.(*layerStore)
+	if !ok {
+		return "", errors.New("unsupported layer store")
+	}
+	ls.layerL.Lock()
+	defer ls.layerL.Unlock()
+
+	rl, ok := ls.layerMap[layer]
+	if !ok {
+		return "", ErrLayerDoesNotExist
+	}
+
+	path, err := ls.driver.Get(rl.cacheID, "")
+	if err != nil {
+		return "", err
+	}
+
+	if err := ls.driver.Put(rl.cacheID); err != nil {
+		return "", err
+	}
+
+	return path, nil
+}
+
+// RWLayerMetadata returns the graph metadata for the provided
+// mount name.
+func RWLayerMetadata(s Store, name string) (map[string]string, error) {
+	ls, ok := s.(*layerStore)
+	if !ok {
+		return nil, errors.New("unsupported layer store")
+	}
+	ls.mountL.Lock()
+	defer ls.mountL.Unlock()
+
+	ml, ok := ls.mounts[name]
+	if !ok {
+		return nil, errors.New("mount does not exist")
+	}
+
+	return ls.driver.GetMetadata(ml.mountID)
+}
+
+func (ls *layerStore) RegisterDiffID(graphID string, size int64) (Layer, error) {
+	var err error // this is used for cleanup in existingLayer case
+	diffID, err := digest.FromBytes([]byte(graphID))
+	if err != nil {
+		return nil, err
+	}
+
+	// Create new roLayer
+	layer := &roLayer{
+		cacheID:        graphID,
+		diffID:         DiffID(diffID),
+		referenceCount: 1,
+		layerStore:     ls,
+		references:     map[Layer]struct{}{},
+		size:           size,
+	}
+
+	tx, err := ls.store.StartTransaction()
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err != nil {
+			if err := tx.Cancel(); err != nil {
+				logrus.Errorf("Error canceling metadata transaction %q: %s", tx.String(), err)
+			}
+		}
+	}()
+
+	layer.chainID = createChainIDFromParent("", layer.diffID)
+
+	if !ls.driver.Exists(layer.cacheID) {
+		return nil, fmt.Errorf("layer %q is unknown to driver", layer.cacheID)
+	}
+	if err = storeLayer(tx, layer); err != nil {
+		return nil, err
+	}
+
+	ls.layerL.Lock()
+	defer ls.layerL.Unlock()
+
+	if existingLayer := ls.getAndRetainLayer(layer.chainID); existingLayer != nil {
+		// Set error for cleanup, but do not return
+		err = errors.New("layer already exists")
+		return existingLayer.getReference(), nil
+	}
+
+	if err = tx.Commit(layer.chainID); err != nil {
+		return nil, err
+	}
+
+	ls.layerMap[layer.chainID] = layer
+
+	return layer.getReference(), nil
+}

layer/migration.go

@@ -0,0 +1,251 @@
+package layer
+
+import (
+	"compress/gzip"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/docker/pkg/ioutils"
+	"github.com/vbatts/tar-split/tar/asm"
+	"github.com/vbatts/tar-split/tar/storage"
+)
+
+func (ls *layerStore) MountByGraphID(name string, graphID string, parent ChainID) (l RWLayer, err error) {
+	ls.mountL.Lock()
+	defer ls.mountL.Unlock()
+	m, ok := ls.mounts[name]
+	if ok {
+		if m.parent.chainID != parent {
+			return nil, errors.New("name conflict, mismatched parent")
+		}
+		if m.mountID != graphID {
+			return nil, errors.New("mount already exists")
+		}
+
+		return m, nil
+	}
+
+	if !ls.driver.Exists(graphID) {
+		return nil, errors.New("graph ID does not exist")
+	}
+
+	var p *roLayer
+	if string(parent) != "" {
+		ls.layerL.Lock()
+		p = ls.getAndRetainLayer(parent)
+		ls.layerL.Unlock()
+		if p == nil {
+			return nil, ErrLayerDoesNotExist
+		}
+
+		// Release parent chain if error
+		defer func() {
+			if err != nil {
+				ls.layerL.Lock()
+				ls.releaseLayer(p)
+				ls.layerL.Unlock()
+			}
+		}()
+	}
+
+	// TODO: Ensure graphID has correct parent
+
+	m = &mountedLayer{
+		name:       name,
+		parent:     p,
+		mountID:    graphID,
+		layerStore: ls,
+	}
+
+	// Check for existing init layer
+	initID := fmt.Sprintf("%s-init", graphID)
+	if ls.driver.Exists(initID) {
+		m.initID = initID
+	}
+
+	if err = ls.saveMount(m); err != nil {
+		return nil, err
+	}
+
+	// TODO: provide a mount label
+	if err = ls.mount(m, ""); err != nil {
+		return nil, err
+	}
+
+	return m, nil
+}
+
+func (ls *layerStore) migrateLayer(tx MetadataTransaction, tarDataFile string, layer *roLayer) error {
+	var ar io.Reader
+	var tdf *os.File
+	var err error
+	if tarDataFile != "" {
+		tdf, err = os.Open(tarDataFile)
+		if err != nil {
+			if !os.IsNotExist(err) {
+				return err
+			}
+			tdf = nil
+		}
+		defer tdf.Close()
+	}
+	if tdf != nil {
+		tsw, err := tx.TarSplitWriter()
+		if err != nil {
+			return err
+		}
+
+		defer tsw.Close()
+
+		uncompressed, err := gzip.NewReader(tdf)
+		if err != nil {
+			return err
+		}
+		defer uncompressed.Close()
+
+		tr := io.TeeReader(uncompressed, tsw)
+		trc := ioutils.NewReadCloserWrapper(tr, uncompressed.Close)
+
+		ar, err = ls.assembleTar(layer.cacheID, trc, &layer.size)
+		if err != nil {
+			return err
+		}
+
+	} else {
+		var graphParent string
+		if layer.parent != nil {
+			graphParent = layer.parent.cacheID
+		}
+		archiver, err := ls.driver.Diff(layer.cacheID, graphParent)
+		if err != nil {
+			return err
+		}
+		defer archiver.Close()
+
+		tsw, err := tx.TarSplitWriter()
+		if err != nil {
+			return err
+		}
+		metaPacker := storage.NewJSONPacker(tsw)
+		packerCounter := &packSizeCounter{metaPacker, &layer.size}
+		defer tsw.Close()
+
+		ar, err = asm.NewInputTarStream(archiver, packerCounter, nil)
+		if err != nil {
+			return err
+		}
+	}
+
+	digester := digest.Canonical.New()
+	_, err = io.Copy(digester.Hash(), ar)
+	if err != nil {
+		return err
+	}
+
+	layer.diffID = DiffID(digester.Digest())
+
+	return nil
+}
+
+func (ls *layerStore) RegisterByGraphID(graphID string, parent ChainID, tarDataFile string) (Layer, error) {
+	// err is used to hold the error which will always trigger
+	// cleanup of creates sources but may not be an error returned
+	// to the caller (already exists).
+	var err error
+	var p *roLayer
+	if string(parent) != "" {
+		p = ls.get(parent)
+		if p == nil {
+			return nil, ErrLayerDoesNotExist
+		}
+
+		// Release parent chain if error
+		defer func() {
+			if err != nil {
+				ls.layerL.Lock()
+				ls.releaseLayer(p)
+				ls.layerL.Unlock()
+			}
+		}()
+	}
+
+	// Create new roLayer
+	layer := &roLayer{
+		parent:         p,
+		cacheID:        graphID,
+		referenceCount: 1,
+		layerStore:     ls,
+		references:     map[Layer]struct{}{},
+	}
+
+	tx, err := ls.store.StartTransaction()
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		if err != nil {
+			logrus.Debugf("Cleaning up transaction after failed migration for %s: %v", graphID, err)
+			if err := tx.Cancel(); err != nil {
+				logrus.Errorf("Error canceling metadata transaction %q: %s", tx.String(), err)
+			}
+		}
+	}()
+
+	if err = ls.migrateLayer(tx, tarDataFile, layer); err != nil {
+		return nil, err
+	}
+
+	layer.chainID = createChainIDFromParent(parent, layer.diffID)
+
+	if err = storeLayer(tx, layer); err != nil {
+		return nil, err
+	}
+
+	ls.layerL.Lock()
+	defer ls.layerL.Unlock()
+
+	if existingLayer := ls.getAndRetainLayer(layer.chainID); existingLayer != nil {
+		// Set error for cleanup, but do not return
+		err = errors.New("layer already exists")
+		return existingLayer.getReference(), nil
+	}
+
+	if err = tx.Commit(layer.chainID); err != nil {
+		return nil, err
+	}
+
+	ls.layerMap[layer.chainID] = layer
+
+	return layer.getReference(), nil
+}
+
+type unpackSizeCounter struct {
+	unpacker storage.Unpacker
+	size     *int64
+}
+
+func (u *unpackSizeCounter) Next() (*storage.Entry, error) {
+	e, err := u.unpacker.Next()
+	if err == nil && u.size != nil {
+		*u.size += e.Size
+	}
+	return e, err
+}
+
+type packSizeCounter struct {
+	packer storage.Packer
+	size   *int64
+}
+
+func (p *packSizeCounter) AddEntry(e storage.Entry) (int, error) {
+	n, err := p.packer.AddEntry(e)
+	if err == nil && p.size != nil {
+		*p.size += e.Size
+	}
+	return n, err
+}

layer/migration_test.go

@@ -0,0 +1,385 @@
+package layer
+
+import (
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/docker/docker/daemon/graphdriver"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/stringid"
+	"github.com/vbatts/tar-split/tar/asm"
+	"github.com/vbatts/tar-split/tar/storage"
+)
+
+func writeTarSplitFile(name string, tarContent []byte) error {
+	f, err := os.OpenFile(name, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	fz := gzip.NewWriter(f)
+
+	metaPacker := storage.NewJSONPacker(fz)
+	defer fz.Close()
+
+	rdr, err := asm.NewInputTarStream(bytes.NewReader(tarContent), metaPacker, nil)
+	if err != nil {
+		return err
+	}
+
+	if _, err := io.Copy(ioutil.Discard, rdr); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func TestLayerMigration(t *testing.T) {
+	td, err := ioutil.TempDir("", "migration-test-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(td)
+
+	layer1Files := []FileApplier{
+		newTestFile("/root/.bashrc", []byte("# Boring configuration"), 0644),
+		newTestFile("/etc/profile", []byte("# Base configuration"), 0644),
+	}
+
+	layer2Files := []FileApplier{
+		newTestFile("/root/.bashrc", []byte("# Updated configuration"), 0644),
+	}
+
+	tar1, err := tarFromFiles(layer1Files...)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tar2, err := tarFromFiles(layer2Files...)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	graph, err := newVFSGraphDriver(filepath.Join(td, "graphdriver-"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	graphID1 := stringid.GenerateRandomID()
+	if err := graph.Create(graphID1, "", ""); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := graph.ApplyDiff(graphID1, "", archive.Reader(bytes.NewReader(tar1))); err != nil {
+		t.Fatal(err)
+	}
+
+	tf1 := filepath.Join(td, "tar1.json.gz")
+	if err := writeTarSplitFile(tf1, tar1); err != nil {
+		t.Fatal(err)
+	}
+
+	fms, err := NewFSMetadataStore(filepath.Join(td, "layers"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	ls, err := NewStore(fms, graph)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	layer1a, err := ls.(*layerStore).RegisterByGraphID(graphID1, "", tf1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	layer1b, err := ls.Register(bytes.NewReader(tar1), "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assertReferences(t, layer1a, layer1b)
+
+	// Attempt register, should be same
+	layer2a, err := ls.Register(bytes.NewReader(tar2), layer1a.ChainID())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	graphID2 := stringid.GenerateRandomID()
+	if err := graph.Create(graphID2, graphID1, ""); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := graph.ApplyDiff(graphID2, graphID1, archive.Reader(bytes.NewReader(tar2))); err != nil {
+		t.Fatal(err)
+	}
+
+	tf2 := filepath.Join(td, "tar2.json.gz")
+	if err := writeTarSplitFile(tf2, tar2); err != nil {
+		t.Fatal(err)
+	}
+
+	layer2b, err := ls.(*layerStore).RegisterByGraphID(graphID2, layer1a.ChainID(), tf2)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assertReferences(t, layer2a, layer2b)
+
+	if metadata, err := ls.Release(layer2a); err != nil {
+		t.Fatal(err)
+	} else if len(metadata) > 0 {
+		t.Fatalf("Unexpected layer removal after first release: %#v", metadata)
+	}
+
+	metadata, err := ls.Release(layer2b)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assertMetadata(t, metadata, createMetadata(layer2a))
+}
+
+func tarFromFilesInGraph(graph graphdriver.Driver, graphID, parentID string, files ...FileApplier) ([]byte, error) {
+	t, err := tarFromFiles(files...)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := graph.Create(graphID, parentID, ""); err != nil {
+		return nil, err
+	}
+	if _, err := graph.ApplyDiff(graphID, parentID, archive.Reader(bytes.NewReader(t))); err != nil {
+		return nil, err
+	}
+
+	ar, err := graph.Diff(graphID, parentID)
+	if err != nil {
+		return nil, err
+	}
+	defer ar.Close()
+
+	return ioutil.ReadAll(ar)
+}
+
+func TestLayerMigrationNoTarsplit(t *testing.T) {
+	td, err := ioutil.TempDir("", "migration-test-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(td)
+
+	layer1Files := []FileApplier{
+		newTestFile("/root/.bashrc", []byte("# Boring configuration"), 0644),
+		newTestFile("/etc/profile", []byte("# Base configuration"), 0644),
+	}
+
+	layer2Files := []FileApplier{
+		newTestFile("/root/.bashrc", []byte("# Updated configuration"), 0644),
+	}
+
+	graph, err := newVFSGraphDriver(filepath.Join(td, "graphdriver-"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	graphID1 := stringid.GenerateRandomID()
+	graphID2 := stringid.GenerateRandomID()
+
+	tar1, err := tarFromFilesInGraph(graph, graphID1, "", layer1Files...)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tar2, err := tarFromFilesInGraph(graph, graphID2, graphID1, layer2Files...)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fms, err := NewFSMetadataStore(filepath.Join(td, "layers"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	ls, err := NewStore(fms, graph)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	layer1a, err := ls.(*layerStore).RegisterByGraphID(graphID1, "", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	layer1b, err := ls.Register(bytes.NewReader(tar1), "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assertReferences(t, layer1a, layer1b)
+
+	// Attempt register, should be same
+	layer2a, err := ls.Register(bytes.NewReader(tar2), layer1a.ChainID())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	layer2b, err := ls.(*layerStore).RegisterByGraphID(graphID2, layer1a.ChainID(), "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assertReferences(t, layer2a, layer2b)
+
+	if metadata, err := ls.Release(layer2a); err != nil {
+		t.Fatal(err)
+	} else if len(metadata) > 0 {
+		t.Fatalf("Unexpected layer removal after first release: %#v", metadata)
+	}
+
+	metadata, err := ls.Release(layer2b)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assertMetadata(t, metadata, createMetadata(layer2a))
+}
+
+func TestMountMigration(t *testing.T) {
+	ls, cleanup := newTestStore(t)
+	defer cleanup()
+
+	baseFiles := []FileApplier{
+		newTestFile("/root/.bashrc", []byte("# Boring configuration"), 0644),
+		newTestFile("/etc/profile", []byte("# Base configuration"), 0644),
+	}
+	initFiles := []FileApplier{
+		newTestFile("/etc/hosts", []byte{}, 0644),
+		newTestFile("/etc/resolv.conf", []byte{}, 0644),
+	}
+	mountFiles := []FileApplier{
+		newTestFile("/etc/hosts", []byte("localhost 127.0.0.1"), 0644),
+		newTestFile("/root/.bashrc", []byte("# Updated configuration"), 0644),
+		newTestFile("/root/testfile1.txt", []byte("nothing valuable"), 0644),
+	}
+
+	initTar, err := tarFromFiles(initFiles...)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mountTar, err := tarFromFiles(mountFiles...)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	graph := ls.(*layerStore).driver
+
+	layer1, err := createLayer(ls, "", initWithFiles(baseFiles...))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	graphID1 := layer1.(*referencedCacheLayer).cacheID
+
+	containerID := stringid.GenerateRandomID()
+	containerInit := fmt.Sprintf("%s-init", containerID)
+
+	if err := graph.Create(containerInit, graphID1, ""); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := graph.ApplyDiff(containerInit, graphID1, archive.Reader(bytes.NewReader(initTar))); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := graph.Create(containerID, containerInit, ""); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := graph.ApplyDiff(containerID, containerInit, archive.Reader(bytes.NewReader(mountTar))); err != nil {
+		t.Fatal(err)
+	}
+
+	rwLayer1, err := ls.(*layerStore).MountByGraphID("migration-mount", containerID, layer1.ChainID())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	changes, err := ls.Changes("migration-mount")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if expected := 5; len(changes) != expected {
+		t.Logf("Changes %#v", changes)
+		t.Fatalf("Wrong number of changes %d, expected %d", len(changes), expected)
+	}
+
+	sortChanges(changes)
+
+	assertChange(t, changes[0], archive.Change{
+		Path: "/etc",
+		Kind: archive.ChangeModify,
+	})
+	assertChange(t, changes[1], archive.Change{
+		Path: "/etc/hosts",
+		Kind: archive.ChangeModify,
+	})
+	assertChange(t, changes[2], archive.Change{
+		Path: "/root",
+		Kind: archive.ChangeModify,
+	})
+	assertChange(t, changes[3], archive.Change{
+		Path: "/root/.bashrc",
+		Kind: archive.ChangeModify,
+	})
+	assertChange(t, changes[4], archive.Change{
+		Path: "/root/testfile1.txt",
+		Kind: archive.ChangeAdd,
+	})
+
+	if expectedCount := 1; rwLayer1.(*mountedLayer).activityCount != expectedCount {
+		t.Fatalf("Wrong activity count %d, expected %d", rwLayer1.(*mountedLayer).activityCount, expectedCount)
+	}
+
+	rwLayer2, err := ls.Mount("migration-mount", layer1.ChainID(), "", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if rwLayer1 != rwLayer2 {
+		t.Fatalf("Wrong rwlayer %v, expected %v", rwLayer2, rwLayer1)
+	}
+
+	if expectedCount := 2; rwLayer2.(*mountedLayer).activityCount != expectedCount {
+		t.Fatalf("Wrong activity count %d, expected %d", rwLayer2.(*mountedLayer).activityCount, expectedCount)
+	}
+
+	if metadata, err := ls.Release(layer1); err != nil {
+		t.Fatal(err)
+	} else if len(metadata) > 0 {
+		t.Fatalf("Expected no layers to be deleted, deleted %#v", metadata)
+	}
+
+	if err := ls.Unmount("migration-mount"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ls.DeleteMount("migration-mount"); err == nil {
+		t.Fatal("Expected error deleting active mount")
+	}
+	if err := ls.Unmount("migration-mount"); err != nil {
+		t.Fatal(err)
+	}
+	metadata, err := ls.DeleteMount("migration-mount")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(metadata) == 0 {
+		t.Fatal("Expected base layer to be deleted when deleting mount")
+	}
+
+	assertMetadata(t, metadata, createMetadata(layer1))
+}

layer/mount_test.go

@@ -0,0 +1,217 @@
+package layer
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"testing"
+
+	"github.com/docker/docker/pkg/archive"
+)
+
+func TestMountInit(t *testing.T) {
+	ls, cleanup := newTestStore(t)
+	defer cleanup()
+
+	basefile := newTestFile("testfile.txt", []byte("base data!"), 0644)
+	initfile := newTestFile("testfile.txt", []byte("init data!"), 0777)
+
+	li := initWithFiles(basefile)
+	layer, err := createLayer(ls, "", li)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mountInit := func(root string) error {
+		return initfile.ApplyFile(root)
+	}
+
+	m, err := ls.Mount("fun-mount", layer.ChainID(), "", mountInit)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	path, err := m.Path()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	f, err := os.Open(filepath.Join(path, "testfile.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+
+	fi, err := f.Stat()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	b, err := ioutil.ReadAll(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if expected := "init data!"; string(b) != expected {
+		t.Fatalf("Unexpected test file contents %q, expected %q", string(b), expected)
+	}
+
+	if fi.Mode().Perm() != 0777 {
+		t.Fatalf("Unexpected filemode %o, expecting %o", fi.Mode().Perm(), 0777)
+	}
+}
+
+func TestMountSize(t *testing.T) {
+	ls, cleanup := newTestStore(t)
+	defer cleanup()
+
+	content1 := []byte("Base contents")
+	content2 := []byte("Mutable contents")
+	contentInit := []byte("why am I excluded from the size ☹")
+
+	li := initWithFiles(newTestFile("file1", content1, 0644))
+	layer, err := createLayer(ls, "", li)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mountInit := func(root string) error {
+		return newTestFile("file-init", contentInit, 0777).ApplyFile(root)
+	}
+
+	m, err := ls.Mount("mount-size", layer.ChainID(), "", mountInit)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	path, err := m.Path()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := ioutil.WriteFile(filepath.Join(path, "file2"), content2, 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	mountSize, err := m.Size()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if expected := len(content2); int(mountSize) != expected {
+		t.Fatalf("Unexpected mount size %d, expected %d", int(mountSize), expected)
+	}
+}
+
+func TestMountChanges(t *testing.T) {
+	ls, cleanup := newTestStore(t)
+	defer cleanup()
+
+	basefiles := []FileApplier{
+		newTestFile("testfile1.txt", []byte("base data!"), 0644),
+		newTestFile("testfile2.txt", []byte("base data!"), 0644),
+		newTestFile("testfile3.txt", []byte("base data!"), 0644),
+	}
+	initfile := newTestFile("testfile1.txt", []byte("init data!"), 0777)
+
+	li := initWithFiles(basefiles...)
+	layer, err := createLayer(ls, "", li)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mountInit := func(root string) error {
+		return initfile.ApplyFile(root)
+	}
+
+	m, err := ls.Mount("mount-changes", layer.ChainID(), "", mountInit)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	path, err := m.Path()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := os.Chmod(filepath.Join(path, "testfile1.txt"), 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := ioutil.WriteFile(filepath.Join(path, "testfile1.txt"), []byte("mount data!"), 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := os.Remove(filepath.Join(path, "testfile2.txt")); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := os.Chmod(filepath.Join(path, "testfile3.txt"), 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := ioutil.WriteFile(filepath.Join(path, "testfile4.txt"), []byte("mount data!"), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	changes, err := ls.Changes("mount-changes")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if expected := 4; len(changes) != expected {
+		t.Fatalf("Wrong number of changes %d, expected %d", len(changes), expected)
+	}
+
+	sortChanges(changes)
+
+	assertChange(t, changes[0], archive.Change{
+		Path: "/testfile1.txt",
+		Kind: archive.ChangeModify,
+	})
+	assertChange(t, changes[1], archive.Change{
+		Path: "/testfile2.txt",
+		Kind: archive.ChangeDelete,
+	})
+	assertChange(t, changes[2], archive.Change{
+		Path: "/testfile3.txt",
+		Kind: archive.ChangeModify,
+	})
+	assertChange(t, changes[3], archive.Change{
+		Path: "/testfile4.txt",
+		Kind: archive.ChangeAdd,
+	})
+}
+
+func assertChange(t *testing.T, actual, expected archive.Change) {
+	if actual.Path != expected.Path {
+		t.Fatalf("Unexpected change path %s, expected %s", actual.Path, expected.Path)
+	}
+	if actual.Kind != expected.Kind {
+		t.Fatalf("Unexpected change type %s, expected %s", actual.Kind, expected.Kind)
+	}
+}
+
+func sortChanges(changes []archive.Change) {
+	cs := &changeSorter{
+		changes: changes,
+	}
+	sort.Sort(cs)
+}
+
+type changeSorter struct {
+	changes []archive.Change
+}
+
+func (cs *changeSorter) Len() int {
+	return len(cs.changes)
+}
+
+func (cs *changeSorter) Swap(i, j int) {
+	cs.changes[i], cs.changes[j] = cs.changes[j], cs.changes[i]
+}
+
+func (cs *changeSorter) Less(i, j int) bool {
+	return cs.changes[i].Path < cs.changes[j].Path
+}

layer/mounted_layer.go

@@ -0,0 +1,64 @@
+package layer
+
+import "io"
+
+type mountedLayer struct {
+	name          string
+	mountID       string
+	initID        string
+	parent        *roLayer
+	path          string
+	layerStore    *layerStore
+	activityCount int
+}
+
+func (ml *mountedLayer) cacheParent() string {
+	if ml.initID != "" {
+		return ml.initID
+	}
+	if ml.parent != nil {
+		return ml.parent.cacheID
+	}
+	return ""
+}
+
+func (ml *mountedLayer) TarStream() (io.Reader, error) {
+	archiver, err := ml.layerStore.driver.Diff(ml.mountID, ml.cacheParent())
+	if err != nil {
+		return nil, err
+	}
+	return autoClosingReader{archiver}, nil
+}
+
+func (ml *mountedLayer) Path() (string, error) {
+	if ml.path == "" {
+		return "", ErrNotMounted
+	}
+	return ml.path, nil
+}
+
+func (ml *mountedLayer) Parent() Layer {
+	if ml.parent != nil {
+		return ml.parent
+	}
+
+	// Return a nil interface instead of an interface wrapping a nil
+	// pointer.
+	return nil
+}
+
+func (ml *mountedLayer) Size() (int64, error) {
+	return ml.layerStore.driver.DiffSize(ml.mountID, ml.cacheParent())
+}
+
+type autoClosingReader struct {
+	source io.ReadCloser
+}
+
+func (r autoClosingReader) Read(p []byte) (n int, err error) {
+	n, err = r.source.Read(p)
+	if err != nil {
+		r.source.Close()
+	}
+	return
+}

layer/ro_layer.go

@@ -0,0 +1,110 @@
+package layer
+
+import "io"
+
+type roLayer struct {
+	chainID    ChainID
+	diffID     DiffID
+	parent     *roLayer
+	cacheID    string
+	size       int64
+	layerStore *layerStore
+
+	referenceCount int
+	references     map[Layer]struct{}
+}
+
+func (rl *roLayer) TarStream() (io.Reader, error) {
+	r, err := rl.layerStore.store.TarSplitReader(rl.chainID)
+	if err != nil {
+		return nil, err
+	}
+
+	return rl.layerStore.assembleTar(rl.cacheID, r, nil)
+}
+
+func (rl *roLayer) ChainID() ChainID {
+	return rl.chainID
+}
+
+func (rl *roLayer) DiffID() DiffID {
+	return rl.diffID
+}
+
+func (rl *roLayer) Parent() Layer {
+	if rl.parent == nil {
+		return nil
+	}
+	return rl.parent
+}
+
+func (rl *roLayer) Size() (size int64, err error) {
+	if rl.parent != nil {
+		size, err = rl.parent.Size()
+		if err != nil {
+			return
+		}
+	}
+
+	return size + rl.size, nil
+}
+
+func (rl *roLayer) DiffSize() (size int64, err error) {
+	return rl.size, nil
+}
+
+func (rl *roLayer) Metadata() (map[string]string, error) {
+	return rl.layerStore.driver.GetMetadata(rl.cacheID)
+}
+
+type referencedCacheLayer struct {
+	*roLayer
+}
+
+func (rl *roLayer) getReference() Layer {
+	ref := &referencedCacheLayer{
+		roLayer: rl,
+	}
+	rl.references[ref] = struct{}{}
+
+	return ref
+}
+
+func (rl *roLayer) hasReference(ref Layer) bool {
+	_, ok := rl.references[ref]
+	return ok
+}
+
+func (rl *roLayer) hasReferences() bool {
+	return len(rl.references) > 0
+}
+
+func (rl *roLayer) deleteReference(ref Layer) {
+	delete(rl.references, ref)
+}
+
+func (rl *roLayer) depth() int {
+	if rl.parent == nil {
+		return 1
+	}
+	return rl.parent.depth() + 1
+}
+
+func storeLayer(tx MetadataTransaction, layer *roLayer) error {
+	if err := tx.SetDiffID(layer.diffID); err != nil {
+		return err
+	}
+	if err := tx.SetSize(layer.size); err != nil {
+		return err
+	}
+	if err := tx.SetCacheID(layer.cacheID); err != nil {
+		return err
+	}
+	if layer.parent != nil {
+		if err := tx.SetParent(layer.parent.chainID); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}