beenull/moby
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add layer store

Browse source 
Layer store manages read-only and read-write layers on a union file system.
Read only layers are always referenced by content addresses.
Read-write layer identifiers are handled by the caller but upon registering
its difference, the committed read-only layer will be referenced by content
hash.

Signed-off-by: Derek McGowan <derek@mcgstyle.net> (github: dmcgowan)
Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
This commit is contained in:
Derek McGowan 2015-11-18 14:15:00 -08:00 committed by Aaron Lehmann
parent 48c7a1618d
commit 500e77bad0
13 changed files with 3278 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
layer/empty.go Normal file
View file

@ -0,0 +1,47 @@
package layer
import (
"archive/tar"
"bytes"
"io"
)
// DigestSHA256EmptyTar is the canonical sha256 digest of empty tar file -
// (1024 NULL bytes)
const DigestSHA256EmptyTar = DiffID("sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef")
type emptyLayer struct{}
// EmptyLayer is a layer that corresponds to empty tar.
var EmptyLayer = &emptyLayer{}
func (el *emptyLayer) TarStream() (io.Reader, error) {
buf := new(bytes.Buffer)
tarWriter := tar.NewWriter(buf)
tarWriter.Close()
return buf, nil
}
func (el *emptyLayer) ChainID() ChainID {
return ChainID(DigestSHA256EmptyTar)
}
func (el *emptyLayer) DiffID() DiffID {
return DigestSHA256EmptyTar
}
func (el *emptyLayer) Parent() Layer {
return nil
}
func (el *emptyLayer) Size() (size int64, err error) {
return 0, nil
}
func (el *emptyLayer) DiffSize() (size int64, err error) {
return 0, nil
}
func (el *emptyLayer) Metadata() (map[string]string, error) {
return make(map[string]string), nil
}

46
layer/empty_test.go Normal file
View file

@ -0,0 +1,46 @@
package layer
import (
"io"
"testing"
"github.com/docker/distribution/digest"
)
func TestEmptyLayer(t *testing.T) {
if EmptyLayer.ChainID() != ChainID(DigestSHA256EmptyTar) {
t.Fatal("wrong ID for empty layer")
}
if EmptyLayer.DiffID() != DigestSHA256EmptyTar {
t.Fatal("wrong DiffID for empty layer")
}
if EmptyLayer.Parent() != nil {
t.Fatal("expected no parent for empty layer")
}
if size, err := EmptyLayer.Size(); err != nil || size != 0 {
t.Fatal("expected zero size for empty layer")
}
if diffSize, err := EmptyLayer.DiffSize(); err != nil || diffSize != 0 {
t.Fatal("expected zero diffsize for empty layer")
}
tarStream, err := EmptyLayer.TarStream()
if err != nil {
t.Fatalf("error streaming tar for empty layer: %v", err)
}
digester := digest.Canonical.New()
_, err = io.Copy(digester.Hash(), tarStream)
if err != nil {
t.Fatalf("error hashing empty tar layer: %v", err)
}
if digester.Digest() != digest.Digest(DigestSHA256EmptyTar) {
t.Fatal("empty layer tar stream hashes to wrong value")
}
}

318
layer/filestore.go Normal file
View file

@ -0,0 +1,318 @@
package layer
import (
"compress/gzip"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"regexp"
"strconv"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/digest"
"github.com/docker/docker/pkg/ioutils"
)
var (
stringIDRegexp = regexp.MustCompile(`^[a-f0-9]{64}(-init)?$`)
supportedAlgorithms = []digest.Algorithm{
digest.SHA256,
// digest.SHA384, // Currently not used
// digest.SHA512, // Currently not used
}
)
type fileMetadataStore struct {
root string
}
type fileMetadataTransaction struct {
store *fileMetadataStore
root string
}
// NewFSMetadataStore returns an instance of a metadata store
// which is backed by files on disk using the provided root
// as the root of metadata files.
func NewFSMetadataStore(root string) (MetadataStore, error) {
if err := os.MkdirAll(root, 0700); err != nil {
return nil, err
}
return &fileMetadataStore{
root: root,
}, nil
}
func (fms *fileMetadataStore) getLayerDirectory(layer ChainID) string {
dgst := digest.Digest(layer)
return filepath.Join(fms.root, string(dgst.Algorithm()), dgst.Hex())
}
func (fms *fileMetadataStore) getLayerFilename(layer ChainID, filename string) string {
return filepath.Join(fms.getLayerDirectory(layer), filename)
}
func (fms *fileMetadataStore) getMountDirectory(mount string) string {
return filepath.Join(fms.root, "mounts", mount)
}
func (fms *fileMetadataStore) getMountFilename(mount, filename string) string {
return filepath.Join(fms.getMountDirectory(mount), filename)
}
func (fms *fileMetadataStore) StartTransaction() (MetadataTransaction, error) {
tmpDir := filepath.Join(fms.root, "tmp")
if err := os.MkdirAll(tmpDir, 0755); err != nil {
return nil, err
}
td, err := ioutil.TempDir(tmpDir, "layer-")
if err != nil {
return nil, err
}
// Create a new tempdir
return &fileMetadataTransaction{
store: fms,
root: td,
}, nil
}
func (fm *fileMetadataTransaction) SetSize(size int64) error {
content := fmt.Sprintf("%d", size)
return ioutil.WriteFile(filepath.Join(fm.root, "size"), []byte(content), 0644)
}
func (fm *fileMetadataTransaction) SetParent(parent ChainID) error {
return ioutil.WriteFile(filepath.Join(fm.root, "parent"), []byte(digest.Digest(parent).String()), 0644)
}
func (fm *fileMetadataTransaction) SetDiffID(diff DiffID) error {
return ioutil.WriteFile(filepath.Join(fm.root, "diff"), []byte(digest.Digest(diff).String()), 0644)
}
func (fm *fileMetadataTransaction) SetCacheID(cacheID string) error {
return ioutil.WriteFile(filepath.Join(fm.root, "cache-id"), []byte(cacheID), 0644)
}
func (fm *fileMetadataTransaction) TarSplitWriter() (io.WriteCloser, error) {
f, err := os.OpenFile(filepath.Join(fm.root, "tar-split.json.gz"), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return nil, err
}
fz := gzip.NewWriter(f)
return ioutils.NewWriteCloserWrapper(fz, func() error {
fz.Close()
return f.Close()
}), nil
}
func (fm *fileMetadataTransaction) Commit(layer ChainID) error {
finalDir := fm.store.getLayerDirectory(layer)
if err := os.MkdirAll(filepath.Dir(finalDir), 0755); err != nil {
return err
}
return os.Rename(fm.root, finalDir)
}
func (fm *fileMetadataTransaction) Cancel() error {
return os.RemoveAll(fm.root)
}
func (fm *fileMetadataTransaction) String() string {
return fm.root
}
func (fms *fileMetadataStore) GetSize(layer ChainID) (int64, error) {
content, err := ioutil.ReadFile(fms.getLayerFilename(layer, "size"))
if err != nil {
return 0, err
}
size, err := strconv.ParseInt(string(content), 10, 64)
if err != nil {
return 0, err
}
return size, nil
}
func (fms *fileMetadataStore) GetParent(layer ChainID) (ChainID, error) {
content, err := ioutil.ReadFile(fms.getLayerFilename(layer, "parent"))
if err != nil {
if os.IsNotExist(err) {
return "", nil
}
return "", err
}
dgst, err := digest.ParseDigest(string(content))
if err != nil {
return "", err
}
return ChainID(dgst), nil
}
func (fms *fileMetadataStore) GetDiffID(layer ChainID) (DiffID, error) {
content, err := ioutil.ReadFile(fms.getLayerFilename(layer, "diff"))
if err != nil {
return "", err
}
dgst, err := digest.ParseDigest(string(content))
if err != nil {
return "", err
}
return DiffID(dgst), nil
}
func (fms *fileMetadataStore) GetCacheID(layer ChainID) (string, error) {
content, err := ioutil.ReadFile(fms.getLayerFilename(layer, "cache-id"))
if err != nil {
return "", err
}
if !stringIDRegexp.MatchString(string(content)) {
return "", errors.New("invalid cache id value")
}
return string(content), nil
}
func (fms *fileMetadataStore) TarSplitReader(layer ChainID) (io.ReadCloser, error) {
fz, err := os.Open(fms.getLayerFilename(layer, "tar-split.json.gz"))
if err != nil {
return nil, err
}
f, err := gzip.NewReader(fz)
if err != nil {
return nil, err
}
return ioutils.NewReadCloserWrapper(f, func() error {
f.Close()
return fz.Close()
}), nil
}
func (fms *fileMetadataStore) SetMountID(mount string, mountID string) error {
if err := os.MkdirAll(fms.getMountDirectory(mount), 0755); err != nil {
return err
}
return ioutil.WriteFile(fms.getMountFilename(mount, "mount-id"), []byte(mountID), 0644)
}
func (fms *fileMetadataStore) SetInitID(mount string, init string) error {
if err := os.MkdirAll(fms.getMountDirectory(mount), 0755); err != nil {
return err
}
return ioutil.WriteFile(fms.getMountFilename(mount, "init-id"), []byte(init), 0644)
}
func (fms *fileMetadataStore) SetMountParent(mount string, parent ChainID) error {
if err := os.MkdirAll(fms.getMountDirectory(mount), 0755); err != nil {
return err
}
return ioutil.WriteFile(fms.getMountFilename(mount, "parent"), []byte(digest.Digest(parent).String()), 0644)
}
func (fms *fileMetadataStore) GetMountID(mount string) (string, error) {
content, err := ioutil.ReadFile(fms.getMountFilename(mount, "mount-id"))
if err != nil {
return "", err
}
if !stringIDRegexp.MatchString(string(content)) {
return "", errors.New("invalid mount id value")
}
return string(content), nil
}
func (fms *fileMetadataStore) GetInitID(mount string) (string, error) {
content, err := ioutil.ReadFile(fms.getMountFilename(mount, "init-id"))
if err != nil {
if os.IsNotExist(err) {
return "", nil
}
return "", err
}
if !stringIDRegexp.MatchString(string(content)) {
return "", errors.New("invalid init id value")
}
return string(content), nil
}
func (fms *fileMetadataStore) GetMountParent(mount string) (ChainID, error) {
content, err := ioutil.ReadFile(fms.getMountFilename(mount, "parent"))
if err != nil {
if os.IsNotExist(err) {
return "", nil
}
return "", err
}
dgst, err := digest.ParseDigest(string(content))
if err != nil {
return "", err
}
return ChainID(dgst), nil
}
func (fms *fileMetadataStore) List() ([]ChainID, []string, error) {
var ids []ChainID
for _, algorithm := range supportedAlgorithms {
fileInfos, err := ioutil.ReadDir(filepath.Join(fms.root, string(algorithm)))
if err != nil {
if os.IsNotExist(err) {
continue
}
return nil, nil, err
}
for _, fi := range fileInfos {
if fi.IsDir() && fi.Name() != "mounts" {
dgst := digest.NewDigestFromHex(string(algorithm), fi.Name())
if err := dgst.Validate(); err != nil {
logrus.Debugf("Ignoring invalid digest %s:%s", algorithm, fi.Name())
} else {
ids = append(ids, ChainID(dgst))
}
}
}
}
fileInfos, err := ioutil.ReadDir(filepath.Join(fms.root, "mounts"))
if err != nil {
if os.IsNotExist(err) {
return ids, []string{}, nil
}
return nil, nil, err
}
var mounts []string
for _, fi := range fileInfos {
if fi.IsDir() {
mounts = append(mounts, fi.Name())
}
}
return ids, mounts, nil
}
func (fms *fileMetadataStore) Remove(layer ChainID) error {
return os.RemoveAll(fms.getLayerDirectory(layer))
}
func (fms *fileMetadataStore) RemoveMount(mount string) error {
return os.RemoveAll(fms.getMountDirectory(mount))
}

119
layer/filestore_test.go Normal file
View file

@ -0,0 +1,119 @@
package layer
import (
"fmt"
"io/ioutil"
"math/rand"
"os"
"path/filepath"
"strings"
"syscall"
"testing"
"github.com/docker/distribution/digest"
)
func randomLayerID(seed int64) ChainID {
r := rand.New(rand.NewSource(seed))
dgst, err := digest.FromBytes([]byte(fmt.Sprintf("%d", r.Int63())))
if err != nil {
panic(err)
}
return ChainID(dgst)
}
func newFileMetadataStore(t *testing.T) (*fileMetadataStore, string, func()) {
td, err := ioutil.TempDir("", "layers-")
if err != nil {
t.Fatal(err)
}
fms, err := NewFSMetadataStore(td)
if err != nil {
t.Fatal(err)
}
return fms.(*fileMetadataStore), td, func() {
if err := os.RemoveAll(td); err != nil {
t.Logf("Failed to cleanup %q: %s", td, err)
}
}
}
func assertNotDirectoryError(t *testing.T, err error) {
perr, ok := err.(*os.PathError)
if !ok {
t.Fatalf("Unexpected error %#v, expected path error", err)
}
if perr.Err != syscall.ENOTDIR {
t.Fatalf("Unexpected error %s, expected %s", perr.Err, syscall.ENOTDIR)
}
}
func assertPermissionError(t *testing.T, err error) {
perr, ok := err.(*os.PathError)
if !ok {
t.Fatalf("Unexpected error %#v, expected path error", err)
}
if perr.Err != syscall.EACCES {
t.Fatalf("Unexpected error %s, expected %s", perr.Err, syscall.EACCES)
}
}
func TestCommitFailure(t *testing.T) {
fms, td, cleanup := newFileMetadataStore(t)
defer cleanup()
if err := ioutil.WriteFile(filepath.Join(td, "sha256"), []byte("was here first!"), 0644); err != nil {
t.Fatal(err)
}
tx, err := fms.StartTransaction()
if err != nil {
t.Fatal(err)
}
if err := tx.SetSize(0); err != nil {
t.Fatal(err)
}
err = tx.Commit(randomLayerID(5))
if err == nil {
t.Fatalf("Expected error committing with invalid layer parent directory")
}
assertNotDirectoryError(t, err)
}
func TestStartTransactionFailure(t *testing.T) {
fms, td, cleanup := newFileMetadataStore(t)
defer cleanup()
if err := ioutil.WriteFile(filepath.Join(td, "tmp"), []byte("was here first!"), 0644); err != nil {
t.Fatal(err)
}
_, err := fms.StartTransaction()
if err == nil {
t.Fatalf("Expected error starting transaction with invalid layer parent directory")
}
assertNotDirectoryError(t, err)
if err := os.Remove(filepath.Join(td, "tmp")); err != nil {
t.Fatal(err)
}
tx, err := fms.StartTransaction()
if err != nil {
t.Fatal(err)
}
if expected := filepath.Join(td, "tmp"); strings.HasPrefix(expected, tx.String()) {
t.Fatalf("Unexpected transaction string %q, expected prefix %q", tx.String(), expected)
}
if err := tx.Cancel(); err != nil {
t.Fatal(err)
}
}

238
layer/layer.go Normal file
View file

@ -0,0 +1,238 @@
// Package layer is package for managing read only
// and read-write mounts on the union file system
// driver. Read-only mounts are refenced using a
// content hash and are protected from mutation in
// the exposed interface. The tar format is used
// to create read only layers and export both
// read only and writable layers. The exported
// tar data for a read only layer should match
// the tar used to create the layer.
package layer
import (
"errors"
"io"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/digest"
"github.com/docker/docker/pkg/archive"
)
var (
// ErrLayerDoesNotExist is used when an operation is
// attempted on a layer which does not exist.
ErrLayerDoesNotExist = errors.New("layer does not exist")
// ErrLayerNotRetained is used when a release is
// attempted on a layer which is not retained.
ErrLayerNotRetained = errors.New("layer not retained")
// ErrMountDoesNotExist is used when an operation is
// attempted on a mount layer which does not exist.
ErrMountDoesNotExist = errors.New("mount does not exist")
// ErrActiveMount is used when an operation on a
// mount is attempted but the layer is still
// mounted and the operation cannot be performed.
ErrActiveMount = errors.New("mount still active")
// ErrNotMounted is used when requesting an active
// mount but the layer is not mounted.
ErrNotMounted = errors.New("not mounted")
// ErrMaxDepthExceeded is used when a layer is attempted
// to be created which would result in a layer depth
// greater than the 125 max.
ErrMaxDepthExceeded = errors.New("max depth exceeded")
)
// ChainID is the content-addressable ID of a layer.
type ChainID digest.Digest
// String returns a string rendition of a layer ID
func (id ChainID) String() string {
return string(id)
}
// DiffID is the hash of an individual layer tar.
type DiffID digest.Digest
// String returns a string rendition of a layer DiffID
func (diffID DiffID) String() string {
return string(diffID)
}
// TarStreamer represents an object which may
// have its contents exported as a tar stream.
type TarStreamer interface {
// TarStream returns a tar archive stream
// for the contents of a layer.
TarStream() (io.Reader, error)
}
// Layer represents a read only layer
type Layer interface {
TarStreamer
// ChainID returns the content hash of the entire layer chain. The hash
// chain is made up of DiffID of top layer and all of its parents.
ChainID() ChainID
// DiffID returns the content hash of the layer
// tar stream used to create this layer.
DiffID() DiffID
// Parent returns the next layer in the layer chain.
Parent() Layer
// Size returns the size of the entire layer chain. The size
// is calculated from the total size of all files in the layers.
Size() (int64, error)
// DiffSize returns the size difference of the top layer
// from parent layer.
DiffSize() (int64, error)
// Metadata returns the low level storage metadata associated
// with layer.
Metadata() (map[string]string, error)
}
// RWLayer represents a layer which is
// read and writable
type RWLayer interface {
TarStreamer
// Path returns the filesystem path to the writable
// layer.
Path() (string, error)
// Parent returns the layer which the writable
// layer was created from.
Parent() Layer
// Size represents the size of the writable layer
// as calculated by the total size of the files
// changed in the mutable layer.
Size() (int64, error)
}
// Metadata holds information about a
// read only layer
type Metadata struct {
// ChainID is the content hash of the layer
ChainID ChainID
// DiffID is the hash of the tar data used to
// create the layer
DiffID DiffID
// Size is the size of the layer and all parents
Size int64
// DiffSize is the size of the top layer
DiffSize int64
}
// MountInit is a function to initialize a
// writable mount. Changes made here will
// not be included in the Tar stream of the
// RWLayer.
type MountInit func(root string) error
// Store represents a backend for managing both
// read-only and read-write layers.
type Store interface {
Register(io.Reader, ChainID) (Layer, error)
Get(ChainID) (Layer, error)
Release(Layer) ([]Metadata, error)
Mount(id string, parent ChainID, label string, init MountInit) (RWLayer, error)
Unmount(id string) error
DeleteMount(id string) ([]Metadata, error)
Changes(id string) ([]archive.Change, error)
}
// MetadataTransaction represents functions for setting layer metadata
// with a single transaction.
type MetadataTransaction interface {
SetSize(int64) error
SetParent(parent ChainID) error
SetDiffID(DiffID) error
SetCacheID(string) error
TarSplitWriter() (io.WriteCloser, error)
Commit(ChainID) error
Cancel() error
String() string
}
// MetadataStore represents a backend for persisting
// metadata about layers and providing the metadata
// for restoring a Store.
type MetadataStore interface {
// StartTransaction starts an update for new metadata
// which will be used to represent an ID on commit.
StartTransaction() (MetadataTransaction, error)
GetSize(ChainID) (int64, error)
GetParent(ChainID) (ChainID, error)
GetDiffID(ChainID) (DiffID, error)
GetCacheID(ChainID) (string, error)
TarSplitReader(ChainID) (io.ReadCloser, error)
SetMountID(string, string) error
SetInitID(string, string) error
SetMountParent(string, ChainID) error
GetMountID(string) (string, error)
GetInitID(string) (string, error)
GetMountParent(string) (ChainID, error)
// List returns the full list of referened
// read-only and read-write layers
List() ([]ChainID, []string, error)
Remove(ChainID) error
RemoveMount(string) error
}
// CreateChainID returns ID for a layerDigest slice
func CreateChainID(dgsts []DiffID) ChainID {
return createChainIDFromParent("", dgsts...)
}
func createChainIDFromParent(parent ChainID, dgsts ...DiffID) ChainID {
if len(dgsts) == 0 {
return parent
}
if parent == "" {
return createChainIDFromParent(ChainID(dgsts[0]), dgsts[1:]...)
}
// H = "H(n-1) SHA256(n)"
dgst, err := digest.FromBytes([]byte(string(parent) + " " + string(dgsts[0])))
if err != nil {
// Digest calculation is not expected to throw an error,
// any error at this point is a program error
panic(err)
}
return createChainIDFromParent(ChainID(dgst), dgsts[1:]...)
}
// ReleaseAndLog releases the provided layer from the given layer
// store, logging any error and release metadata
func ReleaseAndLog(ls Store, l Layer) {
metadata, err := ls.Release(l)
if err != nil {
logrus.Errorf("Error releasing layer %s: %v", l.ChainID(), err)
}
LogReleaseMetadata(metadata)
}
// LogReleaseMetadata logs a metadata array, use this to
// ensure consistent logging for release metadata
func LogReleaseMetadata(metadatas []Metadata) {
for _, metadata := range metadatas {
logrus.Infof("Layer %s cleaned up", metadata.ChainID)
}
}

649
layer/layer_store.go Normal file
View file

@ -0,0 +1,649 @@
package layer
import (
"errors"
"fmt"
"io"
"io/ioutil"
"runtime"
"sync"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/digest"
"github.com/docker/docker/daemon/graphdriver"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/stringid"
"github.com/vbatts/tar-split/tar/asm"
"github.com/vbatts/tar-split/tar/storage"
)
// maxLayerDepth represents the maximum number of
// layers which can be chained together. 125 was
// chosen to account for the 127 max in some
// graphdrivers plus the 2 additional layers
// used to create a rwlayer.
const maxLayerDepth = 125
type layerStore struct {
store MetadataStore
driver graphdriver.Driver
layerMap map[ChainID]*roLayer
layerL sync.Mutex
mounts map[string]*mountedLayer
mountL sync.Mutex
}
// NewStore creates a new Store instance using
// the provided metadata store and graph driver.
// The metadata store will be used to restore
// the Store.
func NewStore(store MetadataStore, driver graphdriver.Driver) (Store, error) {
ls := &layerStore{
store: store,
driver: driver,
layerMap: map[ChainID]*roLayer{},
mounts: map[string]*mountedLayer{},
}
ids, mounts, err := store.List()
if err != nil {
return nil, err
}
for _, id := range ids {
l, err := ls.loadLayer(id)
if err != nil {
logrus.Debugf("Failed to load layer %s: %s", id, err)
}
if l.parent != nil {
l.parent.referenceCount++
}
}
for _, mount := range mounts {
if err := ls.loadMount(mount); err != nil {
logrus.Debugf("Failed to load mount %s: %s", mount, err)
}
}
return ls, nil
}
func (ls *layerStore) loadLayer(layer ChainID) (*roLayer, error) {
cl, ok := ls.layerMap[layer]
if ok {
return cl, nil
}
diff, err := ls.store.GetDiffID(layer)
if err != nil {
return nil, err
}
size, err := ls.store.GetSize(layer)
if err != nil {
return nil, err
}
cacheID, err := ls.store.GetCacheID(layer)
if err != nil {
return nil, err
}
parent, err := ls.store.GetParent(layer)
if err != nil {
return nil, err
}
cl = &roLayer{
chainID: layer,
diffID: diff,
size: size,
cacheID: cacheID,
layerStore: ls,
references: map[Layer]struct{}{},
}
if parent != "" {
p, err := ls.loadLayer(parent)
if err != nil {
return nil, err
}
cl.parent = p
}
ls.layerMap[cl.chainID] = cl
return cl, nil
}
func (ls *layerStore) loadMount(mount string) error {
if _, ok := ls.mounts[mount]; ok {
return nil
}
mountID, err := ls.store.GetMountID(mount)
if err != nil {
return err
}
initID, err := ls.store.GetInitID(mount)
if err != nil {
return err
}
parent, err := ls.store.GetMountParent(mount)
if err != nil {
return err
}
ml := &mountedLayer{
name: mount,
mountID: mountID,
initID: initID,
layerStore: ls,
}
if parent != "" {
p, err := ls.loadLayer(parent)
if err != nil {
return err
}
ml.parent = p
p.referenceCount++
}
ls.mounts[ml.name] = ml
return nil
}
func (ls *layerStore) applyTar(tx MetadataTransaction, ts io.Reader, parent string, layer *roLayer) error {
digester := digest.Canonical.New()
tr := io.TeeReader(ts, digester.Hash())
tsw, err := tx.TarSplitWriter()
if err != nil {
return err
}
metaPacker := storage.NewJSONPacker(tsw)
defer tsw.Close()
// we're passing nil here for the file putter, because the ApplyDiff will
// handle the extraction of the archive
rdr, err := asm.NewInputTarStream(tr, metaPacker, nil)
if err != nil {
return err
}
applySize, err := ls.driver.ApplyDiff(layer.cacheID, parent, archive.Reader(rdr))
if err != nil {
return err
}
// Discard trailing data but ensure metadata is picked up to reconstruct stream
io.Copy(ioutil.Discard, rdr) // ignore error as reader may be closed
layer.size = applySize
layer.diffID = DiffID(digester.Digest())
logrus.Debugf("Applied tar %s to %s, size: %d", layer.diffID, layer.cacheID, applySize)
return nil
}
func (ls *layerStore) Register(ts io.Reader, parent ChainID) (Layer, error) {
// err is used to hold the error which will always trigger
// cleanup of creates sources but may not be an error returned
// to the caller (already exists).
var err error
var pid string
var p *roLayer
if string(parent) != "" {
p = ls.get(parent)
if p == nil {
return nil, ErrLayerDoesNotExist
}
pid = p.cacheID
// Release parent chain if error
defer func() {
if err != nil {
ls.layerL.Lock()
ls.releaseLayer(p)
ls.layerL.Unlock()
}
}()
if p.depth() >= maxLayerDepth {
err = ErrMaxDepthExceeded
return nil, err
}
}
// Create new roLayer
layer := &roLayer{
parent: p,
cacheID: stringid.GenerateRandomID(),
referenceCount: 1,
layerStore: ls,
references: map[Layer]struct{}{},
}
if err = ls.driver.Create(layer.cacheID, pid, ""); err != nil {
return nil, err
}
tx, err := ls.store.StartTransaction()
if err != nil {
return nil, err
}
defer func() {
if err != nil {
logrus.Debugf("Cleaning up layer %s: %v", layer.cacheID, err)
if err := ls.driver.Remove(layer.cacheID); err != nil {
logrus.Errorf("Error cleaning up cache layer %s: %v", layer.cacheID, err)
}
if err := tx.Cancel(); err != nil {
logrus.Errorf("Error canceling metadata transaction %q: %s", tx.String(), err)
}
}
}()
if err = ls.applyTar(tx, ts, pid, layer); err != nil {
return nil, err
}
if layer.parent == nil {
layer.chainID = ChainID(layer.diffID)
} else {
layer.chainID = createChainIDFromParent(layer.parent.chainID, layer.diffID)
}
if err = storeLayer(tx, layer); err != nil {
return nil, err
}
ls.layerL.Lock()
defer ls.layerL.Unlock()
if existingLayer := ls.getAndRetainLayer(layer.chainID); existingLayer != nil {
// Set error for cleanup, but do not return the error
err = errors.New("layer already exists")
return existingLayer.getReference(), nil
}
if err = tx.Commit(layer.chainID); err != nil {
return nil, err
}
ls.layerMap[layer.chainID] = layer
return layer.getReference(), nil
}
func (ls *layerStore) get(l ChainID) *roLayer {
ls.layerL.Lock()
defer ls.layerL.Unlock()
layer, ok := ls.layerMap[l]
if !ok {
return nil
}
layer.referenceCount++
return layer
}
func (ls *layerStore) Get(l ChainID) (Layer, error) {
layer := ls.get(l)
if layer == nil {
return nil, ErrLayerDoesNotExist
}
return layer.getReference(), nil
}
func (ls *layerStore) deleteLayer(layer *roLayer, metadata *Metadata) error {
err := ls.driver.Remove(layer.cacheID)
if err != nil {
return err
}
err = ls.store.Remove(layer.chainID)
if err != nil {
return err
}
metadata.DiffID = layer.diffID
metadata.ChainID = layer.chainID
metadata.Size, err = layer.Size()
if err != nil {
return err
}
metadata.DiffSize = layer.size
return nil
}
func (ls *layerStore) releaseLayers(l *roLayer, removed *[]Metadata, depth int) error {
if l.referenceCount == 0 {
panic("layer not retained")
}
l.referenceCount--
if l.referenceCount != 0 {
return nil
}
if len(*removed) == 0 && depth > 0 {
panic("cannot remove layer with child")
}
if l.hasReferences() {
panic("cannot delete referenced layer")
}
var metadata Metadata
if err := ls.deleteLayer(l, &metadata); err != nil {
return err
}
delete(ls.layerMap, l.chainID)
*removed = append(*removed, metadata)
if l.parent != nil {
if err := ls.releaseLayers(l.parent, removed, depth+1); err != nil {
return err
}
}
return nil
}
func (ls *layerStore) releaseLayer(layer *roLayer) ([]Metadata, error) {
removed := []Metadata{}
err := ls.releaseLayers(layer, &removed, 0)
return removed, err
}
func (ls *layerStore) Release(l Layer) ([]Metadata, error) {
ls.layerL.Lock()
defer ls.layerL.Unlock()
layer, ok := ls.layerMap[l.ChainID()]
if !ok {
return []Metadata{}, nil
}
if !layer.hasReference(l) {
return nil, ErrLayerNotRetained
}
layer.deleteReference(l)
return ls.releaseLayer(layer)
}
func (ls *layerStore) mount(m *mountedLayer, mountLabel string) error {
dir, err := ls.driver.Get(m.mountID, mountLabel)
if err != nil {
return err
}
m.path = dir
m.activityCount++
return nil
}
func (ls *layerStore) saveMount(mount *mountedLayer) error {
if err := ls.store.SetMountID(mount.name, mount.mountID); err != nil {
return err
}
if mount.initID != "" {
if err := ls.store.SetInitID(mount.name, mount.initID); err != nil {
return err
}
}
if mount.parent != nil {
if err := ls.store.SetMountParent(mount.name, mount.parent.chainID); err != nil {
return err
}
}
ls.mounts[mount.name] = mount
return nil
}
func (ls *layerStore) getAndRetainLayer(layer ChainID) *roLayer {
l, ok := ls.layerMap[layer]
if !ok {
return nil
}
l.referenceCount++
return l
}
func (ls *layerStore) initMount(graphID, parent, mountLabel string, initFunc MountInit) (string, error) {
// Use "<graph-id>-init" to maintain compatibility with graph drivers
// which are expecting this layer with this special name. If all
// graph drivers can be updated to not rely on knowin about this layer
// then the initID should be randomly generated.
initID := fmt.Sprintf("%s-init", graphID)
if err := ls.driver.Create(initID, parent, mountLabel); err != nil {
}
p, err := ls.driver.Get(initID, "")
if err != nil {
return "", err
}
if err := initFunc(p); err != nil {
ls.driver.Put(initID)
return "", err
}
if err := ls.driver.Put(initID); err != nil {
return "", err
}
return initID, nil
}
func (ls *layerStore) Mount(name string, parent ChainID, mountLabel string, initFunc MountInit) (l RWLayer, err error) {
ls.mountL.Lock()
defer ls.mountL.Unlock()
m, ok := ls.mounts[name]
if ok {
// Check if has path
if err := ls.mount(m, mountLabel); err != nil {
return nil, err
}
return m, nil
}
var pid string
var p *roLayer
if string(parent) != "" {
ls.layerL.Lock()
p = ls.getAndRetainLayer(parent)
ls.layerL.Unlock()
if p == nil {
return nil, ErrLayerDoesNotExist
}
pid = p.cacheID
// Release parent chain if error
defer func() {
if err != nil {
ls.layerL.Lock()
ls.releaseLayer(p)
ls.layerL.Unlock()
}
}()
}
mountID := name
if runtime.GOOS != "windows" {
// windows has issues if container ID doesn't match mount ID
mountID = stringid.GenerateRandomID()
}
m = &mountedLayer{
name: name,
parent: p,
mountID: mountID,
layerStore: ls,
}
if initFunc != nil {
pid, err = ls.initMount(m.mountID, pid, mountLabel, initFunc)
if err != nil {
return nil, err
}
m.initID = pid
}
if err = ls.driver.Create(m.mountID, pid, ""); err != nil {
return nil, err
}
if err = ls.saveMount(m); err != nil {
return nil, err
}
if err = ls.mount(m, mountLabel); err != nil {
return nil, err
}
return m, nil
}
func (ls *layerStore) Unmount(name string) error {
ls.mountL.Lock()
defer ls.mountL.Unlock()
m := ls.mounts[name]
if m == nil {
return ErrMountDoesNotExist
}
m.activityCount--
if err := ls.driver.Put(m.mountID); err != nil {
return err
}
return nil
}
func (ls *layerStore) DeleteMount(name string) ([]Metadata, error) {
ls.mountL.Lock()
defer ls.mountL.Unlock()
m := ls.mounts[name]
if m == nil {
return nil, ErrMountDoesNotExist
}
if m.activityCount > 0 {
return nil, ErrActiveMount
}
delete(ls.mounts, name)
if err := ls.driver.Remove(m.mountID); err != nil {
logrus.Errorf("Error removing mounted layer %s: %s", m.name, err)
return nil, err
}
if m.initID != "" {
if err := ls.driver.Remove(m.initID); err != nil {
logrus.Errorf("Error removing init layer %s: %s", m.name, err)
return nil, err
}
}
if err := ls.store.RemoveMount(m.name); err != nil {
logrus.Errorf("Error removing mount metadata: %s: %s", m.name, err)
return nil, err
}
ls.layerL.Lock()
defer ls.layerL.Unlock()
if m.parent != nil {
return ls.releaseLayer(m.parent)
}
return []Metadata{}, nil
}
func (ls *layerStore) Changes(name string) ([]archive.Change, error) {
ls.mountL.Lock()
m := ls.mounts[name]
ls.mountL.Unlock()
if m == nil {
return nil, ErrMountDoesNotExist
}
pid := m.initID
if pid == "" && m.parent != nil {
pid = m.parent.cacheID
}
return ls.driver.Changes(m.mountID, pid)
}
func (ls *layerStore) assembleTar(graphID string, metadata io.ReadCloser, size *int64) (io.Reader, error) {
type diffPathDriver interface {
DiffPath(string) (string, func() error, error)
}
diffDriver, ok := ls.driver.(diffPathDriver)
if !ok {
diffDriver = &naiveDiffPathDriver{ls.driver}
}
// get our relative path to the container
fsPath, releasePath, err := diffDriver.DiffPath(graphID)
if err != nil {
metadata.Close()
return nil, err
}
pR, pW := io.Pipe()
// this will need to be in a goroutine, as we are returning the stream of a
// tar archive, but can not close the metadata reader early (when this
// function returns)...
go func() {
defer releasePath()
defer metadata.Close()
metaUnpacker := storage.NewJSONUnpacker(metadata)
upackerCounter := &unpackSizeCounter{metaUnpacker, size}
fileGetter := storage.NewPathFileGetter(fsPath)
logrus.Debugf("Assembling tar data for %s from %s", graphID, fsPath)
ots := asm.NewOutputTarStream(fileGetter, upackerCounter)
defer ots.Close()
if _, err := io.Copy(pW, ots); err != nil {
pW.CloseWithError(err)
return
}
pW.Close()
}()
return pR, nil
}
type naiveDiffPathDriver struct {
graphdriver.Driver
}
func (n *naiveDiffPathDriver) DiffPath(id string) (string, func() error, error) {
p, err := n.Driver.Get(id, "")
if err != nil {
return "", nil, err
}
return p, func() error {
return n.Driver.Put(id)
}, nil
}

725
layer/layer_test.go Normal file
View file

@ -0,0 +1,725 @@
package layer
import (
"bytes"
"io"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/docker/distribution/digest"
"github.com/docker/docker/daemon/graphdriver"
"github.com/docker/docker/daemon/graphdriver/vfs"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/idtools"
"github.com/docker/docker/pkg/stringid"
)
func init() {
graphdriver.ApplyUncompressedLayer = archive.UnpackLayer
vfs.CopyWithTar = archive.CopyWithTar
}
func newVFSGraphDriver(td string) (graphdriver.Driver, error) {
uidMap := []idtools.IDMap{
{
ContainerID: 0,
HostID: os.Getuid(),
Size: 1,
},
}
gidMap := []idtools.IDMap{
{
ContainerID: 0,
HostID: os.Getgid(),
Size: 1,
},
}
return graphdriver.GetDriver("vfs", td, nil, uidMap, gidMap)
}
func newTestGraphDriver(t *testing.T) (graphdriver.Driver, func()) {
td, err := ioutil.TempDir("", "graph-")
if err != nil {
t.Fatal(err)
}
driver, err := newVFSGraphDriver(td)
if err != nil {
t.Fatal(err)
}
return driver, func() {
os.RemoveAll(td)
}
}
func newTestStore(t *testing.T) (Store, func()) {
td, err := ioutil.TempDir("", "layerstore-")
if err != nil {
t.Fatal(err)
}
graph, graphcleanup := newTestGraphDriver(t)
fms, err := NewFSMetadataStore(td)
if err != nil {
t.Fatal(err)
}
ls, err := NewStore(fms, graph)
if err != nil {
t.Fatal(err)
}
return ls, func() {
graphcleanup()
os.RemoveAll(td)
}
}
type layerInit func(root string) error
func createLayer(ls Store, parent ChainID, layerFunc layerInit) (Layer, error) {
containerID := stringid.GenerateRandomID()
mount, err := ls.Mount(containerID, parent, "", nil)
if err != nil {
return nil, err
}
path, err := mount.Path()
if err != nil {
return nil, err
}
if err := layerFunc(path); err != nil {
return nil, err
}
ts, err := mount.TarStream()
if err != nil {
return nil, err
}
layer, err := ls.Register(ts, parent)
if err != nil {
return nil, err
}
if err := ls.Unmount(containerID); err != nil {
return nil, err
}
if _, err := ls.DeleteMount(containerID); err != nil {
return nil, err
}
return layer, nil
}
type FileApplier interface {
ApplyFile(root string) error
}
type testFile struct {
name string
content []byte
permission os.FileMode
}
func newTestFile(name string, content []byte, perm os.FileMode) FileApplier {
return &testFile{
name: name,
content: content,
permission: perm,
}
}
func (tf *testFile) ApplyFile(root string) error {
fullPath := filepath.Join(root, tf.name)
if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil {
return err
}
// Check if already exists
if stat, err := os.Stat(fullPath); err == nil && stat.Mode().Perm() != tf.permission {
if err := os.Chmod(fullPath, tf.permission); err != nil {
return err
}
}
if err := ioutil.WriteFile(fullPath, tf.content, tf.permission); err != nil {
return err
}
return nil
}
func initWithFiles(files ...FileApplier) layerInit {
return func(root string) error {
for _, f := range files {
if err := f.ApplyFile(root); err != nil {
return err
}
}
return nil
}
}
func getCachedLayer(l Layer) *roLayer {
if rl, ok := l.(*referencedCacheLayer); ok {
return rl.roLayer
}
return l.(*roLayer)
}
func createMetadata(layers ...Layer) []Metadata {
metadata := make([]Metadata, len(layers))
for i := range layers {
size, err := layers[i].Size()
if err != nil {
panic(err)
}
metadata[i].ChainID = layers[i].ChainID()
metadata[i].DiffID = layers[i].DiffID()
metadata[i].Size = size
metadata[i].DiffSize = getCachedLayer(layers[i]).size
}
return metadata
}
func assertMetadata(t *testing.T, metadata, expectedMetadata []Metadata) {
if len(metadata) != len(expectedMetadata) {
t.Fatalf("Unexpected number of deletes %d, expected %d", len(metadata), len(expectedMetadata))
}
for i := range metadata {
if metadata[i] != expectedMetadata[i] {
t.Errorf("Unexpected metadata\n\tExpected: %#v\n\tActual: %#v", expectedMetadata[i], metadata[i])
}
}
if t.Failed() {
t.FailNow()
}
}
func releaseAndCheckDeleted(t *testing.T, ls Store, layer Layer, removed ...Layer) {
layerCount := len(ls.(*layerStore).layerMap)
expectedMetadata := createMetadata(removed...)
metadata, err := ls.Release(layer)
if err != nil {
t.Fatal(err)
}
assertMetadata(t, metadata, expectedMetadata)
if expected := layerCount - len(removed); len(ls.(*layerStore).layerMap) != expected {
t.Fatalf("Unexpected number of layers %d, expected %d", len(ls.(*layerStore).layerMap), expected)
}
}
func cacheID(l Layer) string {
return getCachedLayer(l).cacheID
}
func assertLayerEqual(t *testing.T, l1, l2 Layer) {
if l1.ChainID() != l2.ChainID() {
t.Fatalf("Mismatched ID: %s vs %s", l1.ChainID(), l2.ChainID())
}
if l1.DiffID() != l2.DiffID() {
t.Fatalf("Mismatched DiffID: %s vs %s", l1.DiffID(), l2.DiffID())
}
size1, err := l1.Size()
if err != nil {
t.Fatal(err)
}
size2, err := l2.Size()
if err != nil {
t.Fatal(err)
}
if size1 != size2 {
t.Fatalf("Mismatched size: %d vs %d", size1, size2)
}
if cacheID(l1) != cacheID(l2) {
t.Fatalf("Mismatched cache id: %s vs %s", cacheID(l1), cacheID(l2))
}
p1 := l1.Parent()
p2 := l2.Parent()
if p1 != nil && p2 != nil {
assertLayerEqual(t, p1, p2)
} else if p1 != nil || p2 != nil {
t.Fatalf("Mismatched parents: %v vs %v", p1, p2)
}
}
func TestMountAndRegister(t *testing.T) {
ls, cleanup := newTestStore(t)
defer cleanup()
li := initWithFiles(newTestFile("testfile.txt", []byte("some test data"), 0644))
layer, err := createLayer(ls, "", li)
if err != nil {
t.Fatal(err)
}
size, _ := layer.Size()
t.Logf("Layer size: %d", size)
mount2, err := ls.Mount("new-test-mount", layer.ChainID(), "", nil)
if err != nil {
t.Fatal(err)
}
path2, err := mount2.Path()
if err != nil {
t.Fatal(err)
}
b, err := ioutil.ReadFile(filepath.Join(path2, "testfile.txt"))
if err != nil {
t.Fatal(err)
}
if expected := "some test data"; string(b) != expected {
t.Fatalf("Wrong file data, expected %q, got %q", expected, string(b))
}
if err := ls.Unmount("new-test-mount"); err != nil {
t.Fatal(err)
}
if _, err := ls.DeleteMount("new-test-mount"); err != nil {
t.Fatal(err)
}
}
func TestLayerRelease(t *testing.T) {
ls, cleanup := newTestStore(t)
defer cleanup()
layer1, err := createLayer(ls, "", initWithFiles(newTestFile("layer1.txt", []byte("layer 1 file"), 0644)))
if err != nil {
t.Fatal(err)
}
layer2, err := createLayer(ls, layer1.ChainID(), initWithFiles(newTestFile("layer2.txt", []byte("layer 2 file"), 0644)))
if err != nil {
t.Fatal(err)
}
if _, err := ls.Release(layer1); err != nil {
t.Fatal(err)
}
layer3a, err := createLayer(ls, layer2.ChainID(), initWithFiles(newTestFile("layer3.txt", []byte("layer 3a file"), 0644)))
if err != nil {
t.Fatal(err)
}
layer3b, err := createLayer(ls, layer2.ChainID(), initWithFiles(newTestFile("layer3.txt", []byte("layer 3b file"), 0644)))
if err != nil {
t.Fatal(err)
}
if _, err := ls.Release(layer2); err != nil {
t.Fatal(err)
}
t.Logf("Layer1: %s", layer1.ChainID())
t.Logf("Layer2: %s", layer2.ChainID())
t.Logf("Layer3a: %s", layer3a.ChainID())
t.Logf("Layer3b: %s", layer3b.ChainID())
if expected := 4; len(ls.(*layerStore).layerMap) != expected {
t.Fatalf("Unexpected number of layers %d, expected %d", len(ls.(*layerStore).layerMap), expected)
}
releaseAndCheckDeleted(t, ls, layer3b, layer3b)
releaseAndCheckDeleted(t, ls, layer3a, layer3a, layer2, layer1)
}
func TestStoreRestore(t *testing.T) {
ls, cleanup := newTestStore(t)
defer cleanup()
layer1, err := createLayer(ls, "", initWithFiles(newTestFile("layer1.txt", []byte("layer 1 file"), 0644)))
if err != nil {
t.Fatal(err)
}
layer2, err := createLayer(ls, layer1.ChainID(), initWithFiles(newTestFile("layer2.txt", []byte("layer 2 file"), 0644)))
if err != nil {
t.Fatal(err)
}
if _, err := ls.Release(layer1); err != nil {
t.Fatal(err)
}
layer3, err := createLayer(ls, layer2.ChainID(), initWithFiles(newTestFile("layer3.txt", []byte("layer 3 file"), 0644)))
if err != nil {
t.Fatal(err)
}
if _, err := ls.Release(layer2); err != nil {
t.Fatal(err)
}
m, err := ls.Mount("some-mount_name", layer3.ChainID(), "", nil)
if err != nil {
t.Fatal(err)
}
path, err := m.Path()
if err != nil {
t.Fatal(err)
}
if err := ioutil.WriteFile(filepath.Join(path, "testfile.txt"), []byte("nothing here"), 0644); err != nil {
t.Fatal(err)
}
if err := ls.Unmount("some-mount_name"); err != nil {
t.Fatal(err)
}
ls2, err := NewStore(ls.(*layerStore).store, ls.(*layerStore).driver)
if err != nil {
t.Fatal(err)
}
layer3b, err := ls2.Get(layer3.ChainID())
if err != nil {
t.Fatal(err)
}
assertLayerEqual(t, layer3b, layer3)
// Mount again with same name, should already be loaded
m2, err := ls2.Mount("some-mount_name", layer3b.ChainID(), "", nil)
if err != nil {
t.Fatal(err)
}
path2, err := m2.Path()
if err != nil {
t.Fatal(err)
}
b, err := ioutil.ReadFile(filepath.Join(path2, "testfile.txt"))
if err != nil {
t.Fatal(err)
}
if expected := "nothing here"; string(b) != expected {
t.Fatalf("Unexpected content %q, expected %q", string(b), expected)
}
if err := ls2.Unmount("some-mount_name"); err != nil {
t.Fatal(err)
}
if metadata, err := ls2.DeleteMount("some-mount_name"); err != nil {
t.Fatal(err)
} else if len(metadata) != 0 {
t.Fatalf("Unexpectedly deleted layers: %#v", metadata)
}
releaseAndCheckDeleted(t, ls2, layer3b, layer3, layer2, layer1)
}
func TestTarStreamStability(t *testing.T) {
ls, cleanup := newTestStore(t)
defer cleanup()
files1 := []FileApplier{
newTestFile("/etc/hosts", []byte("mydomain 10.0.0.1"), 0644),
newTestFile("/etc/profile", []byte("PATH=/usr/bin"), 0644),
}
addedFile := newTestFile("/etc/shadow", []byte("root:::::::"), 0644)
files2 := []FileApplier{
newTestFile("/etc/hosts", []byte("mydomain 10.0.0.2"), 0644),
newTestFile("/etc/profile", []byte("PATH=/usr/bin"), 0664),
newTestFile("/root/.bashrc", []byte("PATH=/usr/sbin:/usr/bin"), 0644),
}
tar1, err := tarFromFiles(files1...)
if err != nil {
t.Fatal(err)
}
tar2, err := tarFromFiles(files2...)
if err != nil {
t.Fatal(err)
}
layer1, err := ls.Register(bytes.NewReader(tar1), "")
if err != nil {
t.Fatal(err)
}
// hack layer to add file
p, err := ls.(*layerStore).driver.Get(layer1.(*referencedCacheLayer).cacheID, "")
if err != nil {
t.Fatal(err)
}
if err := addedFile.ApplyFile(p); err != nil {
t.Fatal(err)
}
if err := ls.(*layerStore).driver.Put(layer1.(*referencedCacheLayer).cacheID); err != nil {
t.Fatal(err)
}
layer2, err := ls.Register(bytes.NewReader(tar2), layer1.ChainID())
if err != nil {
t.Fatal(err)
}
id1 := layer1.ChainID()
t.Logf("Layer 1: %s", layer1.ChainID())
t.Logf("Layer 2: %s", layer2.ChainID())
if _, err := ls.Release(layer1); err != nil {
t.Fatal(err)
}
assertLayerDiff(t, tar2, layer2)
layer1b, err := ls.Get(id1)
if err != nil {
t.Logf("Content of layer map: %#v", ls.(*layerStore).layerMap)
t.Fatal(err)
}
if _, err := ls.Release(layer2); err != nil {
t.Fatal(err)
}
assertLayerDiff(t, tar1, layer1b)
if _, err := ls.Release(layer1b); err != nil {
t.Fatal(err)
}
}
func assertLayerDiff(t *testing.T, expected []byte, layer Layer) {
expectedDigest, err := digest.FromBytes(expected)
if err != nil {
t.Fatal(err)
}
if digest.Digest(layer.DiffID()) != expectedDigest {
t.Fatalf("Mismatched diff id for %s, got %s, expected %s", layer.ChainID(), layer.DiffID(), expected)
}
ts, err := layer.TarStream()
if err != nil {
t.Fatal(err)
}
actual, err := ioutil.ReadAll(ts)
if err != nil {
t.Fatal(err)
}
if len(actual) != len(expected) {
logByteDiff(t, actual, expected)
t.Fatalf("Mismatched tar stream size for %s, got %d, expected %d", layer.ChainID(), len(actual), len(expected))
}
actualDigest, err := digest.FromBytes(actual)
if err != nil {
t.Fatal(err)
}
if actualDigest != expectedDigest {
logByteDiff(t, actual, expected)
t.Fatalf("Wrong digest of tar stream, got %s, expected %s", actualDigest, expectedDigest)
}
}
const maxByteLog = 4 * 1024
func logByteDiff(t *testing.T, actual, expected []byte) {
d1, d2 := byteDiff(actual, expected)
if len(d1) == 0 && len(d2) == 0 {
return
}
prefix := len(actual) - len(d1)
if len(d1) > maxByteLog || len(d2) > maxByteLog {
t.Logf("Byte diff after %d matching bytes", prefix)
} else {
t.Logf("Byte diff after %d matching bytes\nActual bytes after prefix:\n%x\nExpected bytes after prefix:\n%x", prefix, d1, d2)
}
}
// byteDiff returns the differing bytes after the matching prefix
func byteDiff(b1, b2 []byte) ([]byte, []byte) {
i := 0
for i < len(b1) && i < len(b2) {
if b1[i] != b2[i] {
break
}
i++
}
return b1[i:], b2[i:]
}
func tarFromFiles(files ...FileApplier) ([]byte, error) {
td, err := ioutil.TempDir("", "tar-")
if err != nil {
return nil, err
}
defer os.RemoveAll(td)
for _, f := range files {
if err := f.ApplyFile(td); err != nil {
return nil, err
}
}
r, err := archive.Tar(td, archive.Uncompressed)
if err != nil {
return nil, err
}
buf := bytes.NewBuffer(nil)
if _, err := io.Copy(buf, r); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// assertReferences asserts that all the references are to the same
// image and represent the full set of references to that image.
func assertReferences(t *testing.T, references ...Layer) {
if len(references) == 0 {
return
}
base := references[0].(*referencedCacheLayer).roLayer
seenReferences := map[Layer]struct{}{
references[0]: {},
}
for i := 1; i < len(references); i++ {
other := references[i].(*referencedCacheLayer).roLayer
if base != other {
t.Fatalf("Unexpected referenced cache layer %s, expecting %s", other.ChainID(), base.ChainID())
}
if _, ok := base.references[references[i]]; !ok {
t.Fatalf("Reference not part of reference list: %v", references[i])
}
if _, ok := seenReferences[references[i]]; ok {
t.Fatalf("Duplicated reference %v", references[i])
}
}
if rc := len(base.references); rc != len(references) {
t.Fatalf("Unexpected number of references %d, expecting %d", rc, len(references))
}
}
func TestRegisterExistingLayer(t *testing.T) {
ls, cleanup := newTestStore(t)
defer cleanup()
baseFiles := []FileApplier{
newTestFile("/etc/profile", []byte("# Base configuration"), 0644),
}
layerFiles := []FileApplier{
newTestFile("/root/.bashrc", []byte("# Root configuration"), 0644),
}
li := initWithFiles(baseFiles...)
layer1, err := createLayer(ls, "", li)
if err != nil {
t.Fatal(err)
}
tar1, err := tarFromFiles(layerFiles...)
if err != nil {
t.Fatal(err)
}
layer2a, err := ls.Register(bytes.NewReader(tar1), layer1.ChainID())
if err != nil {
t.Fatal(err)
}
layer2b, err := ls.Register(bytes.NewReader(tar1), layer1.ChainID())
if err != nil {
t.Fatal(err)
}
assertReferences(t, layer2a, layer2b)
}
func graphDiffSize(ls Store, l Layer) (int64, error) {
cl := getCachedLayer(l)
var parent string
if cl.parent != nil {
parent = cl.parent.cacheID
}
return ls.(*layerStore).driver.DiffSize(cl.cacheID, parent)
}
func TestLayerSize(t *testing.T) {
ls, cleanup := newTestStore(t)
defer cleanup()
content1 := []byte("Base contents")
content2 := []byte("Added contents")
layer1, err := createLayer(ls, "", initWithFiles(newTestFile("file1", content1, 0644)))
if err != nil {
t.Fatal(err)
}
layer2, err := createLayer(ls, layer1.ChainID(), initWithFiles(newTestFile("file2", content2, 0644)))
if err != nil {
t.Fatal(err)
}
layer1DiffSize, err := graphDiffSize(ls, layer1)
if err != nil {
t.Fatal(err)
}
if int(layer1DiffSize) != len(content1) {
t.Fatalf("Unexpected diff size %d, expected %d", layer1DiffSize, len(content1))
}
layer1Size, err := layer1.Size()
if err != nil {
t.Fatal(err)
}
if expected := len(content1); int(layer1Size) != expected {
t.Fatalf("Unexpected size %d, expected %d", layer1Size, expected)
}
layer2DiffSize, err := graphDiffSize(ls, layer2)
if err != nil {
t.Fatal(err)
}
if int(layer2DiffSize) != len(content2) {
t.Fatalf("Unexpected diff size %d, expected %d", layer2DiffSize, len(content2))
}
layer2Size, err := layer2.Size()
if err != nil {
t.Fatal(err)
}
if expected := len(content1) + len(content2); int(layer2Size) != expected {
t.Fatalf("Unexpected size %d, expected %d", layer2Size, expected)
}
}

109
layer/layer_windows.go Normal file
View file

@ -0,0 +1,109 @@
package layer
import (
"errors"
"fmt"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/digest"
)
// GetLayerPath returns the path to a layer
func GetLayerPath(s Store, layer ChainID) (string, error) {
ls, ok := s.(*layerStore)
if !ok {
return "", errors.New("unsupported layer store")
}
ls.layerL.Lock()
defer ls.layerL.Unlock()
rl, ok := ls.layerMap[layer]
if !ok {
return "", ErrLayerDoesNotExist
}
path, err := ls.driver.Get(rl.cacheID, "")
if err != nil {
return "", err
}
if err := ls.driver.Put(rl.cacheID); err != nil {
return "", err
}
return path, nil
}
// RWLayerMetadata returns the graph metadata for the provided
// mount name.
func RWLayerMetadata(s Store, name string) (map[string]string, error) {
ls, ok := s.(*layerStore)
if !ok {
return nil, errors.New("unsupported layer store")
}
ls.mountL.Lock()
defer ls.mountL.Unlock()
ml, ok := ls.mounts[name]
if !ok {
return nil, errors.New("mount does not exist")
}
return ls.driver.GetMetadata(ml.mountID)
}
func (ls *layerStore) RegisterDiffID(graphID string, size int64) (Layer, error) {
var err error // this is used for cleanup in existingLayer case
diffID, err := digest.FromBytes([]byte(graphID))
if err != nil {
return nil, err
}
// Create new roLayer
layer := &roLayer{
cacheID: graphID,
diffID: DiffID(diffID),
referenceCount: 1,
layerStore: ls,
references: map[Layer]struct{}{},
size: size,
}
tx, err := ls.store.StartTransaction()
if err != nil {
return nil, err
}
defer func() {
if err != nil {
if err := tx.Cancel(); err != nil {
logrus.Errorf("Error canceling metadata transaction %q: %s", tx.String(), err)
}
}
}()
layer.chainID = createChainIDFromParent("", layer.diffID)
if !ls.driver.Exists(layer.cacheID) {
return nil, fmt.Errorf("layer %q is unknown to driver", layer.cacheID)
}
if err = storeLayer(tx, layer); err != nil {
return nil, err
}
ls.layerL.Lock()
defer ls.layerL.Unlock()
if existingLayer := ls.getAndRetainLayer(layer.chainID); existingLayer != nil {
// Set error for cleanup, but do not return
err = errors.New("layer already exists")
return existingLayer.getReference(), nil
}
if err = tx.Commit(layer.chainID); err != nil {
return nil, err
}
ls.layerMap[layer.chainID] = layer
return layer.getReference(), nil
}

251
layer/migration.go Normal file
View file

@ -0,0 +1,251 @@
package layer
import (
"compress/gzip"
"errors"
"fmt"
"io"
"os"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/digest"
"github.com/docker/docker/pkg/ioutils"
"github.com/vbatts/tar-split/tar/asm"
"github.com/vbatts/tar-split/tar/storage"
)
func (ls *layerStore) MountByGraphID(name string, graphID string, parent ChainID) (l RWLayer, err error) {
ls.mountL.Lock()
defer ls.mountL.Unlock()
m, ok := ls.mounts[name]
if ok {
if m.parent.chainID != parent {
return nil, errors.New("name conflict, mismatched parent")
}
if m.mountID != graphID {
return nil, errors.New("mount already exists")
}
return m, nil
}
if !ls.driver.Exists(graphID) {
return nil, errors.New("graph ID does not exist")
}
var p *roLayer
if string(parent) != "" {
ls.layerL.Lock()
p = ls.getAndRetainLayer(parent)
ls.layerL.Unlock()
if p == nil {
return nil, ErrLayerDoesNotExist
}
// Release parent chain if error
defer func() {
if err != nil {
ls.layerL.Lock()
ls.releaseLayer(p)
ls.layerL.Unlock()
}
}()
}
// TODO: Ensure graphID has correct parent
m = &mountedLayer{
name: name,
parent: p,
mountID: graphID,
layerStore: ls,
}
// Check for existing init layer
initID := fmt.Sprintf("%s-init", graphID)
if ls.driver.Exists(initID) {
m.initID = initID
}
if err = ls.saveMount(m); err != nil {
return nil, err
}
// TODO: provide a mount label
if err = ls.mount(m, ""); err != nil {
return nil, err
}
return m, nil
}
func (ls *layerStore) migrateLayer(tx MetadataTransaction, tarDataFile string, layer *roLayer) error {
var ar io.Reader
var tdf *os.File
var err error
if tarDataFile != "" {
tdf, err = os.Open(tarDataFile)
if err != nil {
if !os.IsNotExist(err) {
return err
}
tdf = nil
}
defer tdf.Close()
}
if tdf != nil {
tsw, err := tx.TarSplitWriter()
if err != nil {
return err
}
defer tsw.Close()
uncompressed, err := gzip.NewReader(tdf)
if err != nil {
return err
}
defer uncompressed.Close()
tr := io.TeeReader(uncompressed, tsw)
trc := ioutils.NewReadCloserWrapper(tr, uncompressed.Close)
ar, err = ls.assembleTar(layer.cacheID, trc, &layer.size)
if err != nil {
return err
}
} else {
var graphParent string
if layer.parent != nil {
graphParent = layer.parent.cacheID
}
archiver, err := ls.driver.Diff(layer.cacheID, graphParent)
if err != nil {
return err
}
defer archiver.Close()
tsw, err := tx.TarSplitWriter()
if err != nil {
return err
}
metaPacker := storage.NewJSONPacker(tsw)
packerCounter := &packSizeCounter{metaPacker, &layer.size}
defer tsw.Close()
ar, err = asm.NewInputTarStream(archiver, packerCounter, nil)
if err != nil {
return err
}
}
digester := digest.Canonical.New()
_, err = io.Copy(digester.Hash(), ar)
if err != nil {
return err
}
layer.diffID = DiffID(digester.Digest())
return nil
}
func (ls *layerStore) RegisterByGraphID(graphID string, parent ChainID, tarDataFile string) (Layer, error) {
// err is used to hold the error which will always trigger
// cleanup of creates sources but may not be an error returned
// to the caller (already exists).
var err error
var p *roLayer
if string(parent) != "" {
p = ls.get(parent)
if p == nil {
return nil, ErrLayerDoesNotExist
}
// Release parent chain if error
defer func() {
if err != nil {
ls.layerL.Lock()
ls.releaseLayer(p)
ls.layerL.Unlock()
}
}()
}
// Create new roLayer
layer := &roLayer{
parent: p,
cacheID: graphID,
referenceCount: 1,
layerStore: ls,
references: map[Layer]struct{}{},
}
tx, err := ls.store.StartTransaction()
if err != nil {
return nil, err
}
defer func() {
if err != nil {
logrus.Debugf("Cleaning up transaction after failed migration for %s: %v", graphID, err)
if err := tx.Cancel(); err != nil {
logrus.Errorf("Error canceling metadata transaction %q: %s", tx.String(), err)
}
}
}()
if err = ls.migrateLayer(tx, tarDataFile, layer); err != nil {
return nil, err
}
layer.chainID = createChainIDFromParent(parent, layer.diffID)
if err = storeLayer(tx, layer); err != nil {
return nil, err
}
ls.layerL.Lock()
defer ls.layerL.Unlock()
if existingLayer := ls.getAndRetainLayer(layer.chainID); existingLayer != nil {
// Set error for cleanup, but do not return
err = errors.New("layer already exists")
return existingLayer.getReference(), nil
}
if err = tx.Commit(layer.chainID); err != nil {
return nil, err
}
ls.layerMap[layer.chainID] = layer
return layer.getReference(), nil
}
type unpackSizeCounter struct {
unpacker storage.Unpacker
size *int64
}
func (u *unpackSizeCounter) Next() (*storage.Entry, error) {
e, err := u.unpacker.Next()
if err == nil && u.size != nil {
*u.size += e.Size
}
return e, err
}
type packSizeCounter struct {
packer storage.Packer
size *int64
}
func (p *packSizeCounter) AddEntry(e storage.Entry) (int, error) {
n, err := p.packer.AddEntry(e)
if err == nil && p.size != nil {
*p.size += e.Size
}
return n, err
}

385
layer/migration_test.go Normal file
View file

@ -0,0 +1,385 @@
package layer
import (
"bytes"
"compress/gzip"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/docker/docker/daemon/graphdriver"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/stringid"
"github.com/vbatts/tar-split/tar/asm"
"github.com/vbatts/tar-split/tar/storage"
)
func writeTarSplitFile(name string, tarContent []byte) error {
f, err := os.OpenFile(name, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return err
}
defer f.Close()
fz := gzip.NewWriter(f)
metaPacker := storage.NewJSONPacker(fz)
defer fz.Close()
rdr, err := asm.NewInputTarStream(bytes.NewReader(tarContent), metaPacker, nil)
if err != nil {
return err
}
if _, err := io.Copy(ioutil.Discard, rdr); err != nil {
return err
}
return nil
}
func TestLayerMigration(t *testing.T) {
td, err := ioutil.TempDir("", "migration-test-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(td)
layer1Files := []FileApplier{
newTestFile("/root/.bashrc", []byte("# Boring configuration"), 0644),
newTestFile("/etc/profile", []byte("# Base configuration"), 0644),
}
layer2Files := []FileApplier{
newTestFile("/root/.bashrc", []byte("# Updated configuration"), 0644),
}
tar1, err := tarFromFiles(layer1Files...)
if err != nil {
t.Fatal(err)
}
tar2, err := tarFromFiles(layer2Files...)
if err != nil {
t.Fatal(err)
}
graph, err := newVFSGraphDriver(filepath.Join(td, "graphdriver-"))
if err != nil {
t.Fatal(err)
}
graphID1 := stringid.GenerateRandomID()
if err := graph.Create(graphID1, "", ""); err != nil {
t.Fatal(err)
}
if _, err := graph.ApplyDiff(graphID1, "", archive.Reader(bytes.NewReader(tar1))); err != nil {
t.Fatal(err)
}
tf1 := filepath.Join(td, "tar1.json.gz")
if err := writeTarSplitFile(tf1, tar1); err != nil {
t.Fatal(err)
}
fms, err := NewFSMetadataStore(filepath.Join(td, "layers"))
if err != nil {
t.Fatal(err)
}
ls, err := NewStore(fms, graph)
if err != nil {
t.Fatal(err)
}
layer1a, err := ls.(*layerStore).RegisterByGraphID(graphID1, "", tf1)
if err != nil {
t.Fatal(err)
}
layer1b, err := ls.Register(bytes.NewReader(tar1), "")
if err != nil {
t.Fatal(err)
}
assertReferences(t, layer1a, layer1b)
// Attempt register, should be same
layer2a, err := ls.Register(bytes.NewReader(tar2), layer1a.ChainID())
if err != nil {
t.Fatal(err)
}
graphID2 := stringid.GenerateRandomID()
if err := graph.Create(graphID2, graphID1, ""); err != nil {
t.Fatal(err)
}
if _, err := graph.ApplyDiff(graphID2, graphID1, archive.Reader(bytes.NewReader(tar2))); err != nil {
t.Fatal(err)
}
tf2 := filepath.Join(td, "tar2.json.gz")
if err := writeTarSplitFile(tf2, tar2); err != nil {
t.Fatal(err)
}
layer2b, err := ls.(*layerStore).RegisterByGraphID(graphID2, layer1a.ChainID(), tf2)
if err != nil {
t.Fatal(err)
}
assertReferences(t, layer2a, layer2b)
if metadata, err := ls.Release(layer2a); err != nil {
t.Fatal(err)
} else if len(metadata) > 0 {
t.Fatalf("Unexpected layer removal after first release: %#v", metadata)
}
metadata, err := ls.Release(layer2b)
if err != nil {
t.Fatal(err)
}
assertMetadata(t, metadata, createMetadata(layer2a))
}
func tarFromFilesInGraph(graph graphdriver.Driver, graphID, parentID string, files ...FileApplier) ([]byte, error) {
t, err := tarFromFiles(files...)
if err != nil {
return nil, err
}
if err := graph.Create(graphID, parentID, ""); err != nil {
return nil, err
}
if _, err := graph.ApplyDiff(graphID, parentID, archive.Reader(bytes.NewReader(t))); err != nil {
return nil, err
}
ar, err := graph.Diff(graphID, parentID)
if err != nil {
return nil, err
}
defer ar.Close()
return ioutil.ReadAll(ar)
}
func TestLayerMigrationNoTarsplit(t *testing.T) {
td, err := ioutil.TempDir("", "migration-test-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(td)
layer1Files := []FileApplier{
newTestFile("/root/.bashrc", []byte("# Boring configuration"), 0644),
newTestFile("/etc/profile", []byte("# Base configuration"), 0644),
}
layer2Files := []FileApplier{
newTestFile("/root/.bashrc", []byte("# Updated configuration"), 0644),
}
graph, err := newVFSGraphDriver(filepath.Join(td, "graphdriver-"))
if err != nil {
t.Fatal(err)
}
graphID1 := stringid.GenerateRandomID()
graphID2 := stringid.GenerateRandomID()
tar1, err := tarFromFilesInGraph(graph, graphID1, "", layer1Files...)
if err != nil {
t.Fatal(err)
}
tar2, err := tarFromFilesInGraph(graph, graphID2, graphID1, layer2Files...)
if err != nil {
t.Fatal(err)
}
fms, err := NewFSMetadataStore(filepath.Join(td, "layers"))
if err != nil {
t.Fatal(err)
}
ls, err := NewStore(fms, graph)
if err != nil {
t.Fatal(err)
}
layer1a, err := ls.(*layerStore).RegisterByGraphID(graphID1, "", "")
if err != nil {
t.Fatal(err)
}
layer1b, err := ls.Register(bytes.NewReader(tar1), "")
if err != nil {
t.Fatal(err)
}
assertReferences(t, layer1a, layer1b)
// Attempt register, should be same
layer2a, err := ls.Register(bytes.NewReader(tar2), layer1a.ChainID())
if err != nil {
t.Fatal(err)
}
layer2b, err := ls.(*layerStore).RegisterByGraphID(graphID2, layer1a.ChainID(), "")
if err != nil {
t.Fatal(err)
}
assertReferences(t, layer2a, layer2b)
if metadata, err := ls.Release(layer2a); err != nil {
t.Fatal(err)
} else if len(metadata) > 0 {
t.Fatalf("Unexpected layer removal after first release: %#v", metadata)
}
metadata, err := ls.Release(layer2b)
if err != nil {
t.Fatal(err)
}
assertMetadata(t, metadata, createMetadata(layer2a))
}
func TestMountMigration(t *testing.T) {
ls, cleanup := newTestStore(t)
defer cleanup()
baseFiles := []FileApplier{
newTestFile("/root/.bashrc", []byte("# Boring configuration"), 0644),
newTestFile("/etc/profile", []byte("# Base configuration"), 0644),
}
initFiles := []FileApplier{
newTestFile("/etc/hosts", []byte{}, 0644),
newTestFile("/etc/resolv.conf", []byte{}, 0644),
}
mountFiles := []FileApplier{
newTestFile("/etc/hosts", []byte("localhost 127.0.0.1"), 0644),
newTestFile("/root/.bashrc", []byte("# Updated configuration"), 0644),
newTestFile("/root/testfile1.txt", []byte("nothing valuable"), 0644),
}
initTar, err := tarFromFiles(initFiles...)
if err != nil {
t.Fatal(err)
}
mountTar, err := tarFromFiles(mountFiles...)
if err != nil {
t.Fatal(err)
}
graph := ls.(*layerStore).driver
layer1, err := createLayer(ls, "", initWithFiles(baseFiles...))
if err != nil {
t.Fatal(err)
}
graphID1 := layer1.(*referencedCacheLayer).cacheID
containerID := stringid.GenerateRandomID()
containerInit := fmt.Sprintf("%s-init", containerID)
if err := graph.Create(containerInit, graphID1, ""); err != nil {
t.Fatal(err)
}
if _, err := graph.ApplyDiff(containerInit, graphID1, archive.Reader(bytes.NewReader(initTar))); err != nil {
t.Fatal(err)
}
if err := graph.Create(containerID, containerInit, ""); err != nil {
t.Fatal(err)
}
if _, err := graph.ApplyDiff(containerID, containerInit, archive.Reader(bytes.NewReader(mountTar))); err != nil {
t.Fatal(err)
}
rwLayer1, err := ls.(*layerStore).MountByGraphID("migration-mount", containerID, layer1.ChainID())
if err != nil {
t.Fatal(err)
}
changes, err := ls.Changes("migration-mount")
if err != nil {
t.Fatal(err)
}
if expected := 5; len(changes) != expected {
t.Logf("Changes %#v", changes)
t.Fatalf("Wrong number of changes %d, expected %d", len(changes), expected)
}
sortChanges(changes)
assertChange(t, changes[0], archive.Change{
Path: "/etc",
Kind: archive.ChangeModify,
})
assertChange(t, changes[1], archive.Change{
Path: "/etc/hosts",
Kind: archive.ChangeModify,
})
assertChange(t, changes[2], archive.Change{
Path: "/root",
Kind: archive.ChangeModify,
})
assertChange(t, changes[3], archive.Change{
Path: "/root/.bashrc",
Kind: archive.ChangeModify,
})
assertChange(t, changes[4], archive.Change{
Path: "/root/testfile1.txt",
Kind: archive.ChangeAdd,
})
if expectedCount := 1; rwLayer1.(*mountedLayer).activityCount != expectedCount {
t.Fatalf("Wrong activity count %d, expected %d", rwLayer1.(*mountedLayer).activityCount, expectedCount)
}
rwLayer2, err := ls.Mount("migration-mount", layer1.ChainID(), "", nil)
if err != nil {
t.Fatal(err)
}
if rwLayer1 != rwLayer2 {
t.Fatalf("Wrong rwlayer %v, expected %v", rwLayer2, rwLayer1)
}
if expectedCount := 2; rwLayer2.(*mountedLayer).activityCount != expectedCount {
t.Fatalf("Wrong activity count %d, expected %d", rwLayer2.(*mountedLayer).activityCount, expectedCount)
}
if metadata, err := ls.Release(layer1); err != nil {
t.Fatal(err)
} else if len(metadata) > 0 {
t.Fatalf("Expected no layers to be deleted, deleted %#v", metadata)
}
if err := ls.Unmount("migration-mount"); err != nil {
t.Fatal(err)
}
if _, err := ls.DeleteMount("migration-mount"); err == nil {
t.Fatal("Expected error deleting active mount")
}
if err := ls.Unmount("migration-mount"); err != nil {
t.Fatal(err)
}
metadata, err := ls.DeleteMount("migration-mount")
if err != nil {
t.Fatal(err)
}
if len(metadata) == 0 {
t.Fatal("Expected base layer to be deleted when deleting mount")
}
assertMetadata(t, metadata, createMetadata(layer1))
}

217
layer/mount_test.go Normal file
View file

@ -0,0 +1,217 @@
package layer
import (
"io/ioutil"
"os"
"path/filepath"
"sort"
"testing"
"github.com/docker/docker/pkg/archive"
)
func TestMountInit(t *testing.T) {
ls, cleanup := newTestStore(t)
defer cleanup()
basefile := newTestFile("testfile.txt", []byte("base data!"), 0644)
initfile := newTestFile("testfile.txt", []byte("init data!"), 0777)
li := initWithFiles(basefile)
layer, err := createLayer(ls, "", li)
if err != nil {
t.Fatal(err)
}
mountInit := func(root string) error {
return initfile.ApplyFile(root)
}
m, err := ls.Mount("fun-mount", layer.ChainID(), "", mountInit)
if err != nil {
t.Fatal(err)
}
path, err := m.Path()
if err != nil {
t.Fatal(err)
}
f, err := os.Open(filepath.Join(path, "testfile.txt"))
if err != nil {
t.Fatal(err)
}
defer f.Close()
fi, err := f.Stat()
if err != nil {
t.Fatal(err)
}
b, err := ioutil.ReadAll(f)
if err != nil {
t.Fatal(err)
}
if expected := "init data!"; string(b) != expected {
t.Fatalf("Unexpected test file contents %q, expected %q", string(b), expected)
}
if fi.Mode().Perm() != 0777 {
t.Fatalf("Unexpected filemode %o, expecting %o", fi.Mode().Perm(), 0777)
}
}
func TestMountSize(t *testing.T) {
ls, cleanup := newTestStore(t)
defer cleanup()
content1 := []byte("Base contents")
content2 := []byte("Mutable contents")
contentInit := []byte("why am I excluded from the size ☹")
li := initWithFiles(newTestFile("file1", content1, 0644))
layer, err := createLayer(ls, "", li)
if err != nil {
t.Fatal(err)
}
mountInit := func(root string) error {
return newTestFile("file-init", contentInit, 0777).ApplyFile(root)
}
m, err := ls.Mount("mount-size", layer.ChainID(), "", mountInit)
if err != nil {
t.Fatal(err)
}
path, err := m.Path()
if err != nil {
t.Fatal(err)
}
if err := ioutil.WriteFile(filepath.Join(path, "file2"), content2, 0755); err != nil {
t.Fatal(err)
}
mountSize, err := m.Size()
if err != nil {
t.Fatal(err)
}
if expected := len(content2); int(mountSize) != expected {
t.Fatalf("Unexpected mount size %d, expected %d", int(mountSize), expected)
}
}
func TestMountChanges(t *testing.T) {
ls, cleanup := newTestStore(t)
defer cleanup()
basefiles := []FileApplier{
newTestFile("testfile1.txt", []byte("base data!"), 0644),
newTestFile("testfile2.txt", []byte("base data!"), 0644),
newTestFile("testfile3.txt", []byte("base data!"), 0644),
}
initfile := newTestFile("testfile1.txt", []byte("init data!"), 0777)
li := initWithFiles(basefiles...)
layer, err := createLayer(ls, "", li)
if err != nil {
t.Fatal(err)
}
mountInit := func(root string) error {
return initfile.ApplyFile(root)
}
m, err := ls.Mount("mount-changes", layer.ChainID(), "", mountInit)
if err != nil {
t.Fatal(err)
}
path, err := m.Path()
if err != nil {
t.Fatal(err)
}
if err := os.Chmod(filepath.Join(path, "testfile1.txt"), 0755); err != nil {
t.Fatal(err)
}
if err := ioutil.WriteFile(filepath.Join(path, "testfile1.txt"), []byte("mount data!"), 0755); err != nil {
t.Fatal(err)
}
if err := os.Remove(filepath.Join(path, "testfile2.txt")); err != nil {
t.Fatal(err)
}
if err := os.Chmod(filepath.Join(path, "testfile3.txt"), 0755); err != nil {
t.Fatal(err)
}
if err := ioutil.WriteFile(filepath.Join(path, "testfile4.txt"), []byte("mount data!"), 0644); err != nil {
t.Fatal(err)
}
changes, err := ls.Changes("mount-changes")
if err != nil {
t.Fatal(err)
}
if expected := 4; len(changes) != expected {
t.Fatalf("Wrong number of changes %d, expected %d", len(changes), expected)
}
sortChanges(changes)
assertChange(t, changes[0], archive.Change{
Path: "/testfile1.txt",
Kind: archive.ChangeModify,
})
assertChange(t, changes[1], archive.Change{
Path: "/testfile2.txt",
Kind: archive.ChangeDelete,
})
assertChange(t, changes[2], archive.Change{
Path: "/testfile3.txt",
Kind: archive.ChangeModify,
})
assertChange(t, changes[3], archive.Change{
Path: "/testfile4.txt",
Kind: archive.ChangeAdd,
})
}
func assertChange(t *testing.T, actual, expected archive.Change) {
if actual.Path != expected.Path {
t.Fatalf("Unexpected change path %s, expected %s", actual.Path, expected.Path)
}
if actual.Kind != expected.Kind {
t.Fatalf("Unexpected change type %s, expected %s", actual.Kind, expected.Kind)
}
}
func sortChanges(changes []archive.Change) {
cs := &changeSorter{
changes: changes,
}
sort.Sort(cs)
}
type changeSorter struct {
changes []archive.Change
}
func (cs *changeSorter) Len() int {
return len(cs.changes)
}
func (cs *changeSorter) Swap(i, j int) {
cs.changes[i], cs.changes[j] = cs.changes[j], cs.changes[i]
}
func (cs *changeSorter) Less(i, j int) bool {
return cs.changes[i].Path < cs.changes[j].Path
}

64
layer/mounted_layer.go Normal file
View file

@ -0,0 +1,64 @@
package layer
import "io"
type mountedLayer struct {
name string
mountID string
initID string
parent *roLayer
path string
layerStore *layerStore
activityCount int
}
func (ml *mountedLayer) cacheParent() string {
if ml.initID != "" {
return ml.initID
}
if ml.parent != nil {
return ml.parent.cacheID
}
return ""
}
func (ml *mountedLayer) TarStream() (io.Reader, error) {
archiver, err := ml.layerStore.driver.Diff(ml.mountID, ml.cacheParent())
if err != nil {
return nil, err
}
return autoClosingReader{archiver}, nil
}
func (ml *mountedLayer) Path() (string, error) {
if ml.path == "" {
return "", ErrNotMounted
}
return ml.path, nil
}
func (ml *mountedLayer) Parent() Layer {
if ml.parent != nil {
return ml.parent
}
// Return a nil interface instead of an interface wrapping a nil
// pointer.
return nil
}
func (ml *mountedLayer) Size() (int64, error) {
return ml.layerStore.driver.DiffSize(ml.mountID, ml.cacheParent())
}
type autoClosingReader struct {
source io.ReadCloser
}
func (r autoClosingReader) Read(p []byte) (n int, err error) {
n, err = r.source.Read(p)
if err != nil {
r.source.Close()
}
return
}

110
layer/ro_layer.go Normal file
View file

@ -0,0 +1,110 @@
package layer
import "io"
type roLayer struct {
chainID ChainID
diffID DiffID
parent *roLayer
cacheID string
size int64
layerStore *layerStore
referenceCount int
references map[Layer]struct{}
}
func (rl *roLayer) TarStream() (io.Reader, error) {
r, err := rl.layerStore.store.TarSplitReader(rl.chainID)
if err != nil {
return nil, err
}
return rl.layerStore.assembleTar(rl.cacheID, r, nil)
}
func (rl *roLayer) ChainID() ChainID {
return rl.chainID
}
func (rl *roLayer) DiffID() DiffID {
return rl.diffID
}
func (rl *roLayer) Parent() Layer {
if rl.parent == nil {
return nil
}
return rl.parent
}
func (rl *roLayer) Size() (size int64, err error) {
if rl.parent != nil {
size, err = rl.parent.Size()
if err != nil {
return
}
}
return size + rl.size, nil
}
func (rl *roLayer) DiffSize() (size int64, err error) {
return rl.size, nil
}
func (rl *roLayer) Metadata() (map[string]string, error) {
return rl.layerStore.driver.GetMetadata(rl.cacheID)
}
type referencedCacheLayer struct {
*roLayer
}
func (rl *roLayer) getReference() Layer {
ref := &referencedCacheLayer{
roLayer: rl,
}
rl.references[ref] = struct{}{}
return ref
}
func (rl *roLayer) hasReference(ref Layer) bool {
_, ok := rl.references[ref]
return ok
}
func (rl *roLayer) hasReferences() bool {
return len(rl.references) > 0
}
func (rl *roLayer) deleteReference(ref Layer) {
delete(rl.references, ref)
}
func (rl *roLayer) depth() int {
if rl.parent == nil {
return 1
}
return rl.parent.depth() + 1
}
func storeLayer(tx MetadataTransaction, layer *roLayer) error {
if err := tx.SetDiffID(layer.diffID); err != nil {
return err
}
if err := tx.SetSize(layer.size); err != nil {
return err
}
if err := tx.SetCacheID(layer.cacheID); err != nil {
return err
}
if layer.parent != nil {
if err := tx.SetParent(layer.parent.chainID); err != nil {
return err
}
}
return nil
}