Allow drivers to implement ApplyDiff in Differ interface

archive/archive.go

@@ -167,7 +167,7 @@ func TarFilter(path string, options *TarOptions) (io.Reader, error) {
 // The archive may be compressed with one of the following algorithms:
 //  identity (uncompressed), gzip, bzip2, xz.
 // FIXME: specify behavior when target path exists vs. doesn't exist.
-func Untar(archive io.Reader, path string) error {
+func Untar(archive io.Reader, path string, options *TarOptions) error {
 	if archive == nil {
 		return fmt.Errorf("Empty archive")
 	}
@@ -188,8 +188,15 @@ func Untar(archive io.Reader, path string) error {
 	compression := DetectCompression(buf)
 
 	utils.Debugf("Archive compression detected: %s", compression.Extension())
+	args := []string{"--numeric-owner", "-f", "-", "-C", path, "-x" + compression.Flag()}
 
-	cmd := exec.Command("tar", "--numeric-owner", "-f", "-", "-C", path, "-x"+compression.Flag())
+	if options != nil {
+		for _, exclude := range options.Excludes {
+			args = append(args, fmt.Sprintf("--exclude=%s", exclude))
+		}
+	}
+
+	cmd := exec.Command("tar", args...)
 	cmd.Stdin = io.MultiReader(bytes.NewReader(buf), archive)
 	// Hardcode locale environment for predictable outcome regardless of host configuration.
 	//   (see https://github.com/dotcloud/docker/issues/355)
@@ -210,7 +217,7 @@ func TarUntar(src string, filter []string, dst string) error {
 	if err != nil {
 		return err
 	}
-	return Untar(archive, dst)
+	return Untar(archive, dst, nil)
 }
 
 // UntarPath is a convenience function which looks for an archive
@@ -218,7 +225,7 @@ func TarUntar(src string, filter []string, dst string) error {
 func UntarPath(src, dst string) error {
 	if archive, err := os.Open(src); err != nil {
 		return err
-	} else if err := Untar(archive, dst); err != nil {
+	} else if err := Untar(archive, dst, nil); err != nil {
 		return err
 	}
 	return nil
@@ -287,7 +294,7 @@ func CopyFileWithTar(src, dst string) error {
 		return err
 	}
 	tw.Close()
-	return Untar(buf, filepath.Dir(dst))
+	return Untar(buf, filepath.Dir(dst), nil)
 }
 
 // CmdStream executes a command, and returns its stdout as a stream.

archive/archive_test.go

@@ -83,7 +83,7 @@ func tarUntar(t *testing.T, origin string, compression Compression) error {
 		return err
 	}
 	defer os.RemoveAll(tmp)
-	if err := Untar(archive, tmp); err != nil {
+	if err := Untar(archive, tmp, nil); err != nil {
 		return err
 	}
 	if _, err := os.Stat(tmp); err != nil {

archive/diff.go

@@ -13,7 +13,7 @@ func ApplyLayer(dest string, layer Archive) error {
 	// Poor man's diff applyer in 2 steps:
 
 	// Step 1: untar everything in place
-	if err := Untar(layer, dest); err != nil {
+	if err := Untar(layer, dest, nil); err != nil {
 		return err
 	}
 

aufs/aufs.go

@@ -200,18 +200,18 @@ func (a *AufsDriver) Get(id string) (string, error) {
 
 // Returns an archive of the contents for the id
 func (a *AufsDriver) Diff(id string) (archive.Archive, error) {
-	// Exclude top level aufs metadata from the diff
-	return archive.TarFilter(
-		path.Join(a.rootPath(), "diff", id),
-		&archive.TarOptions{
-			Excludes:    []string{".wh*"},
-			Recursive:   true,
-			Compression: archive.Uncompressed,
-		})
+	return archive.TarFilter(path.Join(a.rootPath(), "diff", id), &archive.TarOptions{
+		Recursive:   true,
+		Compression: archive.Uncompressed,
+	})
+}
+
+func (a *AufsDriver) ApplyDiff(id string, diff archive.Archive) error {
+	return archive.Untar(diff, path.Join(a.rootPath(), "diff", id), nil)
 }
 
 // Returns the size of the contents for the id
-func (a *AufsDriver) DiffSize(id string) (int64, error) {
+func (a *AufsDriver) Size(id string) (int64, error) {
 	return utils.TreeSize(path.Join(a.rootPath(), "diff", id))
 }
 

buildfile.go

@@ -476,7 +476,7 @@ func (b *buildFile) Build(context io.Reader) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	if err := archive.Untar(context, name); err != nil {
+	if err := archive.Untar(context, name, nil); err != nil {
 		return "", err
 	}
 	defer os.RemoveAll(name)

commands.go

@@ -1911,7 +1911,7 @@ func (cli *DockerCli) CmdCp(args ...string) error {
 
 	if statusCode == 200 {
 		r := bytes.NewReader(data)
-		if err := archive.Untar(r, copyData.HostPath); err != nil {
+		if err := archive.Untar(r, copyData.HostPath, nil); err != nil {
 			return err
 		}
 	}

container.go

@@ -1485,7 +1485,7 @@ func (container *Container) GetSize() (int64, int64) {
 		driver             = container.runtime.driver
 	)
 
-	sizeRw, err = driver.DiffSize(container.ID)
+	sizeRw, err = driver.Size(container.ID)
 	if err != nil {
 		utils.Errorf("Warning: driver %s couldn't return diff size of container %s: %s", driver, container.ID, err)
 		// FIXME: GetSize should return an error. Not changing it now in case

container_test.go

@@ -1680,3 +1680,67 @@ func TestRestartGhost(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestRemoveFile(t *testing.T) {
+	runtime := mkRuntime(t)
+	defer nuke(runtime)
+
+	container1, _ := mkContainer(runtime, []string{"_", "/bin/sh", "-c", "touch test.txt"}, t)
+	defer runtime.Destroy(container1)
+
+	if container1.State.Running {
+		t.Errorf("Container shouldn't be running")
+	}
+	if err := container1.Run(); err != nil {
+		t.Fatal(err)
+	}
+	if container1.State.Running {
+		t.Errorf("Container shouldn't be running")
+	}
+
+	commit := func(container *Container) (*Image, error) {
+		rwTar, err := container.ExportRw()
+		if err != nil {
+			return nil, err
+		}
+		img, err := runtime.graph.Create(rwTar, container, "unit test commited image", "", nil)
+		if err != nil {
+			return nil, err
+		}
+		return img, nil
+	}
+
+	img, err := commit(container1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	container2, _ := mkContainer(runtime, []string{img.ID, "/bin/sh", "-c", "rm /test.txt"}, t)
+	defer runtime.Destroy(container2)
+
+	if err := container2.Run(); err != nil {
+		t.Fatal(err)
+	}
+
+	containerMount, err := runtime.driver.Get(container2.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(path.Join(containerMount, "test.txt")); err == nil {
+		t.Fatalf("test.txt should not exist")
+	}
+
+	img, err = commit(container2)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mountPoint, err := runtime.driver.Get(img.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	file := path.Join(mountPoint, "test.txt")
+	if _, err := os.Stat(file); err == nil {
+		t.Fatalf("The file %s should not exist\n", file)
+	}
+}

devmapper/driver.go

@@ -2,7 +2,6 @@ package devmapper
 
 import (
 	"fmt"
-	"github.com/dotcloud/docker/archive"
 	"github.com/dotcloud/docker/graphdriver"
 	"os"
 	"path"
@@ -57,18 +56,10 @@ func (d *Driver) Get(id string) (string, error) {
 	return mp, nil
 }
 
-func (d *Driver) DiffSize(id string) (int64, error) {
+func (d *Driver) Size(id string) (int64, error) {
 	return -1, fmt.Errorf("Not implemented")
 }
 
-func (d *Driver) Diff(id string) (archive.Archive, error) {
-	return nil, fmt.Errorf("Not implemented)")
-}
-
-func (d *Driver) Changes(id string) ([]archive.Change, error) {
-	return nil, fmt.Errorf("asdlfj)")
-}
-
 func (d *Driver) mount(id, mp string) error {
 	// Create the target directories if they don't exist
 	if err := os.MkdirAll(mp, 0755); err != nil && !os.IsExist(err) {

graph.go

@@ -151,6 +151,7 @@ func (graph *Graph) Register(jsonData []byte, layerData archive.Archive, img *Im
 	if err != nil {
 		return fmt.Errorf("Driver %s failed to get image rootfs %s: %s", graph.driver, img.ID, err)
 	}
+	img.graph = graph
 	if err := StoreImage(img, jsonData, layerData, tmp, rootfs); err != nil {
 		return err
 	}
@@ -158,7 +159,6 @@ func (graph *Graph) Register(jsonData []byte, layerData archive.Archive, img *Im
 	if err := os.Rename(tmp, graph.imageRoot(img.ID)); err != nil {
 		return err
 	}
-	img.graph = graph
 	graph.idIndex.Add(img.ID)
 	return nil
 }

graphdriver/driver.go

@@ -15,12 +15,15 @@ type Driver interface {
 	Remove(id string) error
 
 	Get(id string) (dir string, err error)
+	Size(id string) (bytes int64, err error)
 
-	DiffSize(id string) (bytes int64, err error)
+	Cleanup() error
+}
+
+type Differ interface {
 	Diff(id string) (archive.Archive, error)
 	Changes(id string) ([]archive.Change, error)
-
-	Cleanup() error
+	ApplyDiff(id string, diff archive.Archive) error
 }
 
 var (

graphdriver/dummy/driver.go

@@ -73,14 +73,6 @@ func (d *Driver) Get(id string) (string, error) {
 	return dir, nil
 }
 
-func (d *Driver) DiffSize(id string) (int64, error) {
+func (d *Driver) Size(id string) (int64, error) {
 	return -1, fmt.Errorf("Not implemented")
 }
-
-func (d *Driver) Diff(id string) (archive.Archive, error) {
-	return nil, fmt.Errorf("Not implemented)")
-}
-
-func (d *Driver) Changes(id string) ([]archive.Change, error) {
-	return nil, fmt.Errorf("asdlfj)")
-}

image.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/dotcloud/docker/archive"
+	"github.com/dotcloud/docker/graphdriver"
 	"github.com/dotcloud/docker/utils"
 	"io"
 	"io/ioutil"
@@ -70,12 +71,18 @@ func StoreImage(img *Image, jsonData []byte, layerData archive.Archive, root, ro
 
 	// If layerData is not nil, unpack it into the new layer
 	if layerData != nil {
-		start := time.Now()
-		utils.Debugf("Start untar layer")
-		if err := archive.Untar(layerData, layer); err != nil {
-			return err
+		if differ, ok := img.graph.driver.(graphdriver.Differ); ok {
+			if err := differ.ApplyDiff(img.ID, layerData); err != nil {
+				return err
+			}
+		} else {
+			start := time.Now()
+			utils.Debugf("Start untar layer")
+			if err := archive.ApplyLayer(layer, layerData); err != nil {
+				return err
+			}
+			utils.Debugf("Untar time: %vs", time.Now().Sub(start).Seconds())
 		}
-		utils.Debugf("Untar time: %vs", time.Now().Sub(start).Seconds())
 	}
 
 	// If raw json is provided, then use it

runtime.go

@@ -18,6 +18,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"sort"
 	"strings"
 	"time"
@@ -733,13 +734,54 @@ func (runtime *Runtime) Unmount(container *Container) error {
 }
 
 func (runtime *Runtime) Changes(container *Container) ([]archive.Change, error) {
-	// FIXME: Remove Changes method from runtime
-	return runtime.driver.Changes(container.ID)
+	if differ, ok := runtime.driver.(graphdriver.Differ); ok {
+		return differ.Changes(container.ID)
+	}
+	cDir, err := runtime.driver.Get(container.ID)
+	if err != nil {
+		return nil, fmt.Errorf("Error getting container rootfs %s from driver %s: %s", container.ID, container.runtime.driver, err)
+	}
+	initDir, err := runtime.driver.Get(container.ID + "-init")
+	if err != nil {
+		return nil, fmt.Errorf("Error getting container init rootfs %s from driver %s: %s", container.ID, container.runtime.driver, err)
+	}
+	return archive.ChangesDirs(cDir, initDir)
 }
 
 func (runtime *Runtime) Diff(container *Container) (archive.Archive, error) {
-	// FIXME: Remove Diff method from runtime
-	return runtime.driver.Diff(container.ID)
+	if differ, ok := runtime.driver.(graphdriver.Differ); ok {
+		return differ.Diff(container.ID)
+	}
+
+	changes, err := runtime.Changes(container)
+	if err != nil {
+		return nil, err
+	}
+
+	cDir, err := runtime.driver.Get(container.ID)
+	if err != nil {
+		return nil, fmt.Errorf("Error getting container rootfs %s from driver %s: %s", container.ID, container.runtime.driver, err)
+	}
+
+	files := make([]string, 0)
+	deletions := make([]string, 0)
+	for _, change := range changes {
+		if change.Kind == archive.ChangeModify || change.Kind == archive.ChangeAdd {
+			files = append(files, change.Path)
+		}
+		if change.Kind == archive.ChangeDelete {
+			base := filepath.Base(change.Path)
+			dir := filepath.Dir(change.Path)
+			deletions = append(deletions, filepath.Join(dir, ".wh."+base))
+		}
+	}
+	// FIXME: Why do we create whiteout files inside Tar code ?
+	return archive.TarFilter(cDir, &archive.TarOptions{
+		Compression: archive.Uncompressed,
+		Includes:    files,
+		Recursive:   false,
+		CreateFiles: deletions,
+	})
 }
 
 func linkLxcStart(root string) error {

runtime_test.go

@@ -143,7 +143,7 @@ func setupBaseImage() {
 	if img, err := runtime.repositories.LookupImage(unitTestImageName); err != nil || img.ID != unitTestImageID {
 		// Retrieve the Image
 		if err := srv.ImagePull(unitTestImageName, "", os.Stdout, utils.NewStreamFormatter(false), nil, nil, true); err != nil {
-			log.Fatalf("Unable to pull the test image:", err)
+			log.Fatalf("Unable to pull the test image: %s", err)
 		}
 	}
 }