Merge pull request #5780 from vbatts/vbatts-start_tarsum_test

tarsum: start a test for TarSum

utils/tarsum_test.go

@@ -0,0 +1,224 @@
+package utils
+
+import (
+	"bytes"
+	"crypto/rand"
+	"fmt"
+	"github.com/dotcloud/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar"
+	"io"
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+type testLayer struct {
+	filename string
+	options  *sizedOptions
+	jsonfile string
+	gzip     bool
+	tarsum   string
+}
+
+var testLayers = []testLayer{
+	{
+		filename: "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar",
+		jsonfile: "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/json",
+		tarsum:   "tarsum+sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b"},
+	{
+		filename: "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar",
+		jsonfile: "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/json",
+		gzip:     true,
+		tarsum:   "tarsum+sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b"},
+	{
+		filename: "testdata/511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/layer.tar",
+		jsonfile: "testdata/511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/json",
+		tarsum:   "tarsum+sha256:ac672ee85da9ab7f9667ae3c32841d3e42f33cc52c273c23341dabba1c8b0c8b"},
+	{
+		options: &sizedOptions{1, 1024 * 1024, false, false}, // a 1mb file (in memory)
+		tarsum:  "tarsum+sha256:8bf12d7e67c51ee2e8306cba569398b1b9f419969521a12ffb9d8875e8836738"},
+}
+
+type sizedOptions struct {
+	num      int64
+	size     int64
+	isRand   bool
+	realFile bool
+}
+
+// make a tar:
+// * num is the number of files the tar should have
+// * size is the bytes per file
+// * isRand is whether the contents of the files should be a random chunk (otherwise it's all zeros)
+// * realFile will write to a TempFile, instead of an in memory buffer
+func sizedTar(opts sizedOptions) io.Reader {
+	var (
+		fh  io.ReadWriter
+		err error
+	)
+	if opts.realFile {
+		fh, err = ioutil.TempFile("", "tarsum")
+		if err != nil {
+			return nil
+		}
+	} else {
+		fh = bytes.NewBuffer([]byte{})
+	}
+	tarW := tar.NewWriter(fh)
+	for i := int64(0); i < opts.num; i++ {
+		err := tarW.WriteHeader(&tar.Header{
+			Name: fmt.Sprintf("/testdata%d", i),
+			Mode: 0755,
+			Uid:  0,
+			Gid:  0,
+			Size: opts.size,
+		})
+		if err != nil {
+			return nil
+		}
+		var rBuf []byte
+		if opts.isRand {
+			rBuf = make([]byte, 8)
+			_, err = rand.Read(rBuf)
+			if err != nil {
+				return nil
+			}
+		} else {
+			rBuf = []byte{0, 0, 0, 0, 0, 0, 0, 0}
+		}
+
+		for i := int64(0); i < opts.size/int64(8); i++ {
+			tarW.Write(rBuf)
+		}
+	}
+	return fh
+}
+
+func TestTarSums(t *testing.T) {
+	for _, layer := range testLayers {
+		var (
+			fh  io.Reader
+			err error
+		)
+		if len(layer.filename) > 0 {
+			fh, err = os.Open(layer.filename)
+			if err != nil {
+				t.Errorf("failed to open %s: %s", layer.filename, err)
+				continue
+			}
+		} else if layer.options != nil {
+			fh = sizedTar(*layer.options)
+		} else {
+			// What else is there to test?
+			t.Errorf("what to do with %#V", layer)
+			continue
+		}
+		if file, ok := fh.(*os.File); ok {
+			defer file.Close()
+		}
+
+		//                                  double negatives!
+		ts := &TarSum{Reader: fh, DisableCompression: !layer.gzip}
+		_, err = io.Copy(ioutil.Discard, ts)
+		if err != nil {
+			t.Errorf("failed to copy from %s: %s", layer.filename, err)
+			continue
+		}
+		var gotSum string
+		if len(layer.jsonfile) > 0 {
+			jfh, err := os.Open(layer.jsonfile)
+			if err != nil {
+				t.Errorf("failed to open %s: %s", layer.jsonfile, err)
+				continue
+			}
+			buf, err := ioutil.ReadAll(jfh)
+			if err != nil {
+				t.Errorf("failed to readAll %s: %s", layer.jsonfile, err)
+				continue
+			}
+			gotSum = ts.Sum(buf)
+		} else {
+			gotSum = ts.Sum(nil)
+		}
+
+		if layer.tarsum != gotSum {
+			t.Errorf("expecting [%s], but got [%s]", layer.tarsum, gotSum)
+		}
+	}
+}
+
+func Benchmark9kTar(b *testing.B) {
+	buf := bytes.NewBuffer([]byte{})
+	fh, err := os.Open("testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar")
+	if err != nil {
+		b.Error(err)
+		return
+	}
+	n, err := io.Copy(buf, fh)
+	fh.Close()
+
+	b.SetBytes(n)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		ts := &TarSum{Reader: buf, DisableCompression: true}
+		io.Copy(ioutil.Discard, ts)
+		ts.Sum(nil)
+	}
+}
+
+func Benchmark9kTarGzip(b *testing.B) {
+	buf := bytes.NewBuffer([]byte{})
+	fh, err := os.Open("testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar")
+	if err != nil {
+		b.Error(err)
+		return
+	}
+	n, err := io.Copy(buf, fh)
+	fh.Close()
+
+	b.SetBytes(n)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		ts := &TarSum{Reader: buf, DisableCompression: false}
+		io.Copy(ioutil.Discard, ts)
+		ts.Sum(nil)
+	}
+}
+
+// this is a single big file in the tar archive
+func Benchmark1mbSingleFileTar(b *testing.B) {
+	benchmarkTar(b, sizedOptions{1, 1024 * 1024, true, true}, false)
+}
+
+// this is a single big file in the tar archive
+func Benchmark1mbSingleFileTarGzip(b *testing.B) {
+	benchmarkTar(b, sizedOptions{1, 1024 * 1024, true, true}, true)
+}
+
+// this is 1024 1k files in the tar archive
+func Benchmark1kFilesTar(b *testing.B) {
+	benchmarkTar(b, sizedOptions{1024, 1024, true, true}, false)
+}
+
+// this is 1024 1k files in the tar archive
+func Benchmark1kFilesTarGzip(b *testing.B) {
+	benchmarkTar(b, sizedOptions{1024, 1024, true, true}, true)
+}
+
+func benchmarkTar(b *testing.B, opts sizedOptions, isGzip bool) {
+	var fh *os.File
+	tarReader := sizedTar(opts)
+	if br, ok := tarReader.(*os.File); ok {
+		fh = br
+	}
+	defer os.Remove(fh.Name())
+	defer fh.Close()
+
+	b.SetBytes(opts.size * opts.num)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		ts := &TarSum{Reader: fh, DisableCompression: !isGzip}
+		io.Copy(ioutil.Discard, ts)
+		ts.Sum(nil)
+		fh.Seek(0, 0)
+	}
+}

utils/testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/json

@@ -0,0 +1 @@
+{"id":"46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457","parent":"def3f9165934325dfd027c86530b2ea49bb57a0963eb1336b3a0415ff6fd56de","created":"2014-04-07T02:45:52.610504484Z","container":"e0f07f8d72cae171a3dcc35859960e7e956e0628bce6fedc4122bf55b2c287c7","container_config":{"Hostname":"88807319f25e","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["HOME=/","PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","sed -ri 's/^(%wheel.*)(ALL)$/\\1NOPASSWD: \\2/' /etc/sudoers"],"Image":"def3f9165934325dfd027c86530b2ea49bb57a0963eb1336b3a0415ff6fd56de","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":[]},"docker_version":"0.9.1-dev","config":{"Hostname":"88807319f25e","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["HOME=/","PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":null,"Image":"def3f9165934325dfd027c86530b2ea49bb57a0963eb1336b3a0415ff6fd56de","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":[]},"architecture":"amd64","os":"linux","Size":3425}

utils/testdata/511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/json

@@ -0,0 +1 @@
+{"id":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158","comment":"Imported from -","created":"2013-06-13T14:03:50.821769-07:00","container_config":{"Hostname":"","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":null,"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":null},"docker_version":"0.4.0","architecture":"x86_64","Size":0}