bump to master

CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## 0.3.1 (2013-05-08)
+ + Builder: Implement the autorun capability within docker builder
+ + Builder: Add caching to docker builder
+ + Builder: Add support for docker builder with native API as top level command
+ + Runtime: Add go version to debug infos
+ + Builder: Implement ENV within docker builder
+ + Registry: Add docker search top level command in order to search a repository
+ + Images: output graph of images to dot (graphviz)
+ + Documentation: new introduction and high-level overview
+ + Documentation: Add the documentation for docker builder
+ + Website: new high-level overview
+ - Makefile: Swap "go get" for "go get -d", especially to compile on go1.1rc
+ - Images: fix ByParent function
+ - Builder: Check the command existance prior create and add Unit tests for the case
+ - Registry: Fix pull for official images with specific tag
+ - Registry: Fix issue when login in with a different user and trying to push
+ - Documentation: CSS fix for docker documentation to make REST API docs look better.
+ - Documentation: Fixed CouchDB example page header mistake
+ - Documentation: fixed README formatting
+ * Registry: Improve checksum - async calculation
+ * Runtime: kernel version - don't show the dash if flavor is empty
+ * Documentation: updated www.docker.io website.
+ * Builder: use any whitespaces instead of tabs
+ * Packaging: packaging ubuntu; issue #510: Use goland-stable PPA package to build docker
+
 ## 0.3.0 (2013-05-06)
  + Registry: Implement the new registry
  + Documentation: new example: sharing data between 2 couchdb databases

Makefile

@@ -39,7 +39,7 @@ $(DOCKER_BIN): $(DOCKER_DIR)
 $(DOCKER_DIR):
 	@mkdir -p $(dir $@)
 	@if [ -h $@ ]; then rm -f $@; fi; ln -sf $(CURDIR)/ $@
-	@(cd $(DOCKER_MAIN); go get $(GO_OPTIONS))
+	@(cd $(DOCKER_MAIN); go get -d $(GO_OPTIONS))
 
 whichrelease:
 	echo $(RELEASE_VERSION)

api_params.go

@@ -18,6 +18,7 @@ type ApiInfo struct {
 	Version     string
 	Images      int
 	Debug       bool
+	GoVersion   string
 	NFd         int `json:",omitempty"`
 	NGoroutines int `json:",omitempty"`
 }

commands.go

@@ -22,7 +22,7 @@ import (
 	"unicode"
 )
 
-const VERSION = "0.3.0"
+const VERSION = "0.3.1"
 
 var (
 	GIT_COMMIT string
@@ -362,7 +362,7 @@ func CmdInfo(args ...string) error {
 	if err != nil {
 		return err
 	}
-	fmt.Printf("containers: %d\nversion: %s\nimages: %d\n", out.Containers, out.Version, out.Images)
+	fmt.Printf("containers: %d\nversion: %s\nimages: %d\nGo version: %s\n", out.Containers, out.Version, out.Images, out.GoVersion)
 	if out.Debug {
 		fmt.Println("debug mode enabled")
 		fmt.Printf("fds: %d\ngoroutines: %d\n", out.NFd, out.NGoroutines)

docs/sources/remote-api/api.rst

@@ -874,6 +874,7 @@ Display system-wide information
 		"Containers":11,
 		"Version":"0.2.2",
 		"Images":16,
+		"GoVersion":"go1.0.3",
 		"Debug":false
 	   }
 

docs/theme/docker/static/css/main.css

@@ -330,3 +330,7 @@ section.header {
 @media (max-width: 480px) {
   
 }
+/* Misc fixes */
+table th {
+  text-align: left;
+}

docs/theme/docker/static/css/main.less

@@ -449,4 +449,9 @@ section.header {
 @media (max-width: 480px) {
 
 
+}
+
+/* Misc fixes */
+table th {
+  text-align: left;
 }

graph.go

@@ -9,14 +9,18 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"sync"
 	"time"
 )
 
 // A Graph is a store for versioned filesystem images and the relationship between them.
 type Graph struct {
-	Root       string
-	idIndex    *TruncIndex
-	httpClient *http.Client
+	Root         string
+	idIndex      *TruncIndex
+	httpClient   *http.Client
+	checksumLock map[string]*sync.Mutex
+	lockSumFile  *sync.Mutex
+	lockSumMap   *sync.Mutex
 }
 
 // NewGraph instantiates a new graph at the given root path in the filesystem.
@@ -27,12 +31,15 @@ func NewGraph(root string) (*Graph, error) {
 		return nil, err
 	}
 	// Create the root directory if it doesn't exists
-	if err := os.Mkdir(root, 0700); err != nil && !os.IsExist(err) {
+	if err := os.MkdirAll(root, 0700); err != nil && !os.IsExist(err) {
 		return nil, err
 	}
 	graph := &Graph{
-		Root:    abspath,
-		idIndex: NewTruncIndex(),
+		Root:         abspath,
+		idIndex:      NewTruncIndex(),
+		checksumLock: make(map[string]*sync.Mutex),
+		lockSumFile:  &sync.Mutex{},
+		lockSumMap:   &sync.Mutex{},
 	}
 	if err := graph.restore(); err != nil {
 		return nil, err
@@ -82,6 +89,11 @@ func (graph *Graph) Get(name string) (*Image, error) {
 		return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.Id)
 	}
 	img.graph = graph
+	graph.lockSumMap.Lock()
+	defer graph.lockSumMap.Unlock()
+	if _, exists := graph.checksumLock[img.Id]; !exists {
+		graph.checksumLock[img.Id] = &sync.Mutex{}
+	}
 	return img, nil
 }
 
@@ -103,7 +115,7 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut
 	if err := graph.Register(layerData, img); err != nil {
 		return nil, err
 	}
-	img.Checksum()
+	go img.Checksum()
 	return img, nil
 }
 
@@ -131,6 +143,7 @@ func (graph *Graph) Register(layerData Archive, img *Image) error {
 	}
 	img.graph = graph
 	graph.idIndex.Add(img.Id)
+	graph.checksumLock[img.Id] = &sync.Mutex{}
 	return nil
 }
 

image.go

@@ -35,8 +35,9 @@ func LoadImage(root string) (*Image, error) {
 	if err != nil {
 		return nil, err
 	}
-	var img Image
-	if err := json.Unmarshal(jsonData, &img); err != nil {
+	img := &Image{}
+
+	if err := json.Unmarshal(jsonData, img); err != nil {
 		return nil, err
 	}
 	if err := ValidateId(img.Id); err != nil {
@@ -52,8 +53,7 @@ func LoadImage(root string) (*Image, error) {
 	} else if !stat.IsDir() {
 		return nil, fmt.Errorf("Couldn't load image %s: %s is not a directory", img.Id, layerPath(root))
 	}
-
-	return &img, nil
+	return img, nil
 }
 
 func StoreImage(img *Image, layerData Archive, root string) error {
@@ -261,19 +261,22 @@ func (img *Image) layer() (string, error) {
 }
 
 func (img *Image) Checksum() (string, error) {
+	img.graph.checksumLock[img.Id].Lock()
+	defer img.graph.checksumLock[img.Id].Unlock()
+
 	root, err := img.root()
 	if err != nil {
 		return "", err
 	}
 
 	checksumDictPth := path.Join(root, "..", "..", "checksums")
-	checksums := new(map[string]string)
+	checksums := make(map[string]string)
 
 	if checksumDict, err := ioutil.ReadFile(checksumDictPth); err == nil {
-		if err := json.Unmarshal(checksumDict, checksums); err != nil {
+		if err := json.Unmarshal(checksumDict, &checksums); err != nil {
 			return "", err
 		}
-		if checksum, ok := (*checksums)[img.Id]; ok {
+		if checksum, ok := checksums[img.Id]; ok {
 			return checksum, nil
 		}
 	}
@@ -299,20 +302,26 @@ func (img *Image) Checksum() (string, error) {
 	if _, err := h.Write([]byte("\n")); err != nil {
 		return "", err
 	}
+
 	if _, err := io.Copy(h, layerData); err != nil {
 		return "", err
 	}
 
 	hash := "sha256:" + hex.EncodeToString(h.Sum(nil))
-	if *checksums == nil {
-		*checksums = map[string]string{}
+	checksums[img.Id] = hash
+
+	// Reload the json file to make sure not to overwrite faster sums
+	img.graph.lockSumFile.Lock()
+	defer img.graph.lockSumFile.Unlock()
+	if checksumDict, err := ioutil.ReadFile(checksumDictPth); err == nil {
+		if err := json.Unmarshal(checksumDict, &checksums); err != nil {
+			return "", err
+		}
 	}
-	(*checksums)[img.Id] = hash
 	checksumJson, err := json.Marshal(checksums)
 	if err != nil {
 		return hash, err
 	}
-
 	if err := ioutil.WriteFile(checksumDictPth, checksumJson, 0600); err != nil {
 		return hash, err
 	}

packaging/ubuntu/changelog

@@ -1,3 +1,30 @@
+lxc-docker (0.3.1-1) precise; urgency=low
+ - Builder: Implement the autorun capability within docker builder
+ - Builder: Add caching to docker builder
+ - Builder: Add support for docker builder with native API as top level command
+ - Runtime: Add go version to debug infos
+ - Builder: Implement ENV within docker builder
+ - Registry: Add docker search top level command in order to search a repository
+ - Images: output graph of images to dot (graphviz)
+ - Documentation: new introduction and high-level overview
+ - Documentation: Add the documentation for docker builder
+ - Website: new high-level overview
+ - Makefile: Swap "go get" for "go get -d", especially to compile on go1.1rc
+ - Images: fix ByParent function
+ - Builder: Check the command existance prior create and add Unit tests for the case
+ - Registry: Fix pull for official images with specific tag
+ - Registry: Fix issue when login in with a different user and trying to push
+ - Documentation: CSS fix for docker documentation to make REST API docs look better.
+ - Documentation: Fixed CouchDB example page header mistake
+ - Documentation: fixed README formatting
+ - Registry: Improve checksum - async calculation
+ - Runtime: kernel version - don't show the dash if flavor is empty
+ - Documentation: updated www.docker.io website.
+ - Builder: use any whitespaces instead of tabs
+ - Packaging: packaging ubuntu; issue #510: Use goland-stable PPA package to build docker
+
+ -- dotCloud <ops@dotcloud.com>  Fri, 8 May 2013 00:00:00 -0700
+
 lxc-docker (0.3.0-1) precise; urgency=low
   - Registry: Implement the new registry
   - Documentation: new example: sharing data between 2 couchdb databases

registry.go

@@ -194,18 +194,16 @@ func (graph *Graph) getRemoteTags(stdout io.Writer, registries []string, reposit
 			return nil, fmt.Errorf("Repository not found")
 		}
 
-		result := new(map[string]string)
+		result := make(map[string]string)
 
 		rawJson, err := ioutil.ReadAll(res.Body)
 		if err != nil {
 			return nil, err
 		}
-		if err = json.Unmarshal(rawJson, result); err != nil {
+		if err = json.Unmarshal(rawJson, &result); err != nil {
 			return nil, err
 		}
-
-		return *result, nil
-
+		return result, nil
 	}
 	return nil, fmt.Errorf("Could not reach any registry endpoint")
 }
@@ -308,6 +306,50 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re
 		return fmt.Errorf("Index response didn't contain any endpoints")
 	}
 
+	checksumsJson, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return err
+	}
+
+	// Reload the json file to make sure not to overwrite faster sums
+	err = func() error {
+		localChecksums := make(map[string]string)
+		remoteChecksums := []struct {
+			Id       string `json: "id"`
+			Checksum string `json: "checksum"`
+		}{}
+		checksumDictPth := path.Join(graph.Root, "..", "checksums")
+
+		if err := json.Unmarshal(checksumsJson, &remoteChecksums); err != nil {
+			return err
+		}
+
+		graph.lockSumFile.Lock()
+		defer graph.lockSumFile.Unlock()
+
+		if checksumDict, err := ioutil.ReadFile(checksumDictPth); err == nil {
+			if err := json.Unmarshal(checksumDict, &localChecksums); err != nil {
+				return err
+			}
+		}
+
+		for _, elem := range remoteChecksums {
+			localChecksums[elem.Id] = elem.Checksum
+		}
+
+		checksumsJson, err = json.Marshal(localChecksums)
+		if err != nil {
+			return err
+		}
+		if err := ioutil.WriteFile(checksumDictPth, checksumsJson, 0600); err != nil {
+			return err
+		}
+		return nil
+	}()
+	if err != nil {
+		return err
+	}
+
 	var tagsList map[string]string
 	if askedTag == "" {
 		tagsList, err = graph.getRemoteTags(stdout, endpoints, remote, token)
@@ -427,9 +469,15 @@ func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, t
 	if err != nil {
 		return fmt.Errorf("Failed to upload layer: %s", err)
 	}
-	res3.Body.Close()
+	defer res3.Body.Close()
+
 	if res3.StatusCode != 200 {
-		return fmt.Errorf("Received HTTP code %d while uploading layer", res3.StatusCode)
+		errBody, err := ioutil.ReadAll(res3.Body)
+		if err != nil {
+			return fmt.Errorf("HTTP code %d while uploading metadata and error when"+
+				" trying to parse response body: %v", res.StatusCode, err)
+		}
+		return fmt.Errorf("Received HTTP code %d while uploading layer: %s", res3.StatusCode, errBody)
 	}
 	return nil
 }
@@ -612,8 +660,7 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re
 }
 
 func (graph *Graph) Checksums(output io.Writer, repo Repository) ([]map[string]string, error) {
-	var result []map[string]string
-	checksums := map[string]string{}
+	checksums := make(map[string]string)
 	for _, id := range repo {
 		img, err := graph.Get(id)
 		if err != nil {
@@ -634,7 +681,7 @@ func (graph *Graph) Checksums(output io.Writer, repo Repository) ([]map[string]s
 		}
 	}
 	i := 0
-	result = make([]map[string]string, len(checksums))
+	result := make([]map[string]string, len(checksums))
 	for id, sum := range checksums {
 		result[i] = map[string]string{
 			"id":       id,

server.go

@@ -201,6 +201,7 @@ func (srv *Server) DockerInfo() ApiInfo {
 	out.Containers = len(srv.runtime.List())
 	out.Version = VERSION
 	out.Images = imgcount
+	out.GoVersion = runtime.Version()
 	if os.Getenv("DEBUG") != "" {
 		out.Debug = true
 		out.NFd = getTotalUsedFds()