beenull/moby
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Fixed checksum computing. Ensure checksum is computed when image metadata is loaded from disk. Fixed docker push workflow. Moved hash computing to utils

Browse source
This commit is contained in:
shin- 2013-04-26 14:00:09 -07:00
parent 3c85e9390e
commit be75608906
5 changed files with 155 additions and 93 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
container.go
View file

@ -1,8 +1,6 @@
package docker
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"github.com/dotcloud/docker/rcli"
@ -698,15 +696,11 @@ func (container *Container) ExportRw() (Archive, error) {
}
func (container *Container) RwChecksum() (string, error) {
h := sha256.New()
rwData, err := container.ExportRw()
rwData, err := Tar(container.rwPath(), Xz)
if err != nil {
return "", err
}
if _, err := io.Copy(h, rwData); err != nil {
return "", err
}
return "sha256:"+hex.EncodeToString(h.Sum(nil)), nil
return HashData(rwData)
}
func (container *Container) Export() (Archive, error) {

9
graph.go
View file

@ -81,6 +81,13 @@ func (graph *Graph) Get(name string) (*Image, error) {
return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.Id)
}
img.graph = graph
if img.Checksum == "" {
err := img.FixChecksum()
if err != nil {
return nil, err
}
}
return img, nil
}
@ -98,7 +105,7 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut
img.Parent = container.Image
img.Container = container.Id
img.ContainerConfig = *container.Config
// FIXME: If an image is pulled from a raw URL (not created from a container),
// FIXME: If an image is imported from a raw URL (not created from a container),
// its checksum will not be computed, which will cause a push to fail
checksum, err := container.RwChecksum()
if err != nil {

29
image.go
View file

@ -52,6 +52,7 @@ func LoadImage(root string) (*Image, error) {
} else if !stat.IsDir() {
return nil, fmt.Errorf("Couldn't load image %s: %s is not a directory", img.Id, layerPath(root))
}
return &img, nil
}
@ -258,3 +259,31 @@ func (img *Image) layer() (string, error) {
}
return layerPath(root), nil
}
func (img *Image) FixChecksum() error {
layer, err := img.layer()
if err != nil {
return err
}
layerData, err := Tar(layer, Xz)
if err != nil {
return err
}
sum, err := HashData(layerData)
if err != nil {
return err
}
img.Checksum = sum
jsonData, err := json.Marshal(img)
if err != nil {
return err
}
root, err := img.root()
if err != nil {
return err
}
if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil {
return err
}
return nil
}

190
registry.go
View file

@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"github.com/dotcloud/docker/auth"
"github.com/shin-/cookiejar"
"io"
"io/ioutil"
"net/http"
@ -14,7 +15,6 @@ import (
)
//FIXME: Set the endpoint in a conf file or via commandline
//const INDEX_ENDPOINT = "http://registry-creack.dotcloud.com/v1"
const INDEX_ENDPOINT = auth.INDEX_SERVER + "/v1"
// Build an Image object from raw json data
@ -47,6 +47,13 @@ func NewMultipleImgJson(src []byte) ([]*Image, error) {
return ret, nil
}
func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) {
for _, cookie := range c.Jar.Cookies(req.URL) {
req.AddCookie(cookie)
}
return c.Do(req)
}
// Retrieve the history of a given image from the Registry.
// Return a list of the parent's json (requested image included)
func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([]*Image, error) {
@ -57,7 +64,6 @@ func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([]
return nil, err
}
req.Header["X-Docker-Token"] = token
// req.SetBasicAuth(authConfig.Username, authConfig.Password)
res, err := client.Do(req)
if err != nil || res.StatusCode != 200 {
if res != nil {
@ -82,7 +88,9 @@ func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([]
func (graph *Graph) getHttpClient() *http.Client {
if graph.httpClient == nil {
graph.httpClient = new(http.Client)
graph.httpClient.Jar = cookiejar.NewCookieJar()
}
Debugf("cookies: %v",graph.httpClient.Jar)
return graph.httpClient
}
@ -111,7 +119,6 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, tok
return nil, nil, fmt.Errorf("Failed to download json: %s", err)
}
req.Header["X-Docker-Token"] = token
// req.SetBasicAuth(authConfig.Username, authConfig.Password)
res, err := client.Do(req)
if err != nil {
return nil, nil, fmt.Errorf("Failed to download json: %s", err)
@ -139,7 +146,6 @@ func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, tok
return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err)
}
req.Header["X-Docker-Token"] = token
// req.SetBasicAuth(authConfig.Username, authConfig.Password)
res, err = client.Do(req)
if err != nil {
return nil, nil, err
@ -271,7 +277,6 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re
return fmt.Errorf("Index response didn't contain any endpoints")
}
// FIXME: If askedTag is empty, fetch all tags.
var tagsList map[string]string
if askedTag == "" {
tagsList, err = graph.getRemoteTags(stdout, endpoints, remote, token)
@ -316,86 +321,86 @@ func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, re
return nil
}
// Push a local image to the registry with its history if needed
func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, token []string) error {
func pushImageRec(graph *Graph, stdout io.Writer, img *Image, registry string, token []string) error {
if parent, err := img.GetParent(); err != nil {
return err
} else if parent != nil {
if err := pushImageRec(graph, stdout, parent, registry, token); err != nil {
return err
}
}
client := graph.getHttpClient()
registry = "https://" + registry + "/v1"
jsonRaw, err := ioutil.ReadFile(path.Join(graph.Root, img.Id, "json"))
if err != nil {
return fmt.Errorf("Error while retreiving the path for {%s}: %s", img.Id, err)
}
// FIXME: Factorize the code
// FIXME: Do the puts in goroutines
if err := imgOrig.WalkHistory(func(img *Image) error {
fmt.Fprintf(stdout, "Pushing %s metadata\r\n", img.Id)
jsonRaw, err := ioutil.ReadFile(path.Join(graph.Root, img.Id, "json"))
if err != nil {
return fmt.Errorf("Error while retreiving the path for {%s}: %s", img.Id, err)
}
fmt.Fprintf(stdout, "Pushing %s metadata\r\n", img.Id)
// FIXME: try json with UTF8
jsonData := strings.NewReader(string(jsonRaw))
req, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/json", jsonData)
if err != nil {
return err
}
req.Header.Add("Content-type", "application/json")
req.Header["X-Docker-Token"] = token
res, err := client.Do(req)
if err != nil {
return fmt.Errorf("Failed to upload metadata: %s", err)
}
defer res.Body.Close()
if res.StatusCode != 200 {
switch res.StatusCode {
case 204:
// Case where the image is already on the Registry
fmt.Fprintf(stdout, "Image %s already uploaded ; skipping.", img.Id)
return nil
default:
errBody, err := ioutil.ReadAll(res.Body)
if err != nil {
errBody = []byte(err.Error())
}
return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody)
}
}
fmt.Fprintf(stdout, "Pushing %s fs layer\r\n", img.Id)
layerData2, err := Tar(path.Join(graph.Root, img.Id, "layer"), Xz)
tmp, err := ioutil.ReadAll(layerData2)
if err != nil {
return err
}
layerLength := len(tmp)
layerData, err := Tar(path.Join(graph.Root, img.Id, "layer"), Xz)
if err != nil {
return fmt.Errorf("Failed to generate layer archive: %s", err)
}
req3, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/layer",
ProgressReader(layerData.(io.ReadCloser), layerLength, stdout))
if err != nil {
return err
}
req3.ContentLength = int64(tmpLayer.Size)
req3.TransferEncoding = []string{"none"}
req3.Header["X-Docker-Token"] = token
res3, err := client.Do(req3)
if err != nil {
return fmt.Errorf("Failed to upload layer: %s", err)
}
res3.Body.Close()
if res3.StatusCode != 200 {
return fmt.Errorf("Received HTTP code %d while uploading layer", res3.StatusCode)
}
return nil
}); err != nil {
// FIXME: try json with UTF8
jsonData := strings.NewReader(string(jsonRaw))
req, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/json", jsonData)
if err != nil {
return err
}
req.Header.Add("Content-type", "application/json")
req.Header.Set("Authorization", "Token " + strings.Join(token, ","))
res, err := doWithCookies(client, req)
if err != nil {
return fmt.Errorf("Failed to upload metadata: %s", err)
}
defer res.Body.Close()
if len(res.Cookies()) > 0 {
client.Jar.SetCookies(req.URL, res.Cookies())
}
if res.StatusCode != 200 {
errBody, err := ioutil.ReadAll(res.Body)
if err != nil {
errBody = []byte(err.Error())
}
var jsonBody map[string]string
if err := json.Unmarshal(errBody, &jsonBody); err != nil {
errBody = []byte(err.Error())
} else if jsonBody["error"] == "Image already exists" {
fmt.Fprintf(stdout, "Image %v already uploaded ; skipping\n", img.Id)
return nil
}
return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody)
}
fmt.Fprintf(stdout, "Pushing %s fs layer\r\n", img.Id)
layerData, err := Tar(path.Join(graph.Root, img.Id, "layer"), Xz)
if err != nil {
return fmt.Errorf("Failed to generate layer archive: %s", err)
}
req3, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/layer",
layerData)
if err != nil {
return err
}
req3.ContentLength = -1
req3.TransferEncoding = []string{"chunked"}
req3.Header.Set("Authorization", "Token " + strings.Join(token, ","))
fmt.Printf("%v", req3.Header)
res3, err := doWithCookies(client, req3)
if err != nil {
return fmt.Errorf("Failed to upload layer: %s", err)
}
res3.Body.Close()
if res3.StatusCode != 200 {
return fmt.Errorf("Received HTTP code %d while uploading layer", res3.StatusCode)
}
return nil
}
// Push a local image to the registry with its history if needed
func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, registry string, token []string) error {
registry = "https://" + registry + "/v1"
return pushImageRec(graph, stdout, imgOrig, registry, token)
}
// push a tag on the registry.
// Remote has the format '<user>/<repo>
func (graph *Graph) pushTag(remote, revision, tag, registry string, token []string) error {
@ -413,9 +418,13 @@ func (graph *Graph) pushTag(remote, revision, tag, registry string, token []stri
client := graph.getHttpClient()
req, err := http.NewRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision))
if err != nil {
return err
}
req.Header.Add("Content-type", "application/json")
req.Header["X-Docker-Token"] = token
res, err := client.Do(req)
req.Header.Set("Authorization", "Token " + strings.Join(token, ","))
req.ContentLength = int64(len(revision))
res, err := doWithCookies(client, req)
if err != nil {
return err
}
@ -441,7 +450,7 @@ func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId, registry
}
fmt.Fprintf(stdout, "Registering tag %s:%s\r\n", remote, tag)
// And then the tag
if err = graph.pushTag(remote, imgId, registry, tag, token); err != nil {
if err = graph.pushTag(remote, imgId, tag, registry, token); err != nil {
return err
}
return nil
@ -453,15 +462,25 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re
client := graph.getHttpClient()
checksums, err := graph.Checksums(localRepo)
imgList := make([]map[string]string, len(checksums))
if err != nil {
return err
}
req, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote, nil)
for i, obj := range checksums {
imgList[i] = map[string]string{"id": obj["id"]}
}
imgListJson, err := json.Marshal(imgList)
if err != nil {
return err
}
req, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote, bytes.NewReader(imgListJson))
if err != nil {
return err
}
req.SetBasicAuth(authConfig.Username, authConfig.Password)
req.ContentLength = int64(len(imgListJson))
req.Header.Set("X-Docker-Token", "true")
res, err := client.Do(req)
if err != nil {
@ -470,11 +489,12 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re
res.Body.Close()
for res.StatusCode >= 300 && res.StatusCode < 400 {
Debugf("Redirected to %s\n", res.Header.Get("Location"))
req, err = http.NewRequest("PUT", res.Header.Get("Location"), nil)
req, err = http.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJson))
if err != nil {
return err
}
req.SetBasicAuth(authConfig.Username, authConfig.Password)
req.ContentLength = int64(len(imgListJson))
req.Header.Set("X-Docker-Token", "true")
res, err = client.Do(req)
if err != nil {
@ -490,6 +510,7 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re
var token, endpoints []string
if res.Header.Get("X-Docker-Token") != "" {
token = res.Header["X-Docker-Token"]
Debugf("Auth token: %v", token)
} else {
Debugf("Response headers:\n %s\n", res.Header)
return fmt.Errorf("Index response didn't contain an access token")
@ -515,13 +536,14 @@ func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Re
if err != nil {
return err
}
req2, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote+"/images", bytes.NewBuffer(checksumsJson))
req2, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote+"/images", bytes.NewReader(checksumsJson))
if err != nil {
return err
}
req2.SetBasicAuth(authConfig.Username, authConfig.Password)
req2.Header["X-Docker-Endpoints"] = endpoints
res2, err := client.Do(req)
req2.ContentLength = int64(len(checksumsJson))
res2, err := client.Do(req2)
if err != nil {
return err
}

10
utils.go
View file

@ -2,6 +2,8 @@ package docker
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"github.com/dotcloud/docker/rcli"
@ -456,3 +458,11 @@ func FindCgroupMountpoint(cgroupType string) (string, error) {
return "", fmt.Errorf("cgroup mountpoint not found for %s", cgroupType)
}
func HashData(src io.Reader) (string, error) {
h := sha256.New()
if _, err := io.Copy(h, src); err != nil {
return "", err
}
return "sha256:"+hex.EncodeToString(h.Sum(nil)), nil
}