From cbfdb45ad6006d8fc6c0253498e74141a004dad8 Mon Sep 17 00:00:00 2001 From: Justin Cormack Date: Thu, 1 Dec 2016 06:26:59 -0800 Subject: [PATCH 1/4] Merge pull request #29027 from runcom/fix-jq contrib: download-frozen-image-v2.sh requires jq (cherry picked from commit a227ea62e6f4bc316e0ee14188fc54a3e517c3bc) Signed-off-by: Sebastiaan van Stijn --- contrib/download-frozen-image-v2.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contrib/download-frozen-image-v2.sh b/contrib/download-frozen-image-v2.sh index 111e3fa2ba..d3c88c17d1 100755 --- a/contrib/download-frozen-image-v2.sh +++ b/contrib/download-frozen-image-v2.sh @@ -11,6 +11,10 @@ if ! command -v curl &> /dev/null; then echo >&2 'error: "curl" not found!' exit 1 fi +if ! command -v jq &> /dev/null; then + echo >&2 'error: "jq" not found!' + exit 1 +fi usage() { echo "usage: $0 dir image[:tag][@digest] ..." From ec25773e315287e9e6c508b68dd8684477846770 Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Mon, 13 Feb 2017 19:53:19 +0100 Subject: [PATCH 2/4] Merge pull request #29864 from tianon/schemaVersion-2 Update "download-frozen-image-v2.sh" for schemaVersion 2 (cherry picked from commit 5870675ebc0db9fbe07736a59a6226b805ba00dc) Signed-off-by: Sebastiaan van Stijn --- contrib/download-frozen-image-v2.sh | 230 +++++++++++++++++++++++----- 1 file changed, 190 insertions(+), 40 deletions(-) diff --git a/contrib/download-frozen-image-v2.sh b/contrib/download-frozen-image-v2.sh index d3c88c17d1..6d0a8d6e8a 100755 --- a/contrib/download-frozen-image-v2.sh +++ b/contrib/download-frozen-image-v2.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -eo pipefail # hello-world latest ef872312fe1b 3 months ago 910 B # hello-world latest ef872312fe1bbc5e05aae626791a47ee9b032efa8f3bda39cc0be7b56bfe59b9 3 months ago 910 B @@ -31,8 +31,19 @@ mkdir -p "$dir" # hacky workarounds for Bash 3 support (no associative arrays) images=() rm -f "$dir"/tags-*.tmp +manifestJsonEntries=() +doNotGenerateManifestJson= # repositories[busybox]='"latest": "...", "ubuntu-14.04": "..."' +# bash v4 on Windows CI requires CRLF separator +newlineIFS=$'\n' +if [ "$(go env GOHOSTOS)" = 'windows' ]; then + major=$(echo ${BASH_VERSION%%[^0.9]} | cut -d. -f1) + if [ "$major" -ge 4 ]; then + newlineIFS=$'\r\n' + fi +fi + while [ $# -gt 0 ]; do imageTag="$1" shift @@ -48,30 +59,187 @@ while [ $# -gt 0 ]; do imageFile="${image//\//_}" # "/" can't be in filenames :) - token="$(curl -sSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$image:pull" | jq --raw-output .token)" + token="$(curl -fsSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$image:pull" | jq --raw-output '.token')" - manifestJson="$(curl -sSL -H "Authorization: Bearer $token" "https://registry-1.docker.io/v2/$image/manifests/$digest")" + manifestJson="$( + curl -fsSL \ + -H "Authorization: Bearer $token" \ + -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \ + -H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \ + "https://registry-1.docker.io/v2/$image/manifests/$digest" + )" if [ "${manifestJson:0:1}" != '{' ]; then echo >&2 "error: /v2/$image/manifests/$digest returned something unexpected:" echo >&2 " $manifestJson" exit 1 fi - layersFs=$(echo "$manifestJson" | jq --raw-output '.fsLayers | .[] | .blobSum') + imageIdentifier="$image:$tag@$digest" - IFS=$'\n' - # bash v4 on Windows CI requires CRLF separator - if [ "$(go env GOHOSTOS)" = 'windows' ]; then - major=$(echo ${BASH_VERSION%%[^0.9]} | cut -d. -f1) - if [ "$major" -ge 4 ]; then - IFS=$'\r\n' - fi - fi - layers=( ${layersFs} ) - unset IFS + schemaVersion="$(echo "$manifestJson" | jq --raw-output '.schemaVersion')" + case "$schemaVersion" in + 2) + mediaType="$(echo "$manifestJson" | jq --raw-output '.mediaType')" - history=$(echo "$manifestJson" | jq '.history | [.[] | .v1Compatibility]') - imageId=$(echo "$history" | jq --raw-output .[0] | jq --raw-output .id) + case "$mediaType" in + application/vnd.docker.distribution.manifest.v2+json) + configDigest="$(echo "$manifestJson" | jq --raw-output '.config.digest')" + imageId="${configDigest#*:}" # strip off "sha256:" + + configFile="$imageId.json" + curl -fsSL \ + -H "Authorization: Bearer $token" \ + "https://registry-1.docker.io/v2/$image/blobs/$configDigest" \ + -o "$dir/$configFile" + + layersFs="$(echo "$manifestJson" | jq --raw-output --compact-output '.layers[]')" + IFS="$newlineIFS" + layers=( $layersFs ) + unset IFS + + echo "Downloading '$imageIdentifier' (${#layers[@]} layers)..." + layerId= + layerFiles=() + for i in "${!layers[@]}"; do + layerMeta="${layers[$i]}" + + layerMediaType="$(echo "$layerMeta" | jq --raw-output '.mediaType')" + layerDigest="$(echo "$layerMeta" | jq --raw-output '.digest')" + + # save the previous layer's ID + parentId="$layerId" + # create a new fake layer ID based on this layer's digest and the previous layer's fake ID + layerId="$(echo "$parentId"$'\n'"$layerDigest" | sha256sum | cut -d' ' -f1)" + # this accounts for the possibility that an image contains the same layer twice (and thus has a duplicate digest value) + + mkdir -p "$dir/$layerId" + echo '1.0' > "$dir/$layerId/VERSION" + + if [ ! -s "$dir/$layerId/json" ]; then + parentJson="$(printf ', parent: "%s"' "$parentId")" + addJson="$(printf '{ id: "%s"%s }' "$layerId" "${parentId:+$parentJson}")" + # this starter JSON is taken directly from Docker's own "docker save" output for unimportant layers + jq "$addJson + ." > "$dir/$layerId/json" <<-'EOJSON' + { + "created": "0001-01-01T00:00:00Z", + "container_config": { + "Hostname": "", + "Domainname": "", + "User": "", + "AttachStdin": false, + "AttachStdout": false, + "AttachStderr": false, + "Tty": false, + "OpenStdin": false, + "StdinOnce": false, + "Env": null, + "Cmd": null, + "Image": "", + "Volumes": null, + "WorkingDir": "", + "Entrypoint": null, + "OnBuild": null, + "Labels": null + } + } + EOJSON + fi + + case "$layerMediaType" in + application/vnd.docker.image.rootfs.diff.tar.gzip) + layerTar="$layerId/layer.tar" + layerFiles=( "${layerFiles[@]}" "$layerTar" ) + # TODO figure out why "-C -" doesn't work here + # "curl: (33) HTTP server doesn't seem to support byte ranges. Cannot resume." + # "HTTP/1.1 416 Requested Range Not Satisfiable" + if [ -f "$dir/$layerTar" ]; then + # TODO hackpatch for no -C support :'( + echo "skipping existing ${layerId:0:12}" + continue + fi + curl -fSL --progress \ + -H "Authorization: Bearer $token" \ + "https://registry-1.docker.io/v2/$image/blobs/$layerDigest" \ + -o "$dir/$layerTar" + ;; + + *) + echo >&2 "error: unknown layer mediaType ($imageIdentifier, $layerDigest): '$layerMediaType'" + exit 1 + ;; + esac + done + + # change "$imageId" to be the ID of the last layer we added (needed for old-style "repositories" file which is created later -- specifically for older Docker daemons) + imageId="$layerId" + + # munge the top layer image manifest to have the appropriate image configuration for older daemons + imageOldConfig="$(jq --raw-output --compact-output '{ id: .id } + if .parent then { parent: .parent } else {} end' "$dir/$imageId/json")" + jq --raw-output "$imageOldConfig + del(.history, .rootfs)" "$dir/$configFile" > "$dir/$imageId/json" + + manifestJsonEntry="$( + echo '{}' | jq --raw-output '. + { + Config: "'"$configFile"'", + RepoTags: ["'"${image#library\/}:$tag"'"], + Layers: '"$(echo '[]' | jq --raw-output ".$(for layerFile in "${layerFiles[@]}"; do echo " + [ \"$layerFile\" ]"; done)")"' + }' + )" + manifestJsonEntries=( "${manifestJsonEntries[@]}" "$manifestJsonEntry" ) + ;; + + *) + echo >&2 "error: unknown manifest mediaType ($imageIdentifier): '$mediaType'" + exit 1 + ;; + esac + ;; + + 1) + if [ -z "$doNotGenerateManifestJson" ]; then + echo >&2 "warning: '$imageIdentifier' uses schemaVersion '$schemaVersion'" + echo >&2 " this script cannot (currently) recreate the 'image config' to put in a 'manifest.json' (thus any schemaVersion 2+ images will be imported in the old way, and their 'docker history' will suffer)" + echo >&2 + doNotGenerateManifestJson=1 + fi + + layersFs="$(echo "$manifestJson" | jq --raw-output '.fsLayers | .[] | .blobSum')" + IFS="$newlineIFS" + layers=( $layersFs ) + unset IFS + + history="$(echo "$manifestJson" | jq '.history | [.[] | .v1Compatibility]')" + imageId="$(echo "$history" | jq --raw-output '.[0]' | jq --raw-output '.id')" + + echo "Downloading '$imageIdentifier' (${#layers[@]} layers)..." + for i in "${!layers[@]}"; do + imageJson="$(echo "$history" | jq --raw-output ".[${i}]")" + layerId="$(echo "$imageJson" | jq --raw-output '.id')" + imageLayer="${layers[$i]}" + + mkdir -p "$dir/$layerId" + echo '1.0' > "$dir/$layerId/VERSION" + + echo "$imageJson" > "$dir/$layerId/json" + + # TODO figure out why "-C -" doesn't work here + # "curl: (33) HTTP server doesn't seem to support byte ranges. Cannot resume." + # "HTTP/1.1 416 Requested Range Not Satisfiable" + if [ -f "$dir/$layerId/layer.tar" ]; then + # TODO hackpatch for no -C support :'( + echo "skipping existing ${layerId:0:12}" + continue + fi + curl -fSL --progress -H "Authorization: Bearer $token" "https://registry-1.docker.io/v2/$image/blobs/$imageLayer" -o "$dir/$layerId/layer.tar" # -C - + done + ;; + + *) + echo >&2 "error: unknown manifest schemaVersion ($imageIdentifier): '$schemaVersion'" + exit 1 + ;; + esac + + echo if [ -s "$dir/tags-$imageFile.tmp" ]; then echo -n ', ' >> "$dir/tags-$imageFile.tmp" @@ -79,30 +247,6 @@ while [ $# -gt 0 ]; do images=( "${images[@]}" "$image" ) fi echo -n '"'"$tag"'": "'"$imageId"'"' >> "$dir/tags-$imageFile.tmp" - - echo "Downloading '${image}:${tag}@${digest}' (${#layers[@]} layers)..." - for i in "${!layers[@]}"; do - imageJson=$(echo "$history" | jq --raw-output .[${i}]) - imageId=$(echo "$imageJson" | jq --raw-output .id) - imageLayer=${layers[$i]} - - mkdir -p "$dir/$imageId" - echo '1.0' > "$dir/$imageId/VERSION" - - echo "$imageJson" > "$dir/$imageId/json" - - # TODO figure out why "-C -" doesn't work here - # "curl: (33) HTTP server doesn't seem to support byte ranges. Cannot resume." - # "HTTP/1.1 416 Requested Range Not Satisfiable" - if [ -f "$dir/$imageId/layer.tar" ]; then - # TODO hackpatch for no -C support :'( - echo "skipping existing ${imageId:0:12}" - continue - fi - token="$(curl -sSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$image:pull" | jq --raw-output .token)" - curl -SL --progress -H "Authorization: Bearer $token" "https://registry-1.docker.io/v2/$image/blobs/$imageLayer" -o "$dir/$imageId/layer.tar" # -C - - done - echo done echo -n '{' > "$dir/repositories" @@ -120,6 +264,12 @@ echo -n $'\n}\n' >> "$dir/repositories" rm -f "$dir"/tags-*.tmp +if [ -z "$doNotGenerateManifestJson" ] && [ "${#manifestJsonEntries[@]}" -gt 0 ]; then + echo '[]' | jq --raw-output ".$(for entry in "${manifestJsonEntries[@]}"; do echo " + [ $entry ]"; done)" > "$dir/manifest.json" +else + rm -f "$dir/manifest.json" +fi + echo "Download of images into '$dir' complete." echo "Use something like the following to load the result into a Docker daemon:" echo " tar -cC '$dir' . | docker load" From ef4466eb1243cc1676d4324a23e41dace7add589 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 31 May 2017 21:03:37 +0200 Subject: [PATCH 3/4] Merge pull request #33443 from DeliangFan/fix_download_image_fails Fix downloading image fails when build docker (cherry picked from commit 555bd548cacfbe02447ebb055d3efa5eeaabe972) Signed-off-by: Sebastiaan van Stijn --- contrib/download-frozen-image-v2.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/download-frozen-image-v2.sh b/contrib/download-frozen-image-v2.sh index 6d0a8d6e8a..582cbb6681 100755 --- a/contrib/download-frozen-image-v2.sh +++ b/contrib/download-frozen-image-v2.sh @@ -157,6 +157,7 @@ while [ $# -gt 0 ]; do echo "skipping existing ${layerId:0:12}" continue fi + token="$(curl -fsSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$image:pull" | jq --raw-output '.token')" curl -fSL --progress \ -H "Authorization: Bearer $token" \ "https://registry-1.docker.io/v2/$image/blobs/$layerDigest" \ @@ -229,6 +230,7 @@ while [ $# -gt 0 ]; do echo "skipping existing ${layerId:0:12}" continue fi + token="$(curl -fsSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$image:pull" | jq --raw-output '.token')" curl -fSL --progress -H "Authorization: Bearer $token" "https://registry-1.docker.io/v2/$image/blobs/$imageLayer" -o "$dir/$layerId/layer.tar" # -C - done ;; From 7388483ef7da3c4eb29036b86112309411f93ef0 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Fri, 23 Jun 2017 20:27:54 -0700 Subject: [PATCH 4/4] Merge pull request #33707 from Syntaxide/s3fix Fix authorization header handling in downloader script. (cherry picked from commit d75eb735eee26521b76c732f8b07cfaf6c460d12) Signed-off-by: Sebastiaan van Stijn --- contrib/download-frozen-image-v2.sh | 56 ++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/contrib/download-frozen-image-v2.sh b/contrib/download-frozen-image-v2.sh index 582cbb6681..e6dfa8962a 100755 --- a/contrib/download-frozen-image-v2.sh +++ b/contrib/download-frozen-image-v2.sh @@ -44,6 +44,42 @@ if [ "$(go env GOHOSTOS)" = 'windows' ]; then fi fi +registryBase='https://registry-1.docker.io' +authBase='https://auth.docker.io' +authService='registry.docker.io' + +# https://github.com/moby/moby/issues/33700 +fetch_blob() { + local token="$1"; shift + local image="$1"; shift + local digest="$1"; shift + local targetFile="$1"; shift + local curlArgs=( "$@" ) + + local curlHeaders="$( + curl -S "${curlArgs[@]}" \ + -H "Authorization: Bearer $token" \ + "$registryBase/v2/$image/blobs/$digest" \ + -o "$targetFile" \ + -D- + )" + curlHeaders="$(echo "$curlHeaders" | tr -d '\r')" + if [ "$(echo "$curlHeaders" | awk 'NR == 1 { print $2; exit }')" != '200' ]; then + rm -f "$targetFile" + + local blobRedirect="$(echo "$curlHeaders" | awk -F ': ' 'tolower($1) == "location" { print $2; exit }')" + if [ -z "$blobRedirect" ]; then + echo >&2 "error: failed fetching '$image' blob '$digest'" + echo "$curlHeaders" | head -1 >&2 + return 1 + fi + + curl -fSL "${curlArgs[@]}" \ + "$blobRedirect" \ + -o "$targetFile" + fi +} + while [ $# -gt 0 ]; do imageTag="$1" shift @@ -59,14 +95,14 @@ while [ $# -gt 0 ]; do imageFile="${image//\//_}" # "/" can't be in filenames :) - token="$(curl -fsSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$image:pull" | jq --raw-output '.token')" + token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')" manifestJson="$( curl -fsSL \ -H "Authorization: Bearer $token" \ -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \ -H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \ - "https://registry-1.docker.io/v2/$image/manifests/$digest" + "$registryBase/v2/$image/manifests/$digest" )" if [ "${manifestJson:0:1}" != '{' ]; then echo >&2 "error: /v2/$image/manifests/$digest returned something unexpected:" @@ -87,10 +123,7 @@ while [ $# -gt 0 ]; do imageId="${configDigest#*:}" # strip off "sha256:" configFile="$imageId.json" - curl -fsSL \ - -H "Authorization: Bearer $token" \ - "https://registry-1.docker.io/v2/$image/blobs/$configDigest" \ - -o "$dir/$configFile" + fetch_blob "$token" "$image" "$configDigest" "$dir/$configFile" -s layersFs="$(echo "$manifestJson" | jq --raw-output --compact-output '.layers[]')" IFS="$newlineIFS" @@ -157,11 +190,8 @@ while [ $# -gt 0 ]; do echo "skipping existing ${layerId:0:12}" continue fi - token="$(curl -fsSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$image:pull" | jq --raw-output '.token')" - curl -fSL --progress \ - -H "Authorization: Bearer $token" \ - "https://registry-1.docker.io/v2/$image/blobs/$layerDigest" \ - -o "$dir/$layerTar" + token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')" + fetch_blob "$token" "$image" "$layerDigest" "$dir/$layerTar" --progress ;; *) @@ -230,8 +260,8 @@ while [ $# -gt 0 ]; do echo "skipping existing ${layerId:0:12}" continue fi - token="$(curl -fsSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$image:pull" | jq --raw-output '.token')" - curl -fSL --progress -H "Authorization: Bearer $token" "https://registry-1.docker.io/v2/$image/blobs/$imageLayer" -o "$dir/$layerId/layer.tar" # -C - + token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')" + fetch_blob "$token" "$image" "$imageLayer" "$dir/$layerId/layer.tar" --progress done ;;