Download support of images with multi-arch manifest
Currently we only support 'application/vnd.docker.distribution.manifest.v2+json' manifest images download, with more multi-arch images used, we need to support download images with 'application/vnd.docker.distribution.manifest.list.v2+json' format(aka "fat manifest"), else we will fail to download those multi-arch ones. This PR adds 'application/vnd.docker.distribution.manifest.list.v2+json' manifest support, thus we can download both multi-arch and legacy images. Signed-off-by: Dennis Chen <dennis.chen@arm.com>
This commit is contained in:
Dennis Chen
parent
e48e938fc2
commit
0af5db511e
2 changed files with 132 additions and 92 deletions
|
|
@ -6,7 +6,6 @@ set -eo pipefail
|
|||
|
||||
# debian latest f6fab3b798be 10 weeks ago 85.1 MB
|
||||
# debian latest f6fab3b798be3174f45aa1eb731f8182705555f89c9026d8c1ef230cbf8301dd 10 weeks ago 85.1 MB
|
||||
|
||||
if ! command -v curl &> /dev/null; then
|
||||
echo >&2 'error: "curl" not found!'
|
||||
exit 1
|
||||
|
|
@ -80,6 +79,109 @@ fetch_blob() {
|
|||
fi
|
||||
}
|
||||
|
||||
# handle 'application/vnd.docker.distribution.manifest.v2+json' manifest
|
||||
handle_single_manifest_v2() {
|
||||
local manifestJson="$1"; shift
|
||||
|
||||
local configDigest="$(echo "$manifestJson" | jq --raw-output '.config.digest')"
|
||||
local imageId="${configDigest#*:}" # strip off "sha256:"
|
||||
|
||||
local configFile="$imageId.json"
|
||||
fetch_blob "$token" "$image" "$configDigest" "$dir/$configFile" -s
|
||||
|
||||
local layersFs="$(echo "$manifestJson" | jq --raw-output --compact-output '.layers[]')"
|
||||
local IFS="$newlineIFS"
|
||||
local layers=( $layersFs )
|
||||
unset IFS
|
||||
|
||||
echo "Downloading '$imageIdentifier' (${#layers[@]} layers)..."
|
||||
local layerId=
|
||||
local layerFiles=()
|
||||
for i in "${!layers[@]}"; do
|
||||
local layerMeta="${layers[$i]}"
|
||||
|
||||
local layerMediaType="$(echo "$layerMeta" | jq --raw-output '.mediaType')"
|
||||
local layerDigest="$(echo "$layerMeta" | jq --raw-output '.digest')"
|
||||
|
||||
# save the previous layer's ID
|
||||
local parentId="$layerId"
|
||||
# create a new fake layer ID based on this layer's digest and the previous layer's fake ID
|
||||
layerId="$(echo "$parentId"$'\n'"$layerDigest" | sha256sum | cut -d' ' -f1)"
|
||||
# this accounts for the possibility that an image contains the same layer twice (and thus has a duplicate digest value)
|
||||
|
||||
mkdir -p "$dir/$layerId"
|
||||
echo '1.0' > "$dir/$layerId/VERSION"
|
||||
|
||||
if [ ! -s "$dir/$layerId/json" ]; then
|
||||
local parentJson="$(printf ', parent: "%s"' "$parentId")"
|
||||
local addJson="$(printf '{ id: "%s"%s }' "$layerId" "${parentId:+$parentJson}")"
|
||||
# this starter JSON is taken directly from Docker's own "docker save" output for unimportant layers
|
||||
jq "$addJson + ." > "$dir/$layerId/json" <<-'EOJSON'
|
||||
{
|
||||
"created": "0001-01-01T00:00:00Z",
|
||||
"container_config": {
|
||||
"Hostname": "",
|
||||
"Domainname": "",
|
||||
"User": "",
|
||||
"AttachStdin": false,
|
||||
"AttachStdout": false,
|
||||
"AttachStderr": false,
|
||||
"Tty": false,
|
||||
"OpenStdin": false,
|
||||
"StdinOnce": false,
|
||||
"Env": null,
|
||||
"Cmd": null,
|
||||
"Image": "",
|
||||
"Volumes": null,
|
||||
"WorkingDir": "",
|
||||
"Entrypoint": null,
|
||||
"OnBuild": null,
|
||||
"Labels": null
|
||||
}
|
||||
}
|
||||
EOJSON
|
||||
fi
|
||||
|
||||
case "$layerMediaType" in
|
||||
application/vnd.docker.image.rootfs.diff.tar.gzip)
|
||||
local layerTar="$layerId/layer.tar"
|
||||
layerFiles=( "${layerFiles[@]}" "$layerTar" )
|
||||
# TODO figure out why "-C -" doesn't work here
|
||||
# "curl: (33) HTTP server doesn't seem to support byte ranges. Cannot resume."
|
||||
# "HTTP/1.1 416 Requested Range Not Satisfiable"
|
||||
if [ -f "$dir/$layerTar" ]; then
|
||||
# TODO hackpatch for no -C support :'(
|
||||
echo "skipping existing ${layerId:0:12}"
|
||||
continue
|
||||
fi
|
||||
local token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')"
|
||||
fetch_blob "$token" "$image" "$layerDigest" "$dir/$layerTar" --progress
|
||||
;;
|
||||
|
||||
*)
|
||||
echo >&2 "error: unknown layer mediaType ($imageIdentifier, $layerDigest): '$layerMediaType'"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# change "$imageId" to be the ID of the last layer we added (needed for old-style "repositories" file which is created later -- specifically for older Docker daemons)
|
||||
imageId="$layerId"
|
||||
|
||||
# munge the top layer image manifest to have the appropriate image configuration for older daemons
|
||||
local imageOldConfig="$(jq --raw-output --compact-output '{ id: .id } + if .parent then { parent: .parent } else {} end' "$dir/$imageId/json")"
|
||||
jq --raw-output "$imageOldConfig + del(.history, .rootfs)" "$dir/$configFile" > "$dir/$imageId/json"
|
||||
|
||||
local manifestJsonEntry="$(
|
||||
echo '{}' | jq --raw-output '. + {
|
||||
Config: "'"$configFile"'",
|
||||
RepoTags: ["'"${image#library\/}:$tag"'"],
|
||||
Layers: '"$(echo '[]' | jq --raw-output ".$(for layerFile in "${layerFiles[@]}"; do echo " + [ \"$layerFile\" ]"; done)")"'
|
||||
}'
|
||||
)"
|
||||
manifestJsonEntries=( "${manifestJsonEntries[@]}" "$manifestJsonEntry" )
|
||||
}
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
imageTag="$1"
|
||||
shift
|
||||
|
|
@ -101,6 +203,7 @@ while [ $# -gt 0 ]; do
|
|||
curl -fsSL \
|
||||
-H "Authorization: Bearer $token" \
|
||||
-H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
|
||||
-H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \
|
||||
-H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \
|
||||
"$registryBase/v2/$image/manifests/$digest"
|
||||
)"
|
||||
|
|
@ -119,105 +222,40 @@ while [ $# -gt 0 ]; do
|
|||
|
||||
case "$mediaType" in
|
||||
application/vnd.docker.distribution.manifest.v2+json)
|
||||
configDigest="$(echo "$manifestJson" | jq --raw-output '.config.digest')"
|
||||
imageId="${configDigest#*:}" # strip off "sha256:"
|
||||
|
||||
configFile="$imageId.json"
|
||||
fetch_blob "$token" "$image" "$configDigest" "$dir/$configFile" -s
|
||||
|
||||
layersFs="$(echo "$manifestJson" | jq --raw-output --compact-output '.layers[]')"
|
||||
handle_single_manifest_v2 "$manifestJson"
|
||||
;;
|
||||
application/vnd.docker.distribution.manifest.list.v2+json)
|
||||
layersFs="$(echo "$manifestJson" | jq --raw-output --compact-output '.manifests[]')"
|
||||
IFS="$newlineIFS"
|
||||
layers=( $layersFs )
|
||||
unset IFS
|
||||
|
||||
echo "Downloading '$imageIdentifier' (${#layers[@]} layers)..."
|
||||
layerId=
|
||||
layerFiles=()
|
||||
found=""
|
||||
# parse first level multi-arch manifest
|
||||
for i in "${!layers[@]}"; do
|
||||
layerMeta="${layers[$i]}"
|
||||
|
||||
layerMediaType="$(echo "$layerMeta" | jq --raw-output '.mediaType')"
|
||||
layerDigest="$(echo "$layerMeta" | jq --raw-output '.digest')"
|
||||
|
||||
# save the previous layer's ID
|
||||
parentId="$layerId"
|
||||
# create a new fake layer ID based on this layer's digest and the previous layer's fake ID
|
||||
layerId="$(echo "$parentId"$'\n'"$layerDigest" | sha256sum | cut -d' ' -f1)"
|
||||
# this accounts for the possibility that an image contains the same layer twice (and thus has a duplicate digest value)
|
||||
|
||||
mkdir -p "$dir/$layerId"
|
||||
echo '1.0' > "$dir/$layerId/VERSION"
|
||||
|
||||
if [ ! -s "$dir/$layerId/json" ]; then
|
||||
parentJson="$(printf ', parent: "%s"' "$parentId")"
|
||||
addJson="$(printf '{ id: "%s"%s }' "$layerId" "${parentId:+$parentJson}")"
|
||||
# this starter JSON is taken directly from Docker's own "docker save" output for unimportant layers
|
||||
jq "$addJson + ." > "$dir/$layerId/json" <<-'EOJSON'
|
||||
{
|
||||
"created": "0001-01-01T00:00:00Z",
|
||||
"container_config": {
|
||||
"Hostname": "",
|
||||
"Domainname": "",
|
||||
"User": "",
|
||||
"AttachStdin": false,
|
||||
"AttachStdout": false,
|
||||
"AttachStderr": false,
|
||||
"Tty": false,
|
||||
"OpenStdin": false,
|
||||
"StdinOnce": false,
|
||||
"Env": null,
|
||||
"Cmd": null,
|
||||
"Image": "",
|
||||
"Volumes": null,
|
||||
"WorkingDir": "",
|
||||
"Entrypoint": null,
|
||||
"OnBuild": null,
|
||||
"Labels": null
|
||||
}
|
||||
}
|
||||
EOJSON
|
||||
maniArch="$(echo "$layerMeta" | jq --raw-output '.platform.architecture')"
|
||||
if [ "$maniArch" = "$(go env GOARCH)" ]; then
|
||||
digest="$(echo "$layerMeta" | jq --raw-output '.digest')"
|
||||
# get second level single manifest
|
||||
submanifestJson="$(
|
||||
curl -fsSL \
|
||||
-H "Authorization: Bearer $token" \
|
||||
-H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
|
||||
-H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \
|
||||
-H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \
|
||||
"$registryBase/v2/$image/manifests/$digest"
|
||||
)"
|
||||
handle_single_manifest_v2 "$submanifestJson"
|
||||
found="found"
|
||||
break
|
||||
fi
|
||||
|
||||
case "$layerMediaType" in
|
||||
application/vnd.docker.image.rootfs.diff.tar.gzip)
|
||||
layerTar="$layerId/layer.tar"
|
||||
layerFiles=( "${layerFiles[@]}" "$layerTar" )
|
||||
# TODO figure out why "-C -" doesn't work here
|
||||
# "curl: (33) HTTP server doesn't seem to support byte ranges. Cannot resume."
|
||||
# "HTTP/1.1 416 Requested Range Not Satisfiable"
|
||||
if [ -f "$dir/$layerTar" ]; then
|
||||
# TODO hackpatch for no -C support :'(
|
||||
echo "skipping existing ${layerId:0:12}"
|
||||
continue
|
||||
fi
|
||||
token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')"
|
||||
fetch_blob "$token" "$image" "$layerDigest" "$dir/$layerTar" --progress
|
||||
;;
|
||||
|
||||
*)
|
||||
echo >&2 "error: unknown layer mediaType ($imageIdentifier, $layerDigest): '$layerMediaType'"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# change "$imageId" to be the ID of the last layer we added (needed for old-style "repositories" file which is created later -- specifically for older Docker daemons)
|
||||
imageId="$layerId"
|
||||
|
||||
# munge the top layer image manifest to have the appropriate image configuration for older daemons
|
||||
imageOldConfig="$(jq --raw-output --compact-output '{ id: .id } + if .parent then { parent: .parent } else {} end' "$dir/$imageId/json")"
|
||||
jq --raw-output "$imageOldConfig + del(.history, .rootfs)" "$dir/$configFile" > "$dir/$imageId/json"
|
||||
|
||||
manifestJsonEntry="$(
|
||||
echo '{}' | jq --raw-output '. + {
|
||||
Config: "'"$configFile"'",
|
||||
RepoTags: ["'"${image#library\/}:$tag"'"],
|
||||
Layers: '"$(echo '[]' | jq --raw-output ".$(for layerFile in "${layerFiles[@]}"; do echo " + [ \"$layerFile\" ]"; done)")"'
|
||||
}'
|
||||
)"
|
||||
manifestJsonEntries=( "${manifestJsonEntries[@]}" "$manifestJsonEntry" )
|
||||
if [ -z "$found" ]; then
|
||||
echo >&2 "error: manifest for $maniArch is not found"
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
|
||||
*)
|
||||
echo >&2 "error: unknown manifest mediaType ($imageIdentifier): '$mediaType'"
|
||||
exit 1
|
||||
|
|
|
|||
|
|
@ -64,7 +64,9 @@ case "$PACKAGE_ARCH" in
|
|||
;;
|
||||
*)
|
||||
DOCKERFILE="Dockerfile.$PACKAGE_ARCH"
|
||||
TEST_IMAGE_NAMESPACE="$PACKAGE_ARCH"
|
||||
if [ "$PACKAGE_ARCH" != "aarch64" ]; then
|
||||
TEST_IMAGE_NAMESPACE="$PACKAGE_ARCH"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
export DOCKERFILE TEST_IMAGE_NAMESPACE
|
||||
|
|
|
|||
Loading…
Reference in a new issue