From 337ba71fc1124603302e28d94e2f08674e31a756 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Tue, 26 Jun 2018 14:49:33 -0700 Subject: [PATCH] distribution: fix passing platform struct to puller Signed-off-by: Tonis Tiigi --- api/server/router/image/backend.go | 3 +- api/server/router/image/image_routes.go | 10 ++- api/types/backend/build.go | 3 +- builder/dockerfile/copy.go | 10 ++- builder/dockerfile/dispatchers.go | 84 +++++++++----------- builder/dockerfile/dispatchers_test.go | 7 +- builder/dockerfile/imagecontext.go | 11 +-- builder/dockerfile/internals.go | 7 +- daemon/cluster/executor/backend.go | 3 +- daemon/cluster/executor/container/adapter.go | 4 +- daemon/images/image_builder.go | 16 ++-- daemon/images/image_pull.go | 9 ++- distribution/config.go | 7 +- distribution/pull.go | 5 +- distribution/pull_v1.go | 3 +- distribution/pull_v2.go | 82 +++++++++++-------- distribution/pull_v2_unix.go | 62 +++++++++++++-- distribution/pull_v2_windows.go | 7 +- distribution/registry_unit_test.go | 3 +- 19 files changed, 208 insertions(+), 128 deletions(-) diff --git a/api/server/router/image/backend.go b/api/server/router/image/backend.go index 93c47cf638..5837f9a9bc 100644 --- a/api/server/router/image/backend.go +++ b/api/server/router/image/backend.go @@ -8,6 +8,7 @@ import ( "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/image" "github.com/docker/docker/api/types/registry" + specs "github.com/opencontainers/image-spec/specs-go/v1" ) // Backend is all the methods that need to be implemented @@ -34,7 +35,7 @@ type importExportBackend interface { } type registryBackend interface { - PullImage(ctx context.Context, image, tag, platform string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error + PullImage(ctx context.Context, image, tag string, platform *specs.Platform, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error PushImage(ctx context.Context, image, tag string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error SearchRegistryForImages(ctx context.Context, filtersArgs string, term string, limit int, authConfig *types.AuthConfig, metaHeaders map[string][]string) (*registry.SearchResults, error) } diff --git a/api/server/router/image/image_routes.go b/api/server/router/image/image_routes.go index 42db0a14e7..9490d7926f 100644 --- a/api/server/router/image/image_routes.go +++ b/api/server/router/image/image_routes.go @@ -35,7 +35,7 @@ func (s *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWrite message = r.Form.Get("message") err error output = ioutils.NewWriteFlusher(w) - platform = &specs.Platform{} + platform *specs.Platform ) defer output.Close() @@ -72,13 +72,17 @@ func (s *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWrite authConfig = &types.AuthConfig{} } } - err = s.backend.PullImage(ctx, image, tag, platform.OS, metaHeaders, authConfig, output) + err = s.backend.PullImage(ctx, image, tag, platform, metaHeaders, authConfig, output) } else { //import src := r.Form.Get("fromSrc") // 'err' MUST NOT be defined within this block, we need any error // generated from the download to be available to the output // stream processing below - err = s.backend.ImportImage(src, repo, platform.OS, tag, message, r.Body, output, r.Form["changes"]) + os := "" + if platform != nil { + os = platform.OS + } + err = s.backend.ImportImage(src, repo, os, tag, message, r.Body, output, r.Form["changes"]) } } if err != nil { diff --git a/api/types/backend/build.go b/api/types/backend/build.go index 31e00ec6ce..1a2e59f2f7 100644 --- a/api/types/backend/build.go +++ b/api/types/backend/build.go @@ -5,6 +5,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/pkg/streamformatter" + specs "github.com/opencontainers/image-spec/specs-go/v1" ) // PullOption defines different modes for accessing images @@ -40,5 +41,5 @@ type GetImageAndLayerOptions struct { PullOption PullOption AuthConfig map[string]types.AuthConfig Output io.Writer - OS string + Platform *specs.Platform } diff --git a/builder/dockerfile/copy.go b/builder/dockerfile/copy.go index 37ee4006df..7e9dc6036a 100644 --- a/builder/dockerfile/copy.go +++ b/builder/dockerfile/copy.go @@ -96,8 +96,14 @@ func (o *copier) createCopyInstruction(args []string, cmdName string) (copyInstr last := len(args) - 1 // Work in platform-specific filepath semantics - inst.dest = fromSlash(args[last], o.platform.OS) - separator := string(separator(o.platform.OS)) + // TODO: This OS switch for paths is NOT correct and should not be supported. + // Maintained for backwards compatibility + pathOS := runtime.GOOS + if o.platform != nil { + pathOS = o.platform.OS + } + inst.dest = fromSlash(args[last], pathOS) + separator := string(separator(pathOS)) infos, err := o.getCopyInfosForSourcePaths(args[0:last], inst.dest) if err != nil { return inst, errors.Wrapf(err, "%s failed", cmdName) diff --git a/builder/dockerfile/dispatchers.go b/builder/dockerfile/dispatchers.go index c30c010ece..5372d2f0f8 100644 --- a/builder/dockerfile/dispatchers.go +++ b/builder/dockerfile/dispatchers.go @@ -28,6 +28,7 @@ import ( "github.com/moby/buildkit/frontend/dockerfile/instructions" "github.com/moby/buildkit/frontend/dockerfile/parser" "github.com/moby/buildkit/frontend/dockerfile/shell" + specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) @@ -103,7 +104,7 @@ func dispatchAdd(d dispatchRequest, c *instructions.AddCommand) error { copyInstruction.chownStr = c.Chown copyInstruction.allowLocalDecompression = true - return d.builder.performCopy(d.state, copyInstruction) + return d.builder.performCopy(d, copyInstruction) } // COPY foo /path @@ -127,7 +128,7 @@ func dispatchCopy(d dispatchRequest, c *instructions.CopyCommand) error { } copyInstruction.chownStr = c.Chown - return d.builder.performCopy(d.state, copyInstruction) + return d.builder.performCopy(d, copyInstruction) } func (d *dispatchRequest) getImageMount(imageRefOrID string) (*imageMount, error) { @@ -145,7 +146,7 @@ func (d *dispatchRequest) getImageMount(imageRefOrID string) (*imageMount, error imageRefOrID = stage.Image localOnly = true } - return d.builder.imageSources.Get(imageRefOrID, localOnly, d.state.operatingSystem) + return d.builder.imageSources.Get(imageRefOrID, localOnly, d.builder.options.Platform) } // FROM [--platform=platform] imagename[:tag | @digest] [AS build-stage-name] @@ -153,22 +154,21 @@ func (d *dispatchRequest) getImageMount(imageRefOrID string) (*imageMount, error func initializeStage(d dispatchRequest, cmd *instructions.Stage) error { d.builder.imageProber.Reset() - // TODO: pass *platform instead, allow autodetect - platform := platforms.DefaultSpec() + var platform *specs.Platform if v := cmd.Platform; v != "" { - // TODO: - // v, err := shlex.ProcessWord(v, toEnvList(metaArgs, nil)) - // if err != nil { - // return nil, nil, errors.Wrapf(err, "failed to process arguments for platform %s", v) - // } + v, err := d.getExpandedString(d.shlex, v) + if err != nil { + return errors.Wrapf(err, "failed to process arguments for platform %s", v) + } p, err := platforms.Parse(v) if err != nil { return errors.Wrapf(err, "failed to parse platform %s", v) } - platform = p + platform = &p } - image, err := d.getFromImage(d.shlex, cmd.BaseName, platform.OS) + + image, err := d.getFromImage(d.shlex, cmd.BaseName, platform) if err != nil { return err } @@ -214,82 +214,72 @@ func dispatchTriggeredOnBuild(d dispatchRequest, triggers []string) error { return nil } -func (d *dispatchRequest) getExpandedImageName(shlex *shell.Lex, name string) (string, error) { +func (d *dispatchRequest) getExpandedString(shlex *shell.Lex, str string) (string, error) { substitutionArgs := []string{} for key, value := range d.state.buildArgs.GetAllMeta() { substitutionArgs = append(substitutionArgs, key+"="+value) } - name, err := shlex.ProcessWord(name, substitutionArgs) + name, err := shlex.ProcessWord(str, substitutionArgs) if err != nil { return "", err } return name, nil } -// getOsFromFlagsAndStage calculates the operating system if we need to pull an image. -// stagePlatform contains the value supplied by optional `--platform=` on -// a current FROM statement. b.builder.options.Platform contains the operating -// system part of the optional flag passed in the API call (or CLI flag -// through `docker build --platform=...`). Precedence is for an explicit -// platform indication in the FROM statement. -func (d *dispatchRequest) getOsFromFlagsAndStage(stageOS string) string { - switch { - case stageOS != "": - return stageOS - case d.builder.options.Platform.OS != "": - // Note this is API "platform", but by this point, as the daemon is not - // multi-arch aware yet, it is guaranteed to only hold the OS part here. - return d.builder.options.Platform.OS - default: - return "" // Auto-select - } -} - -func (d *dispatchRequest) getImageOrStage(name string, stageOS string) (builder.Image, error) { +func (d *dispatchRequest) getImageOrStage(name string, platform *specs.Platform) (builder.Image, error) { var localOnly bool if im, ok := d.stages.getByName(name); ok { name = im.Image localOnly = true } - os := d.getOsFromFlagsAndStage(stageOS) + if platform == nil { + platform = d.builder.options.Platform + } // Windows cannot support a container with no base image unless it is LCOW. if name == api.NoBaseImageSpecifier { + p := platforms.DefaultSpec() + if platform != nil { + p = *platform + } imageImage := &image.Image{} - imageImage.OS = runtime.GOOS + imageImage.OS = p.OS + + // old windows scratch handling + // TODO: scratch should not have an os. It should be nil image. + // Windows supports scratch. What is not supported is running containers + // from it. if runtime.GOOS == "windows" { - switch os { - case "windows": - return nil, errors.New("Windows does not support FROM scratch") - case "linux", "": + if platform == nil || platform.OS == "linux" { if !system.LCOWSupported() { return nil, errors.New("Linux containers are not supported on this system") } imageImage.OS = "linux" - default: - return nil, errors.Errorf("operating system %q is not supported", os) + } else if platform.OS == "windows" { + return nil, errors.New("Windows does not support FROM scratch") + } else { + return nil, errors.Errorf("platform %s is not supported", platforms.Format(p)) } } return builder.Image(imageImage), nil } - imageMount, err := d.builder.imageSources.Get(name, localOnly, os) + imageMount, err := d.builder.imageSources.Get(name, localOnly, platform) if err != nil { return nil, err } return imageMount.Image(), nil } -func (d *dispatchRequest) getFromImage(shlex *shell.Lex, name string, stageOS string) (builder.Image, error) { - name, err := d.getExpandedImageName(shlex, name) +func (d *dispatchRequest) getFromImage(shlex *shell.Lex, name string, platform *specs.Platform) (builder.Image, error) { + name, err := d.getExpandedString(shlex, name) if err != nil { return nil, err } - return d.getImageOrStage(name, stageOS) + return d.getImageOrStage(name, platform) } func dispatchOnbuild(d dispatchRequest, c *instructions.OnbuildCommand) error { - d.state.runConfig.OnBuild = append(d.state.runConfig.OnBuild, c.Expression) return d.builder.commit(d.state, "ONBUILD "+c.Expression) } diff --git a/builder/dockerfile/dispatchers_test.go b/builder/dockerfile/dispatchers_test.go index 36d20a1a82..047a8742e8 100644 --- a/builder/dockerfile/dispatchers_test.go +++ b/builder/dockerfile/dispatchers_test.go @@ -6,6 +6,7 @@ import ( "runtime" "testing" + "github.com/containerd/containerd/platforms" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/backend" "github.com/docker/docker/api/types/container" @@ -22,15 +23,17 @@ import ( func newBuilderWithMockBackend() *Builder { mockBackend := &MockBackend{} + defaultPlatform := platforms.DefaultSpec() + opts := &types.ImageBuildOptions{Platform: &defaultPlatform} ctx := context.Background() b := &Builder{ - options: &types.ImageBuildOptions{Platform: runtime.GOOS}, + options: opts, docker: mockBackend, Stdout: new(bytes.Buffer), clientCtx: ctx, disableCommit: true, imageSources: newImageSources(ctx, builderOptions{ - Options: &types.ImageBuildOptions{Platform: runtime.GOOS}, + Options: opts, Backend: mockBackend, }), imageProber: newImageProber(mockBackend, nil, false), diff --git a/builder/dockerfile/imagecontext.go b/builder/dockerfile/imagecontext.go index 53a4b9774b..08cb396a2b 100644 --- a/builder/dockerfile/imagecontext.go +++ b/builder/dockerfile/imagecontext.go @@ -7,11 +7,12 @@ import ( "github.com/docker/docker/api/types/backend" "github.com/docker/docker/builder" dockerimage "github.com/docker/docker/image" + specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) -type getAndMountFunc func(string, bool, string) (builder.Image, builder.ROLayer, error) +type getAndMountFunc func(string, bool, *specs.Platform) (builder.Image, builder.ROLayer, error) // imageSources mounts images and provides a cache for mounted images. It tracks // all images so they can be unmounted at the end of the build. @@ -22,7 +23,7 @@ type imageSources struct { } func newImageSources(ctx context.Context, options builderOptions) *imageSources { - getAndMount := func(idOrRef string, localOnly bool, osForPull string) (builder.Image, builder.ROLayer, error) { + getAndMount := func(idOrRef string, localOnly bool, platform *specs.Platform) (builder.Image, builder.ROLayer, error) { pullOption := backend.PullOptionNoPull if !localOnly { if options.Options.PullParent { @@ -35,7 +36,7 @@ func newImageSources(ctx context.Context, options builderOptions) *imageSources PullOption: pullOption, AuthConfig: options.Options.AuthConfigs, Output: options.ProgressWriter.Output, - OS: osForPull, + Platform: platform, }) } @@ -45,12 +46,12 @@ func newImageSources(ctx context.Context, options builderOptions) *imageSources } } -func (m *imageSources) Get(idOrRef string, localOnly bool, osForPull string) (*imageMount, error) { +func (m *imageSources) Get(idOrRef string, localOnly bool, platform *specs.Platform) (*imageMount, error) { if im, ok := m.byImageID[idOrRef]; ok { return im, nil } - image, layer, err := m.getImage(idOrRef, localOnly, osForPull) + image, layer, err := m.getImage(idOrRef, localOnly, platform) if err != nil { return nil, err } diff --git a/builder/dockerfile/internals.go b/builder/dockerfile/internals.go index e23c328001..5e2c286d75 100644 --- a/builder/dockerfile/internals.go +++ b/builder/dockerfile/internals.go @@ -150,7 +150,8 @@ func (b *Builder) exportImage(state *dispatchState, layer builder.RWLayer, paren return nil } -func (b *Builder) performCopy(state *dispatchState, inst copyInstruction) error { +func (b *Builder) performCopy(req dispatchRequest, inst copyInstruction) error { + state := req.state srcHash := getSourceHashFromInfos(inst.infos) var chownComment string @@ -168,7 +169,7 @@ func (b *Builder) performCopy(state *dispatchState, inst copyInstruction) error return err } - imageMount, err := b.imageSources.Get(state.imageID, true, state.operatingSystem) + imageMount, err := b.imageSources.Get(state.imageID, true, req.builder.options.Platform) if err != nil { return errors.Wrapf(err, "failed to get destination image %q", state.imageID) } @@ -456,7 +457,7 @@ func hostConfigFromOptions(options *types.ImageBuildOptions) *container.HostConf // is too small for builder scenarios where many users are // using RUN statements to install large amounts of data. // Use 127GB as that's the default size of a VHD in Hyper-V. - if runtime.GOOS == "windows" && options.Platform.OS == "windows" { + if runtime.GOOS == "windows" && options.Platform != nil && options.Platform.OS == "windows" { hc.StorageOpt = make(map[string]string) hc.StorageOpt["size"] = "127GB" } diff --git a/daemon/cluster/executor/backend.go b/daemon/cluster/executor/backend.go index 1f2312ab40..cfbc86ce36 100644 --- a/daemon/cluster/executor/backend.go +++ b/daemon/cluster/executor/backend.go @@ -23,6 +23,7 @@ import ( "github.com/docker/libnetwork/cluster" networktypes "github.com/docker/libnetwork/types" "github.com/docker/swarmkit/agent/exec" + specs "github.com/opencontainers/image-spec/specs-go/v1" ) // Backend defines the executor component for a swarm agent. @@ -69,7 +70,7 @@ type VolumeBackend interface { // ImageBackend is used by an executor to perform image operations type ImageBackend interface { - PullImage(ctx context.Context, image, tag, platform string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error + PullImage(ctx context.Context, image, tag string, platform *specs.Platform, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error GetRepository(context.Context, reference.Named, *types.AuthConfig) (distribution.Repository, bool, error) LookupImage(name string) (*types.ImageInspect, error) } diff --git a/daemon/cluster/executor/container/adapter.go b/daemon/cluster/executor/container/adapter.go index fdf1ee2ec7..6b222f202d 100644 --- a/daemon/cluster/executor/container/adapter.go +++ b/daemon/cluster/executor/container/adapter.go @@ -8,7 +8,6 @@ import ( "fmt" "io" "os" - "runtime" "strings" "syscall" "time" @@ -97,8 +96,7 @@ func (c *containerAdapter) pullImage(ctx context.Context) error { go func() { // TODO @jhowardmsft LCOW Support: This will need revisiting as // the stack is built up to include LCOW support for swarm. - platform := runtime.GOOS - err := c.imageBackend.PullImage(ctx, c.container.image(), "", platform, metaHeaders, authConfig, pw) + err := c.imageBackend.PullImage(ctx, c.container.image(), "", nil, metaHeaders, authConfig, pw) pw.CloseWithError(err) }() diff --git a/daemon/images/image_builder.go b/daemon/images/image_builder.go index b792721ee4..cdf951c6f5 100644 --- a/daemon/images/image_builder.go +++ b/daemon/images/image_builder.go @@ -3,6 +3,7 @@ package images // import "github.com/docker/docker/daemon/images" import ( "context" "io" + "runtime" "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" @@ -14,6 +15,7 @@ import ( "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/pkg/system" "github.com/docker/docker/registry" + specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) @@ -137,7 +139,7 @@ func newROLayerForImage(img *image.Image, layerStore layer.Store) (builder.ROLay } // TODO: could this use the regular daemon PullImage ? -func (i *ImageService) pullForBuilder(ctx context.Context, name string, authConfigs map[string]types.AuthConfig, output io.Writer, os string) (*image.Image, error) { +func (i *ImageService) pullForBuilder(ctx context.Context, name string, authConfigs map[string]types.AuthConfig, output io.Writer, platform *specs.Platform) (*image.Image, error) { ref, err := reference.ParseNormalizedNamed(name) if err != nil { return nil, err @@ -156,7 +158,7 @@ func (i *ImageService) pullForBuilder(ctx context.Context, name string, authConf pullRegistryAuth = &resolvedConfig } - if err := i.pullImageWithReference(ctx, ref, os, nil, pullRegistryAuth, output); err != nil { + if err := i.pullImageWithReference(ctx, ref, platform, nil, pullRegistryAuth, output); err != nil { return nil, err } return i.GetImage(name) @@ -167,10 +169,14 @@ func (i *ImageService) pullForBuilder(ctx context.Context, name string, authConf // leaking of layers. func (i *ImageService) GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ROLayer, error) { if refOrID == "" { // ie FROM scratch - if !system.IsOSSupported(opts.OS) { + os := runtime.GOOS + if opts.Platform != nil { + os = opts.Platform.OS + } + if !system.IsOSSupported(os) { return nil, nil, system.ErrNotSupportedOperatingSystem } - layer, err := newROLayerForImage(nil, i.layerStores[opts.OS]) + layer, err := newROLayerForImage(nil, i.layerStores[os]) return nil, layer, err } @@ -189,7 +195,7 @@ func (i *ImageService) GetImageAndReleasableLayer(ctx context.Context, refOrID s } } - image, err := i.pullForBuilder(ctx, refOrID, opts.AuthConfig, opts.Output, opts.OS) + image, err := i.pullForBuilder(ctx, refOrID, opts.AuthConfig, opts.Output, opts.Platform) if err != nil { return nil, nil, err } diff --git a/daemon/images/image_pull.go b/daemon/images/image_pull.go index ed8ecb9abe..3e6b433037 100644 --- a/daemon/images/image_pull.go +++ b/daemon/images/image_pull.go @@ -15,11 +15,12 @@ import ( "github.com/docker/docker/pkg/progress" "github.com/docker/docker/registry" "github.com/opencontainers/go-digest" + specs "github.com/opencontainers/image-spec/specs-go/v1" ) // PullImage initiates a pull operation. image is the repository name to pull, and // tag may be either empty, or indicate a specific tag to pull. -func (i *ImageService) PullImage(ctx context.Context, image, tag, os string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { +func (i *ImageService) PullImage(ctx context.Context, image, tag string, platform *specs.Platform, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { start := time.Now() // Special case: "pull -a" may send an image name with a // trailing :. This is ugly, but let's not break API @@ -45,12 +46,12 @@ func (i *ImageService) PullImage(ctx context.Context, image, tag, os string, met } } - err = i.pullImageWithReference(ctx, ref, os, metaHeaders, authConfig, outStream) + err = i.pullImageWithReference(ctx, ref, platform, metaHeaders, authConfig, outStream) imageActions.WithValues("pull").UpdateSince(start) return err } -func (i *ImageService) pullImageWithReference(ctx context.Context, ref reference.Named, os string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { +func (i *ImageService) pullImageWithReference(ctx context.Context, ref reference.Named, platform *specs.Platform, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { // Include a buffer so that slow client connections don't affect // transfer performance. progressChan := make(chan progress.Progress, 100) @@ -77,7 +78,7 @@ func (i *ImageService) pullImageWithReference(ctx context.Context, ref reference }, DownloadManager: i.downloadManager, Schema2Types: distribution.ImageTypes, - OS: os, + Platform: platform, } err := distribution.Pull(ctx, ref, imagePullConfig) diff --git a/distribution/config.go b/distribution/config.go index 55f1f8c2df..438051c296 100644 --- a/distribution/config.go +++ b/distribution/config.go @@ -60,9 +60,8 @@ type ImagePullConfig struct { // Schema2Types is the valid schema2 configuration types allowed // by the pull operation. Schema2Types []string - // OS is the requested operating system of the image being pulled to ensure it can be validated - // when the host OS supports multiple image operating systems. - OS string + // Platform is the requested platform of the image being pulled + Platform *specs.Platform } // ImagePushConfig stores push configuration. @@ -171,7 +170,7 @@ func (s *imageConfigStore) PlatformFromConfig(c []byte) (*specs.Platform, error) if !system.IsOSSupported(os) { return nil, system.ErrNotSupportedOperatingSystem } - return &specs.Platform{OS: os, OSVersion: unmarshalledConfig.OSVersion}, nil + return &specs.Platform{OS: os, Architecture: unmarshalledConfig.Architecture, OSVersion: unmarshalledConfig.OSVersion}, nil } type storeLayerProvider struct { diff --git a/distribution/pull.go b/distribution/pull.go index 0c5237f6c0..5de73ae99a 100644 --- a/distribution/pull.go +++ b/distribution/pull.go @@ -11,6 +11,7 @@ import ( refstore "github.com/docker/docker/reference" "github.com/docker/docker/registry" "github.com/opencontainers/go-digest" + specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -20,7 +21,7 @@ type Puller interface { // Pull tries to pull the image referenced by `tag` // Pull returns an error if any, as well as a boolean that determines whether to retry Pull on the next configured endpoint. // - Pull(ctx context.Context, ref reference.Named, os string) error + Pull(ctx context.Context, ref reference.Named, platform *specs.Platform) error } // newPuller returns a Puller interface that will pull from either a v1 or v2 @@ -114,7 +115,7 @@ func Pull(ctx context.Context, ref reference.Named, imagePullConfig *ImagePullCo continue } - if err := puller.Pull(ctx, ref, imagePullConfig.OS); err != nil { + if err := puller.Pull(ctx, ref, imagePullConfig.Platform); err != nil { // Was this pull cancelled? If so, don't try to fall // back. fallback := false diff --git a/distribution/pull_v1.go b/distribution/pull_v1.go index c26d881223..c2c897dc1c 100644 --- a/distribution/pull_v1.go +++ b/distribution/pull_v1.go @@ -25,6 +25,7 @@ import ( "github.com/docker/docker/pkg/progress" "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/registry" + specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sirupsen/logrus" ) @@ -36,7 +37,7 @@ type v1Puller struct { session *registry.Session } -func (p *v1Puller) Pull(ctx context.Context, ref reference.Named, os string) error { +func (p *v1Puller) Pull(ctx context.Context, ref reference.Named, _ *specs.Platform) error { if _, isCanonical := ref.(reference.Canonical); isCanonical { // Allowing fallback, because HTTPS v1 is before HTTP v2 return fallbackError{err: ErrNoSupport{Err: errors.New("Cannot pull by digest with v1 registry")}} diff --git a/distribution/pull_v2.go b/distribution/pull_v2.go index 9e07c88d19..8f05cfa0b2 100644 --- a/distribution/pull_v2.go +++ b/distribution/pull_v2.go @@ -11,6 +11,7 @@ import ( "runtime" "strings" + "github.com/containerd/containerd/platforms" "github.com/docker/distribution" "github.com/docker/distribution/manifest/manifestlist" "github.com/docker/distribution/manifest/schema1" @@ -63,7 +64,7 @@ type v2Puller struct { confirmedV2 bool } -func (p *v2Puller) Pull(ctx context.Context, ref reference.Named, os string) (err error) { +func (p *v2Puller) Pull(ctx context.Context, ref reference.Named, platform *specs.Platform) (err error) { // TODO(tiborvass): was ReceiveTimeout p.repo, p.confirmedV2, err = NewV2Repository(ctx, p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "pull") if err != nil { @@ -71,7 +72,7 @@ func (p *v2Puller) Pull(ctx context.Context, ref reference.Named, os string) (er return err } - if err = p.pullV2Repository(ctx, ref, os); err != nil { + if err = p.pullV2Repository(ctx, ref, platform); err != nil { if _, ok := err.(fallbackError); ok { return err } @@ -86,10 +87,10 @@ func (p *v2Puller) Pull(ctx context.Context, ref reference.Named, os string) (er return err } -func (p *v2Puller) pullV2Repository(ctx context.Context, ref reference.Named, os string) (err error) { +func (p *v2Puller) pullV2Repository(ctx context.Context, ref reference.Named, platform *specs.Platform) (err error) { var layersDownloaded bool if !reference.IsNameOnly(ref) { - layersDownloaded, err = p.pullV2Tag(ctx, ref, os) + layersDownloaded, err = p.pullV2Tag(ctx, ref, platform) if err != nil { return err } @@ -111,7 +112,7 @@ func (p *v2Puller) pullV2Repository(ctx context.Context, ref reference.Named, os if err != nil { return err } - pulledNew, err := p.pullV2Tag(ctx, tagRef, os) + pulledNew, err := p.pullV2Tag(ctx, tagRef, platform) if err != nil { // Since this is the pull-all-tags case, don't // allow an error pulling a particular tag to @@ -327,7 +328,7 @@ func (ld *v2LayerDescriptor) Registered(diffID layer.DiffID) { ld.V2MetadataService.Add(diffID, metadata.V2Metadata{Digest: ld.digest, SourceRepository: ld.repoInfo.Name.Name()}) } -func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named, os string) (tagUpdated bool, err error) { +func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named, platform *specs.Platform) (tagUpdated bool, err error) { manSvc, err := p.repo.Manifests(ctx) if err != nil { return false, err @@ -391,17 +392,17 @@ func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named, os string if p.config.RequireSchema2 { return false, fmt.Errorf("invalid manifest: not schema2") } - id, manifestDigest, err = p.pullSchema1(ctx, ref, v, os) + id, manifestDigest, err = p.pullSchema1(ctx, ref, v, platform) if err != nil { return false, err } case *schema2.DeserializedManifest: - id, manifestDigest, err = p.pullSchema2(ctx, ref, v, os) + id, manifestDigest, err = p.pullSchema2(ctx, ref, v, platform) if err != nil { return false, err } case *manifestlist.DeserializedManifestList: - id, manifestDigest, err = p.pullManifestList(ctx, ref, v, os) + id, manifestDigest, err = p.pullManifestList(ctx, ref, v, platform) if err != nil { return false, err } @@ -437,7 +438,7 @@ func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named, os string return true, nil } -func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Reference, unverifiedManifest *schema1.SignedManifest, requestedOS string) (id digest.Digest, manifestDigest digest.Digest, err error) { +func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Reference, unverifiedManifest *schema1.SignedManifest, platform *specs.Platform) (id digest.Digest, manifestDigest digest.Digest, err error) { var verifiedManifest *schema1.Manifest verifiedManifest, err = verifySchema1Manifest(unverifiedManifest, ref) if err != nil { @@ -513,7 +514,10 @@ func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Reference, unv // we support the operating system, switch to that operating system. // eg FROM supertest2014/nyan with no platform specifier, and docker build // with no --platform= flag under LCOW. - if requestedOS == "" && system.IsOSSupported(configOS) { + requestedOS := "" + if platform != nil { + requestedOS = platform.OS + } else if system.IsOSSupported(configOS) { requestedOS = configOS } @@ -544,7 +548,7 @@ func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Reference, unv return imageID, manifestDigest, nil } -func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *schema2.DeserializedManifest, requestedOS string) (id digest.Digest, manifestDigest digest.Digest, err error) { +func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *schema2.DeserializedManifest, platform *specs.Platform) (id digest.Digest, manifestDigest digest.Digest, err error) { manifestDigest, err = schema2ManifestDigest(ref, mfst) if err != nil { return "", "", err @@ -600,6 +604,11 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s configPlatform *specs.Platform // for LCOW when registering downloaded layers ) + layerStoreOS := runtime.GOOS + if platform != nil { + layerStoreOS = platform.OS + } + // https://github.com/docker/docker/issues/24766 - Err on the side of caution, // explicitly blocking images intended for linux from the Windows daemon. On // Windows, we do this before the attempt to download, effectively serialising @@ -623,13 +632,14 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s if len(descriptors) != len(configRootFS.DiffIDs) { return "", "", errRootFSMismatch } - - // Early bath if the requested OS doesn't match that of the configuration. - // This avoids doing the download, only to potentially fail later. - if !system.IsOSSupported(configPlatform.OS) { - return "", "", fmt.Errorf("cannot download image with operating system %q when requesting %q", configPlatform.OS, requestedOS) + if platform == nil { + // Early bath if the requested OS doesn't match that of the configuration. + // This avoids doing the download, only to potentially fail later. + if !system.IsOSSupported(configPlatform.OS) { + return "", "", fmt.Errorf("cannot download image with operating system %q when requesting %q", configPlatform.OS, layerStoreOS) + } + layerStoreOS = configPlatform.OS } - requestedOS = configPlatform.OS // Populate diff ids in descriptors to avoid downloading foreign layers // which have been side loaded @@ -638,10 +648,6 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s } } - if requestedOS == "" { - requestedOS = runtime.GOOS - } - if p.config.DownloadManager != nil { go func() { var ( @@ -649,7 +655,7 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s rootFS image.RootFS ) downloadRootFS := *image.NewRootFS() - rootFS, release, err = p.config.DownloadManager.Download(ctx, downloadRootFS, requestedOS, descriptors, p.config.ProgressOutput) + rootFS, release, err = p.config.DownloadManager.Download(ctx, downloadRootFS, layerStoreOS, descriptors, p.config.ProgressOutput) if err != nil { // Intentionally do not cancel the config download here // as the error from config download (if there is one) @@ -735,22 +741,22 @@ func receiveConfig(s ImageConfigStore, configChan <-chan []byte, errChan <-chan // pullManifestList handles "manifest lists" which point to various // platform-specific manifests. -func (p *v2Puller) pullManifestList(ctx context.Context, ref reference.Named, mfstList *manifestlist.DeserializedManifestList, requestedOS string) (id digest.Digest, manifestListDigest digest.Digest, err error) { +func (p *v2Puller) pullManifestList(ctx context.Context, ref reference.Named, mfstList *manifestlist.DeserializedManifestList, pp *specs.Platform) (id digest.Digest, manifestListDigest digest.Digest, err error) { manifestListDigest, err = schema2ManifestDigest(ref, mfstList) if err != nil { return "", "", err } - logOS := requestedOS // May be "" indicating any OS - if logOS == "" { - logOS = "*" + var platform specs.Platform + if pp != nil { + platform = *pp } - logrus.Debugf("%s resolved to a manifestList object with %d entries; looking for a %s/%s match", ref, len(mfstList.Manifests), logOS, runtime.GOARCH) + logrus.Debugf("%s resolved to a manifestList object with %d entries; looking for a %s/%s match", ref, len(mfstList.Manifests), platforms.Format(platform), runtime.GOARCH) - manifestMatches := filterManifests(mfstList.Manifests, requestedOS) + manifestMatches := filterManifests(mfstList.Manifests, platform) if len(manifestMatches) == 0 { - errMsg := fmt.Sprintf("no matching manifest for %s/%s in the manifest list entries", logOS, runtime.GOARCH) + errMsg := fmt.Sprintf("no matching manifest for %s in the manifest list entries", platforms.Format(platform)) logrus.Debugf(errMsg) return "", "", errors.New(errMsg) } @@ -781,12 +787,14 @@ func (p *v2Puller) pullManifestList(ctx context.Context, ref reference.Named, mf switch v := manifest.(type) { case *schema1.SignedManifest: - id, _, err = p.pullSchema1(ctx, manifestRef, v, manifestMatches[0].Platform.OS) + platform := toOCIPlatform(manifestMatches[0].Platform) + id, _, err = p.pullSchema1(ctx, manifestRef, v, &platform) if err != nil { return "", "", err } case *schema2.DeserializedManifest: - id, _, err = p.pullSchema2(ctx, manifestRef, v, manifestMatches[0].Platform.OS) + platform := toOCIPlatform(manifestMatches[0].Platform) + id, _, err = p.pullSchema2(ctx, manifestRef, v, &platform) if err != nil { return "", "", err } @@ -956,3 +964,13 @@ func fixManifestLayers(m *schema1.Manifest) error { func createDownloadFile() (*os.File, error) { return ioutil.TempFile("", "GetImageBlob") } + +func toOCIPlatform(p manifestlist.PlatformSpec) specs.Platform { + return specs.Platform{ + OS: p.OS, + Architecture: p.Architecture, + Variant: p.Variant, + OSFeatures: p.OSFeatures, + OSVersion: p.OSVersion, + } +} diff --git a/distribution/pull_v2_unix.go b/distribution/pull_v2_unix.go index 65c9594386..8d2c311f76 100644 --- a/distribution/pull_v2_unix.go +++ b/distribution/pull_v2_unix.go @@ -4,10 +4,11 @@ package distribution // import "github.com/docker/docker/distribution" import ( "context" - "runtime" + "github.com/containerd/containerd/platforms" "github.com/docker/distribution" "github.com/docker/distribution/manifest/manifestlist" + specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sirupsen/logrus" ) @@ -16,15 +17,27 @@ func (ld *v2LayerDescriptor) open(ctx context.Context) (distribution.ReadSeekClo return blobs.Open(ctx, ld.digest) } -func filterManifests(manifests []manifestlist.ManifestDescriptor, _ string) []manifestlist.ManifestDescriptor { +func filterManifests(manifests []manifestlist.ManifestDescriptor, p specs.Platform) []manifestlist.ManifestDescriptor { + p = withDefault(p) var matches []manifestlist.ManifestDescriptor - for _, manifestDescriptor := range manifests { - if manifestDescriptor.Platform.Architecture == runtime.GOARCH && manifestDescriptor.Platform.OS == runtime.GOOS { - matches = append(matches, manifestDescriptor) - - logrus.Debugf("found match for %s/%s with media type %s, digest %s", runtime.GOOS, runtime.GOARCH, manifestDescriptor.MediaType, manifestDescriptor.Digest.String()) + for _, desc := range manifests { + if compareNormalized(toOCIPlatform(desc.Platform), p) { + matches = append(matches, desc) + logrus.Debugf("found match for %s with media type %s, digest %s", platforms.Format(p), desc.MediaType, desc.Digest.String()) } } + + // deprecated: backwards compatibility with older versions that didn't compare variant + if len(matches) == 0 && p.Architecture == "arm" { + p = normalize(p) + for _, desc := range manifests { + if desc.Platform.OS == p.OS && desc.Platform.Architecture == p.Architecture { + matches = append(matches, desc) + logrus.Debugf("found deprecated partial match for %s with media type %s, digest %s", platforms.Format(p), desc.MediaType, desc.Digest.String()) + } + } + } + return matches } @@ -32,3 +45,38 @@ func filterManifests(manifests []manifestlist.ManifestDescriptor, _ string) []ma func checkImageCompatibility(imageOS, imageOSVersion string) error { return nil } + +func withDefault(p specs.Platform) specs.Platform { + def := platforms.DefaultSpec() + if p.OS == "" { + p.OS = def.OS + } + if p.Architecture == "" { + p.Architecture = def.Architecture + p.Variant = def.Variant + } + return p +} + +func compareNormalized(p1, p2 specs.Platform) bool { + // remove after https://github.com/containerd/containerd/pull/2414 + return p1.OS == p2.OS && + p1.Architecture == p2.Architecture && + p1.Variant == p2.Variant +} + +func normalize(p specs.Platform) specs.Platform { + p = platforms.Normalize(p) + // remove after https://github.com/containerd/containerd/pull/2414 + if p.Architecture == "arm" { + if p.Variant == "" { + p.Variant = "v7" + } + } + if p.Architecture == "arm64" { + if p.Variant == "" { + p.Variant = "v8" + } + } + return p +} diff --git a/distribution/pull_v2_windows.go b/distribution/pull_v2_windows.go index fc736e336d..3c9645883f 100644 --- a/distribution/pull_v2_windows.go +++ b/distribution/pull_v2_windows.go @@ -16,6 +16,7 @@ import ( "github.com/docker/distribution/manifest/schema2" "github.com/docker/distribution/registry/client/transport" "github.com/docker/docker/pkg/system" + specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sirupsen/logrus" ) @@ -62,7 +63,7 @@ func (ld *v2LayerDescriptor) open(ctx context.Context) (distribution.ReadSeekClo return rsc, err } -func filterManifests(manifests []manifestlist.ManifestDescriptor, requestedOS string) []manifestlist.ManifestDescriptor { +func filterManifests(manifests []manifestlist.ManifestDescriptor, p specs.Platform) []manifestlist.ManifestDescriptor { version := system.GetOSVersion() osVersion := fmt.Sprintf("%d.%d.%d", version.MajorVersion, version.MinorVersion, version.Build) logrus.Debugf("will prefer Windows entries with version %s", osVersion) @@ -71,8 +72,8 @@ func filterManifests(manifests []manifestlist.ManifestDescriptor, requestedOS st foundWindowsMatch := false for _, manifestDescriptor := range manifests { if (manifestDescriptor.Platform.Architecture == runtime.GOARCH) && - ((requestedOS != "" && manifestDescriptor.Platform.OS == requestedOS) || // Explicit user request for an OS we know we support - (requestedOS == "" && system.IsOSSupported(manifestDescriptor.Platform.OS))) { // No user requested OS, but one we can support + ((p.OS != "" && manifestDescriptor.Platform.OS == p.OS) || // Explicit user request for an OS we know we support + (p.OS == "" && system.IsOSSupported(manifestDescriptor.Platform.OS))) { // No user requested OS, but one we can support matches = append(matches, manifestDescriptor) logrus.Debugf("found match %s/%s %s with media type %s, digest %s", manifestDescriptor.Platform.OS, runtime.GOARCH, manifestDescriptor.Platform.OSVersion, manifestDescriptor.MediaType, manifestDescriptor.Digest.String()) if strings.EqualFold("windows", manifestDescriptor.Platform.OS) { diff --git a/distribution/registry_unit_test.go b/distribution/registry_unit_test.go index 5ae529d23d..3651c4688b 100644 --- a/distribution/registry_unit_test.go +++ b/distribution/registry_unit_test.go @@ -5,7 +5,6 @@ import ( "net/http" "net/http/httptest" "net/url" - "runtime" "strings" "testing" @@ -84,7 +83,7 @@ func testTokenPassThru(t *testing.T, ts *httptest.Server) { logrus.Debug("About to pull") // We expect it to fail, since we haven't mock'd the full registry exchange in our handler above tag, _ := reference.WithTag(n, "tag_goes_here") - _ = p.pullV2Repository(ctx, tag, runtime.GOOS) + _ = p.pullV2Repository(ctx, tag, nil) } func TestTokenPassThru(t *testing.T) {