beenull/moby
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

distribution: fix passing platform struct to puller

Browse source 
Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
This commit is contained in:
Tonis Tiigi 2018-06-26 14:49:33 -07:00
parent 81f862a1fe
commit 337ba71fc1
19 changed files with 208 additions and 128 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
api/server/router/image/backend.go
View file

@ -8,6 +8,7 @@ import (
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/api/types/image"
"github.com/docker/docker/api/types/registry"
specs "github.com/opencontainers/image-spec/specs-go/v1"
)
// Backend is all the methods that need to be implemented
@ -34,7 +35,7 @@ type importExportBackend interface {
}
type registryBackend interface {
PullImage(ctx context.Context, image, tag, platform string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error
PullImage(ctx context.Context, image, tag string, platform *specs.Platform, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error
PushImage(ctx context.Context, image, tag string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error
SearchRegistryForImages(ctx context.Context, filtersArgs string, term string, limit int, authConfig *types.AuthConfig, metaHeaders map[string][]string) (*registry.SearchResults, error)
}

10
api/server/router/image/image_routes.go
View file

@ -35,7 +35,7 @@ func (s *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWrite
message = r.Form.Get("message")
err error
output = ioutils.NewWriteFlusher(w)
platform = &specs.Platform{}
platform *specs.Platform
)
defer output.Close()
@ -72,13 +72,17 @@ func (s *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWrite
authConfig = &types.AuthConfig{}
}
}
err = s.backend.PullImage(ctx, image, tag, platform.OS, metaHeaders, authConfig, output)
err = s.backend.PullImage(ctx, image, tag, platform, metaHeaders, authConfig, output)
} else { //import
src := r.Form.Get("fromSrc")
// 'err' MUST NOT be defined within this block, we need any error
// generated from the download to be available to the output
// stream processing below
err = s.backend.ImportImage(src, repo, platform.OS, tag, message, r.Body, output, r.Form["changes"])
os := ""
if platform != nil {
os = platform.OS
}
err = s.backend.ImportImage(src, repo, os, tag, message, r.Body, output, r.Form["changes"])
}
}
if err != nil {

3
api/types/backend/build.go
View file

@ -5,6 +5,7 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/pkg/streamformatter"
specs "github.com/opencontainers/image-spec/specs-go/v1"
)
// PullOption defines different modes for accessing images
@ -40,5 +41,5 @@ type GetImageAndLayerOptions struct {
PullOption PullOption
AuthConfig map[string]types.AuthConfig
Output io.Writer
OS string
Platform *specs.Platform
}

10
builder/dockerfile/copy.go
View file

@ -96,8 +96,14 @@ func (o *copier) createCopyInstruction(args []string, cmdName string) (copyInstr
last := len(args) - 1
// Work in platform-specific filepath semantics
inst.dest = fromSlash(args[last], o.platform.OS)
separator := string(separator(o.platform.OS))
// TODO: This OS switch for paths is NOT correct and should not be supported.
// Maintained for backwards compatibility
pathOS := runtime.GOOS
if o.platform != nil {
pathOS = o.platform.OS
}
inst.dest = fromSlash(args[last], pathOS)
separator := string(separator(pathOS))
infos, err := o.getCopyInfosForSourcePaths(args[0:last], inst.dest)
if err != nil {
return inst, errors.Wrapf(err, "%s failed", cmdName)

84
builder/dockerfile/dispatchers.go
View file

@ -28,6 +28,7 @@ import (
"github.com/moby/buildkit/frontend/dockerfile/instructions"
"github.com/moby/buildkit/frontend/dockerfile/parser"
"github.com/moby/buildkit/frontend/dockerfile/shell"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
@ -103,7 +104,7 @@ func dispatchAdd(d dispatchRequest, c *instructions.AddCommand) error {
copyInstruction.chownStr = c.Chown
copyInstruction.allowLocalDecompression = true
return d.builder.performCopy(d.state, copyInstruction)
return d.builder.performCopy(d, copyInstruction)
}
// COPY foo /path
@ -127,7 +128,7 @@ func dispatchCopy(d dispatchRequest, c *instructions.CopyCommand) error {
}
copyInstruction.chownStr = c.Chown
return d.builder.performCopy(d.state, copyInstruction)
return d.builder.performCopy(d, copyInstruction)
}
func (d *dispatchRequest) getImageMount(imageRefOrID string) (*imageMount, error) {
@ -145,7 +146,7 @@ func (d *dispatchRequest) getImageMount(imageRefOrID string) (*imageMount, error
imageRefOrID = stage.Image
localOnly = true
}
return d.builder.imageSources.Get(imageRefOrID, localOnly, d.state.operatingSystem)
return d.builder.imageSources.Get(imageRefOrID, localOnly, d.builder.options.Platform)
}
// FROM [--platform=platform] imagename[:tag | @digest] [AS build-stage-name]
@ -153,22 +154,21 @@ func (d *dispatchRequest) getImageMount(imageRefOrID string) (*imageMount, error
func initializeStage(d dispatchRequest, cmd *instructions.Stage) error {
d.builder.imageProber.Reset()
// TODO: pass *platform instead, allow autodetect
platform := platforms.DefaultSpec()
var platform *specs.Platform
if v := cmd.Platform; v != "" {
// TODO:
// v, err := shlex.ProcessWord(v, toEnvList(metaArgs, nil))
// if err != nil {
// return nil, nil, errors.Wrapf(err, "failed to process arguments for platform %s", v)
// }
v, err := d.getExpandedString(d.shlex, v)
if err != nil {
return errors.Wrapf(err, "failed to process arguments for platform %s", v)
}
p, err := platforms.Parse(v)
if err != nil {
return errors.Wrapf(err, "failed to parse platform %s", v)
}
platform = p
platform = &p
}
image, err := d.getFromImage(d.shlex, cmd.BaseName, platform.OS)
image, err := d.getFromImage(d.shlex, cmd.BaseName, platform)
if err != nil {
return err
}
@ -214,82 +214,72 @@ func dispatchTriggeredOnBuild(d dispatchRequest, triggers []string) error {
return nil
}
func (d *dispatchRequest) getExpandedImageName(shlex *shell.Lex, name string) (string, error) {
func (d *dispatchRequest) getExpandedString(shlex *shell.Lex, str string) (string, error) {
substitutionArgs := []string{}
for key, value := range d.state.buildArgs.GetAllMeta() {
substitutionArgs = append(substitutionArgs, key+"="+value)
}
name, err := shlex.ProcessWord(name, substitutionArgs)
name, err := shlex.ProcessWord(str, substitutionArgs)
if err != nil {
return "", err
}
return name, nil
}
// getOsFromFlagsAndStage calculates the operating system if we need to pull an image.
// stagePlatform contains the value supplied by optional `--platform=` on
// a current FROM statement. b.builder.options.Platform contains the operating
// system part of the optional flag passed in the API call (or CLI flag
// through `docker build --platform=...`). Precedence is for an explicit
// platform indication in the FROM statement.
func (d *dispatchRequest) getOsFromFlagsAndStage(stageOS string) string {
switch {
case stageOS != "":
return stageOS
case d.builder.options.Platform.OS != "":
// Note this is API "platform", but by this point, as the daemon is not
// multi-arch aware yet, it is guaranteed to only hold the OS part here.
return d.builder.options.Platform.OS
default:
return "" // Auto-select
}
}
func (d *dispatchRequest) getImageOrStage(name string, stageOS string) (builder.Image, error) {
func (d *dispatchRequest) getImageOrStage(name string, platform *specs.Platform) (builder.Image, error) {
var localOnly bool
if im, ok := d.stages.getByName(name); ok {
name = im.Image
localOnly = true
}
os := d.getOsFromFlagsAndStage(stageOS)
if platform == nil {
platform = d.builder.options.Platform
}
// Windows cannot support a container with no base image unless it is LCOW.
if name == api.NoBaseImageSpecifier {
p := platforms.DefaultSpec()
if platform != nil {
p = *platform
}
imageImage := &image.Image{}
imageImage.OS = runtime.GOOS
imageImage.OS = p.OS
// old windows scratch handling
// TODO: scratch should not have an os. It should be nil image.
// Windows supports scratch. What is not supported is running containers
// from it.
if runtime.GOOS == "windows" {
switch os {
case "windows":
return nil, errors.New("Windows does not support FROM scratch")
case "linux", "":
if platform == nil || platform.OS == "linux" {
if !system.LCOWSupported() {
return nil, errors.New("Linux containers are not supported on this system")
}
imageImage.OS = "linux"
default:
return nil, errors.Errorf("operating system %q is not supported", os)
} else if platform.OS == "windows" {
return nil, errors.New("Windows does not support FROM scratch")
} else {
return nil, errors.Errorf("platform %s is not supported", platforms.Format(p))
}
}
return builder.Image(imageImage), nil
}
imageMount, err := d.builder.imageSources.Get(name, localOnly, os)
imageMount, err := d.builder.imageSources.Get(name, localOnly, platform)
if err != nil {
return nil, err
}
return imageMount.Image(), nil
}
func (d *dispatchRequest) getFromImage(shlex *shell.Lex, name string, stageOS string) (builder.Image, error) {
name, err := d.getExpandedImageName(shlex, name)
func (d *dispatchRequest) getFromImage(shlex *shell.Lex, name string, platform *specs.Platform) (builder.Image, error) {
name, err := d.getExpandedString(shlex, name)
if err != nil {
return nil, err
}
return d.getImageOrStage(name, stageOS)
return d.getImageOrStage(name, platform)
}
func dispatchOnbuild(d dispatchRequest, c *instructions.OnbuildCommand) error {
d.state.runConfig.OnBuild = append(d.state.runConfig.OnBuild, c.Expression)
return d.builder.commit(d.state, "ONBUILD "+c.Expression)
}

7
builder/dockerfile/dispatchers_test.go
View file

@ -6,6 +6,7 @@ import (
"runtime"
"testing"
"github.com/containerd/containerd/platforms"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/backend"
"github.com/docker/docker/api/types/container"
@ -22,15 +23,17 @@ import (
func newBuilderWithMockBackend() *Builder {
mockBackend := &MockBackend{}
defaultPlatform := platforms.DefaultSpec()
opts := &types.ImageBuildOptions{Platform: &defaultPlatform}
ctx := context.Background()
b := &Builder{
options: &types.ImageBuildOptions{Platform: runtime.GOOS},
options: opts,
docker: mockBackend,
Stdout: new(bytes.Buffer),
clientCtx: ctx,
disableCommit: true,
imageSources: newImageSources(ctx, builderOptions{
Options: &types.ImageBuildOptions{Platform: runtime.GOOS},
Options: opts,
Backend: mockBackend,
}),
imageProber: newImageProber(mockBackend, nil, false),

11
builder/dockerfile/imagecontext.go
View file

@ -7,11 +7,12 @@ import (
"github.com/docker/docker/api/types/backend"
"github.com/docker/docker/builder"
dockerimage "github.com/docker/docker/image"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
type getAndMountFunc func(string, bool, string) (builder.Image, builder.ROLayer, error)
type getAndMountFunc func(string, bool, *specs.Platform) (builder.Image, builder.ROLayer, error)
// imageSources mounts images and provides a cache for mounted images. It tracks
// all images so they can be unmounted at the end of the build.
@ -22,7 +23,7 @@ type imageSources struct {
}
func newImageSources(ctx context.Context, options builderOptions) *imageSources {
getAndMount := func(idOrRef string, localOnly bool, osForPull string) (builder.Image, builder.ROLayer, error) {
getAndMount := func(idOrRef string, localOnly bool, platform *specs.Platform) (builder.Image, builder.ROLayer, error) {
pullOption := backend.PullOptionNoPull
if !localOnly {
if options.Options.PullParent {
@ -35,7 +36,7 @@ func newImageSources(ctx context.Context, options builderOptions) *imageSources
PullOption: pullOption,
AuthConfig: options.Options.AuthConfigs,
Output: options.ProgressWriter.Output,
OS: osForPull,
Platform: platform,
})
}
@ -45,12 +46,12 @@ func newImageSources(ctx context.Context, options builderOptions) *imageSources
}
}
func (m *imageSources) Get(idOrRef string, localOnly bool, osForPull string) (*imageMount, error) {
func (m *imageSources) Get(idOrRef string, localOnly bool, platform *specs.Platform) (*imageMount, error) {
if im, ok := m.byImageID[idOrRef]; ok {
return im, nil
}
image, layer, err := m.getImage(idOrRef, localOnly, osForPull)
image, layer, err := m.getImage(idOrRef, localOnly, platform)
if err != nil {
return nil, err
}

7
builder/dockerfile/internals.go
View file

@ -150,7 +150,8 @@ func (b *Builder) exportImage(state *dispatchState, layer builder.RWLayer, paren
return nil
}
func (b *Builder) performCopy(state *dispatchState, inst copyInstruction) error {
func (b *Builder) performCopy(req dispatchRequest, inst copyInstruction) error {
state := req.state
srcHash := getSourceHashFromInfos(inst.infos)
var chownComment string
@ -168,7 +169,7 @@ func (b *Builder) performCopy(state *dispatchState, inst copyInstruction) error
return err
}
imageMount, err := b.imageSources.Get(state.imageID, true, state.operatingSystem)
imageMount, err := b.imageSources.Get(state.imageID, true, req.builder.options.Platform)
if err != nil {
return errors.Wrapf(err, "failed to get destination image %q", state.imageID)
}
@ -456,7 +457,7 @@ func hostConfigFromOptions(options *types.ImageBuildOptions) *container.HostConf
// is too small for builder scenarios where many users are
// using RUN statements to install large amounts of data.
// Use 127GB as that's the default size of a VHD in Hyper-V.
if runtime.GOOS == "windows" && options.Platform.OS == "windows" {
if runtime.GOOS == "windows" && options.Platform != nil && options.Platform.OS == "windows" {
hc.StorageOpt = make(map[string]string)
hc.StorageOpt["size"] = "127GB"
}

3
daemon/cluster/executor/backend.go
View file

@ -23,6 +23,7 @@ import (
"github.com/docker/libnetwork/cluster"
networktypes "github.com/docker/libnetwork/types"
"github.com/docker/swarmkit/agent/exec"
specs "github.com/opencontainers/image-spec/specs-go/v1"
)
// Backend defines the executor component for a swarm agent.
@ -69,7 +70,7 @@ type VolumeBackend interface {
// ImageBackend is used by an executor to perform image operations
type ImageBackend interface {
PullImage(ctx context.Context, image, tag, platform string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error
PullImage(ctx context.Context, image, tag string, platform *specs.Platform, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error
GetRepository(context.Context, reference.Named, *types.AuthConfig) (distribution.Repository, bool, error)
LookupImage(name string) (*types.ImageInspect, error)
}

4
daemon/cluster/executor/container/adapter.go
View file

@ -8,7 +8,6 @@ import (
"fmt"
"io"
"os"
"runtime"
"strings"
"syscall"
"time"
@ -97,8 +96,7 @@ func (c *containerAdapter) pullImage(ctx context.Context) error {
go func() {
// TODO @jhowardmsft LCOW Support: This will need revisiting as
// the stack is built up to include LCOW support for swarm.
platform := runtime.GOOS
err := c.imageBackend.PullImage(ctx, c.container.image(), "", platform, metaHeaders, authConfig, pw)
err := c.imageBackend.PullImage(ctx, c.container.image(), "", nil, metaHeaders, authConfig, pw)
pw.CloseWithError(err)
}()

16
daemon/images/image_builder.go
View file

@ -3,6 +3,7 @@ package images // import "github.com/docker/docker/daemon/images"
import (
"context"
"io"
"runtime"
"github.com/docker/distribution/reference"
"github.com/docker/docker/api/types"
@ -14,6 +15,7 @@ import (
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/pkg/system"
"github.com/docker/docker/registry"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
@ -137,7 +139,7 @@ func newROLayerForImage(img *image.Image, layerStore layer.Store) (builder.ROLay
}
// TODO: could this use the regular daemon PullImage ?
func (i *ImageService) pullForBuilder(ctx context.Context, name string, authConfigs map[string]types.AuthConfig, output io.Writer, os string) (*image.Image, error) {
func (i *ImageService) pullForBuilder(ctx context.Context, name string, authConfigs map[string]types.AuthConfig, output io.Writer, platform *specs.Platform) (*image.Image, error) {
ref, err := reference.ParseNormalizedNamed(name)
if err != nil {
return nil, err
@ -156,7 +158,7 @@ func (i *ImageService) pullForBuilder(ctx context.Context, name string, authConf
pullRegistryAuth = &resolvedConfig
}
if err := i.pullImageWithReference(ctx, ref, os, nil, pullRegistryAuth, output); err != nil {
if err := i.pullImageWithReference(ctx, ref, platform, nil, pullRegistryAuth, output); err != nil {
return nil, err
}
return i.GetImage(name)
@ -167,10 +169,14 @@ func (i *ImageService) pullForBuilder(ctx context.Context, name string, authConf
// leaking of layers.
func (i *ImageService) GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ROLayer, error) {
if refOrID == "" { // ie FROM scratch
if !system.IsOSSupported(opts.OS) {
os := runtime.GOOS
if opts.Platform != nil {
os = opts.Platform.OS
}
if !system.IsOSSupported(os) {
return nil, nil, system.ErrNotSupportedOperatingSystem
}
layer, err := newROLayerForImage(nil, i.layerStores[opts.OS])
layer, err := newROLayerForImage(nil, i.layerStores[os])
return nil, layer, err
}
@ -189,7 +195,7 @@ func (i *ImageService) GetImageAndReleasableLayer(ctx context.Context, refOrID s
}
}
image, err := i.pullForBuilder(ctx, refOrID, opts.AuthConfig, opts.Output, opts.OS)
image, err := i.pullForBuilder(ctx, refOrID, opts.AuthConfig, opts.Output, opts.Platform)
if err != nil {
return nil, nil, err
}

9
daemon/images/image_pull.go
View file

@ -15,11 +15,12 @@ import (
"github.com/docker/docker/pkg/progress"
"github.com/docker/docker/registry"
"github.com/opencontainers/go-digest"
specs "github.com/opencontainers/image-spec/specs-go/v1"
)
// PullImage initiates a pull operation. image is the repository name to pull, and
// tag may be either empty, or indicate a specific tag to pull.
func (i *ImageService) PullImage(ctx context.Context, image, tag, os string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error {
func (i *ImageService) PullImage(ctx context.Context, image, tag string, platform *specs.Platform, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error {
start := time.Now()
// Special case: "pull -a" may send an image name with a
// trailing :. This is ugly, but let's not break API
@ -45,12 +46,12 @@ func (i *ImageService) PullImage(ctx context.Context, image, tag, os string, met
}
}
err = i.pullImageWithReference(ctx, ref, os, metaHeaders, authConfig, outStream)
err = i.pullImageWithReference(ctx, ref, platform, metaHeaders, authConfig, outStream)
imageActions.WithValues("pull").UpdateSince(start)
return err
}
func (i *ImageService) pullImageWithReference(ctx context.Context, ref reference.Named, os string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error {
func (i *ImageService) pullImageWithReference(ctx context.Context, ref reference.Named, platform *specs.Platform, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error {
// Include a buffer so that slow client connections don't affect
// transfer performance.
progressChan := make(chan progress.Progress, 100)
@ -77,7 +78,7 @@ func (i *ImageService) pullImageWithReference(ctx context.Context, ref reference
},
DownloadManager: i.downloadManager,
Schema2Types: distribution.ImageTypes,
OS: os,
Platform: platform,
}
err := distribution.Pull(ctx, ref, imagePullConfig)

7
distribution/config.go
View file

@ -60,9 +60,8 @@ type ImagePullConfig struct {
// Schema2Types is the valid schema2 configuration types allowed
// by the pull operation.
Schema2Types []string
// OS is the requested operating system of the image being pulled to ensure it can be validated
// when the host OS supports multiple image operating systems.
OS string
// Platform is the requested platform of the image being pulled
Platform *specs.Platform
}
// ImagePushConfig stores push configuration.
@ -171,7 +170,7 @@ func (s *imageConfigStore) PlatformFromConfig(c []byte) (*specs.Platform, error)
if !system.IsOSSupported(os) {
return nil, system.ErrNotSupportedOperatingSystem
}
return &specs.Platform{OS: os, OSVersion: unmarshalledConfig.OSVersion}, nil
return &specs.Platform{OS: os, Architecture: unmarshalledConfig.Architecture, OSVersion: unmarshalledConfig.OSVersion}, nil
}
type storeLayerProvider struct {

5
distribution/pull.go
View file

@ -11,6 +11,7 @@ import (
refstore "github.com/docker/docker/reference"
"github.com/docker/docker/registry"
"github.com/opencontainers/go-digest"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
@ -20,7 +21,7 @@ type Puller interface {
// Pull tries to pull the image referenced by `tag`
// Pull returns an error if any, as well as a boolean that determines whether to retry Pull on the next configured endpoint.
//
Pull(ctx context.Context, ref reference.Named, os string) error
Pull(ctx context.Context, ref reference.Named, platform *specs.Platform) error
}
// newPuller returns a Puller interface that will pull from either a v1 or v2
@ -114,7 +115,7 @@ func Pull(ctx context.Context, ref reference.Named, imagePullConfig *ImagePullCo
continue
}
if err := puller.Pull(ctx, ref, imagePullConfig.OS); err != nil {
if err := puller.Pull(ctx, ref, imagePullConfig.Platform); err != nil {
// Was this pull cancelled? If so, don't try to fall
// back.
fallback := false

3
distribution/pull_v1.go
View file

@ -25,6 +25,7 @@ import (
"github.com/docker/docker/pkg/progress"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/registry"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus"
)
@ -36,7 +37,7 @@ type v1Puller struct {
session *registry.Session
}
func (p *v1Puller) Pull(ctx context.Context, ref reference.Named, os string) error {
func (p *v1Puller) Pull(ctx context.Context, ref reference.Named, _ *specs.Platform) error {
if _, isCanonical := ref.(reference.Canonical); isCanonical {
// Allowing fallback, because HTTPS v1 is before HTTP v2
return fallbackError{err: ErrNoSupport{Err: errors.New("Cannot pull by digest with v1 registry")}}

82
distribution/pull_v2.go
View file

@ -11,6 +11,7 @@ import (
"runtime"
"strings"
"github.com/containerd/containerd/platforms"
"github.com/docker/distribution"
"github.com/docker/distribution/manifest/manifestlist"
"github.com/docker/distribution/manifest/schema1"
@ -63,7 +64,7 @@ type v2Puller struct {
confirmedV2 bool
}
func (p *v2Puller) Pull(ctx context.Context, ref reference.Named, os string) (err error) {
func (p *v2Puller) Pull(ctx context.Context, ref reference.Named, platform *specs.Platform) (err error) {
// TODO(tiborvass): was ReceiveTimeout
p.repo, p.confirmedV2, err = NewV2Repository(ctx, p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "pull")
if err != nil {
@ -71,7 +72,7 @@ func (p *v2Puller) Pull(ctx context.Context, ref reference.Named, os string) (er
return err
}
if err = p.pullV2Repository(ctx, ref, os); err != nil {
if err = p.pullV2Repository(ctx, ref, platform); err != nil {
if _, ok := err.(fallbackError); ok {
return err
}
@ -86,10 +87,10 @@ func (p *v2Puller) Pull(ctx context.Context, ref reference.Named, os string) (er
return err
}
func (p *v2Puller) pullV2Repository(ctx context.Context, ref reference.Named, os string) (err error) {
func (p *v2Puller) pullV2Repository(ctx context.Context, ref reference.Named, platform *specs.Platform) (err error) {
var layersDownloaded bool
if !reference.IsNameOnly(ref) {
layersDownloaded, err = p.pullV2Tag(ctx, ref, os)
layersDownloaded, err = p.pullV2Tag(ctx, ref, platform)
if err != nil {
return err
}
@ -111,7 +112,7 @@ func (p *v2Puller) pullV2Repository(ctx context.Context, ref reference.Named, os
if err != nil {
return err
}
pulledNew, err := p.pullV2Tag(ctx, tagRef, os)
pulledNew, err := p.pullV2Tag(ctx, tagRef, platform)
if err != nil {
// Since this is the pull-all-tags case, don't
// allow an error pulling a particular tag to
@ -327,7 +328,7 @@ func (ld *v2LayerDescriptor) Registered(diffID layer.DiffID) {
ld.V2MetadataService.Add(diffID, metadata.V2Metadata{Digest: ld.digest, SourceRepository: ld.repoInfo.Name.Name()})
}
func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named, os string) (tagUpdated bool, err error) {
func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named, platform *specs.Platform) (tagUpdated bool, err error) {
manSvc, err := p.repo.Manifests(ctx)
if err != nil {
return false, err
@ -391,17 +392,17 @@ func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named, os string
if p.config.RequireSchema2 {
return false, fmt.Errorf("invalid manifest: not schema2")
}
id, manifestDigest, err = p.pullSchema1(ctx, ref, v, os)
id, manifestDigest, err = p.pullSchema1(ctx, ref, v, platform)
if err != nil {
return false, err
}
case *schema2.DeserializedManifest:
id, manifestDigest, err = p.pullSchema2(ctx, ref, v, os)
id, manifestDigest, err = p.pullSchema2(ctx, ref, v, platform)
if err != nil {
return false, err
}
case *manifestlist.DeserializedManifestList:
id, manifestDigest, err = p.pullManifestList(ctx, ref, v, os)
id, manifestDigest, err = p.pullManifestList(ctx, ref, v, platform)
if err != nil {
return false, err
}
@ -437,7 +438,7 @@ func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named, os string
return true, nil
}
func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Reference, unverifiedManifest *schema1.SignedManifest, requestedOS string) (id digest.Digest, manifestDigest digest.Digest, err error) {
func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Reference, unverifiedManifest *schema1.SignedManifest, platform *specs.Platform) (id digest.Digest, manifestDigest digest.Digest, err error) {
var verifiedManifest *schema1.Manifest
verifiedManifest, err = verifySchema1Manifest(unverifiedManifest, ref)
if err != nil {
@ -513,7 +514,10 @@ func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Reference, unv
// we support the operating system, switch to that operating system.
// eg FROM supertest2014/nyan with no platform specifier, and docker build
// with no --platform= flag under LCOW.
if requestedOS == "" && system.IsOSSupported(configOS) {
requestedOS := ""
if platform != nil {
requestedOS = platform.OS
} else if system.IsOSSupported(configOS) {
requestedOS = configOS
}
@ -544,7 +548,7 @@ func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Reference, unv
return imageID, manifestDigest, nil
}
func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *schema2.DeserializedManifest, requestedOS string) (id digest.Digest, manifestDigest digest.Digest, err error) {
func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *schema2.DeserializedManifest, platform *specs.Platform) (id digest.Digest, manifestDigest digest.Digest, err error) {
manifestDigest, err = schema2ManifestDigest(ref, mfst)
if err != nil {
return "", "", err
@ -600,6 +604,11 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s
configPlatform *specs.Platform // for LCOW when registering downloaded layers
)
layerStoreOS := runtime.GOOS
if platform != nil {
layerStoreOS = platform.OS
}
// https://github.com/docker/docker/issues/24766 - Err on the side of caution,
// explicitly blocking images intended for linux from the Windows daemon. On
// Windows, we do this before the attempt to download, effectively serialising
@ -623,13 +632,14 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s
if len(descriptors) != len(configRootFS.DiffIDs) {
return "", "", errRootFSMismatch
}
// Early bath if the requested OS doesn't match that of the configuration.
// This avoids doing the download, only to potentially fail later.
if !system.IsOSSupported(configPlatform.OS) {
return "", "", fmt.Errorf("cannot download image with operating system %q when requesting %q", configPlatform.OS, requestedOS)
if platform == nil {
// Early bath if the requested OS doesn't match that of the configuration.
// This avoids doing the download, only to potentially fail later.
if !system.IsOSSupported(configPlatform.OS) {
return "", "", fmt.Errorf("cannot download image with operating system %q when requesting %q", configPlatform.OS, layerStoreOS)
}
layerStoreOS = configPlatform.OS
}
requestedOS = configPlatform.OS
// Populate diff ids in descriptors to avoid downloading foreign layers
// which have been side loaded
@ -638,10 +648,6 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s
}
}
if requestedOS == "" {
requestedOS = runtime.GOOS
}
if p.config.DownloadManager != nil {
go func() {
var (
@ -649,7 +655,7 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s
rootFS image.RootFS
)
downloadRootFS := *image.NewRootFS()
rootFS, release, err = p.config.DownloadManager.Download(ctx, downloadRootFS, requestedOS, descriptors, p.config.ProgressOutput)
rootFS, release, err = p.config.DownloadManager.Download(ctx, downloadRootFS, layerStoreOS, descriptors, p.config.ProgressOutput)
if err != nil {
// Intentionally do not cancel the config download here
// as the error from config download (if there is one)
@ -735,22 +741,22 @@ func receiveConfig(s ImageConfigStore, configChan <-chan []byte, errChan <-chan
// pullManifestList handles "manifest lists" which point to various
// platform-specific manifests.
func (p *v2Puller) pullManifestList(ctx context.Context, ref reference.Named, mfstList *manifestlist.DeserializedManifestList, requestedOS string) (id digest.Digest, manifestListDigest digest.Digest, err error) {
func (p *v2Puller) pullManifestList(ctx context.Context, ref reference.Named, mfstList *manifestlist.DeserializedManifestList, pp *specs.Platform) (id digest.Digest, manifestListDigest digest.Digest, err error) {
manifestListDigest, err = schema2ManifestDigest(ref, mfstList)
if err != nil {
return "", "", err
}
logOS := requestedOS // May be "" indicating any OS
if logOS == "" {
logOS = "*"
var platform specs.Platform
if pp != nil {
platform = *pp
}
logrus.Debugf("%s resolved to a manifestList object with %d entries; looking for a %s/%s match", ref, len(mfstList.Manifests), logOS, runtime.GOARCH)
logrus.Debugf("%s resolved to a manifestList object with %d entries; looking for a %s/%s match", ref, len(mfstList.Manifests), platforms.Format(platform), runtime.GOARCH)
manifestMatches := filterManifests(mfstList.Manifests, requestedOS)
manifestMatches := filterManifests(mfstList.Manifests, platform)
if len(manifestMatches) == 0 {
errMsg := fmt.Sprintf("no matching manifest for %s/%s in the manifest list entries", logOS, runtime.GOARCH)
errMsg := fmt.Sprintf("no matching manifest for %s in the manifest list entries", platforms.Format(platform))
logrus.Debugf(errMsg)
return "", "", errors.New(errMsg)
}
@ -781,12 +787,14 @@ func (p *v2Puller) pullManifestList(ctx context.Context, ref reference.Named, mf
switch v := manifest.(type) {
case *schema1.SignedManifest:
id, _, err = p.pullSchema1(ctx, manifestRef, v, manifestMatches[0].Platform.OS)
platform := toOCIPlatform(manifestMatches[0].Platform)
id, _, err = p.pullSchema1(ctx, manifestRef, v, &platform)
if err != nil {
return "", "", err
}
case *schema2.DeserializedManifest:
id, _, err = p.pullSchema2(ctx, manifestRef, v, manifestMatches[0].Platform.OS)
platform := toOCIPlatform(manifestMatches[0].Platform)
id, _, err = p.pullSchema2(ctx, manifestRef, v, &platform)
if err != nil {
return "", "", err
}
@ -956,3 +964,13 @@ func fixManifestLayers(m *schema1.Manifest) error {
func createDownloadFile() (*os.File, error) {
return ioutil.TempFile("", "GetImageBlob")
}
func toOCIPlatform(p manifestlist.PlatformSpec) specs.Platform {
return specs.Platform{
OS: p.OS,
Architecture: p.Architecture,
Variant: p.Variant,
OSFeatures: p.OSFeatures,
OSVersion: p.OSVersion,
}
}

62
distribution/pull_v2_unix.go
View file

@ -4,10 +4,11 @@ package distribution // import "github.com/docker/docker/distribution"
import (
"context"
"runtime"
"github.com/containerd/containerd/platforms"
"github.com/docker/distribution"
"github.com/docker/distribution/manifest/manifestlist"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus"
)
@ -16,15 +17,27 @@ func (ld *v2LayerDescriptor) open(ctx context.Context) (distribution.ReadSeekClo
return blobs.Open(ctx, ld.digest)
}
func filterManifests(manifests []manifestlist.ManifestDescriptor, _ string) []manifestlist.ManifestDescriptor {
func filterManifests(manifests []manifestlist.ManifestDescriptor, p specs.Platform) []manifestlist.ManifestDescriptor {
p = withDefault(p)
var matches []manifestlist.ManifestDescriptor
for _, manifestDescriptor := range manifests {
if manifestDescriptor.Platform.Architecture == runtime.GOARCH && manifestDescriptor.Platform.OS == runtime.GOOS {
matches = append(matches, manifestDescriptor)
logrus.Debugf("found match for %s/%s with media type %s, digest %s", runtime.GOOS, runtime.GOARCH, manifestDescriptor.MediaType, manifestDescriptor.Digest.String())
for _, desc := range manifests {
if compareNormalized(toOCIPlatform(desc.Platform), p) {
matches = append(matches, desc)
logrus.Debugf("found match for %s with media type %s, digest %s", platforms.Format(p), desc.MediaType, desc.Digest.String())
}
}
// deprecated: backwards compatibility with older versions that didn't compare variant
if len(matches) == 0 && p.Architecture == "arm" {
p = normalize(p)
for _, desc := range manifests {
if desc.Platform.OS == p.OS && desc.Platform.Architecture == p.Architecture {
matches = append(matches, desc)
logrus.Debugf("found deprecated partial match for %s with media type %s, digest %s", platforms.Format(p), desc.MediaType, desc.Digest.String())
}
}
}
return matches
}
@ -32,3 +45,38 @@ func filterManifests(manifests []manifestlist.ManifestDescriptor, _ string) []ma
func checkImageCompatibility(imageOS, imageOSVersion string) error {
return nil
}
func withDefault(p specs.Platform) specs.Platform {
def := platforms.DefaultSpec()
if p.OS == "" {
p.OS = def.OS
}
if p.Architecture == "" {
p.Architecture = def.Architecture
p.Variant = def.Variant
}
return p
}
func compareNormalized(p1, p2 specs.Platform) bool {
// remove after https://github.com/containerd/containerd/pull/2414
return p1.OS == p2.OS &&
p1.Architecture == p2.Architecture &&
p1.Variant == p2.Variant
}
func normalize(p specs.Platform) specs.Platform {
p = platforms.Normalize(p)
// remove after https://github.com/containerd/containerd/pull/2414
if p.Architecture == "arm" {
if p.Variant == "" {
p.Variant = "v7"
}
}
if p.Architecture == "arm64" {
if p.Variant == "" {
p.Variant = "v8"
}
}
return p
}

7
distribution/pull_v2_windows.go
View file

@ -16,6 +16,7 @@ import (
"github.com/docker/distribution/manifest/schema2"
"github.com/docker/distribution/registry/client/transport"
"github.com/docker/docker/pkg/system"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus"
)
@ -62,7 +63,7 @@ func (ld *v2LayerDescriptor) open(ctx context.Context) (distribution.ReadSeekClo
return rsc, err
}
func filterManifests(manifests []manifestlist.ManifestDescriptor, requestedOS string) []manifestlist.ManifestDescriptor {
func filterManifests(manifests []manifestlist.ManifestDescriptor, p specs.Platform) []manifestlist.ManifestDescriptor {
version := system.GetOSVersion()
osVersion := fmt.Sprintf("%d.%d.%d", version.MajorVersion, version.MinorVersion, version.Build)
logrus.Debugf("will prefer Windows entries with version %s", osVersion)
@ -71,8 +72,8 @@ func filterManifests(manifests []manifestlist.ManifestDescriptor, requestedOS st
foundWindowsMatch := false
for _, manifestDescriptor := range manifests {
if (manifestDescriptor.Platform.Architecture == runtime.GOARCH) &&
((requestedOS != "" && manifestDescriptor.Platform.OS == requestedOS) || // Explicit user request for an OS we know we support
(requestedOS == "" && system.IsOSSupported(manifestDescriptor.Platform.OS))) { // No user requested OS, but one we can support
((p.OS != "" && manifestDescriptor.Platform.OS == p.OS) || // Explicit user request for an OS we know we support
(p.OS == "" && system.IsOSSupported(manifestDescriptor.Platform.OS))) { // No user requested OS, but one we can support
matches = append(matches, manifestDescriptor)
logrus.Debugf("found match %s/%s %s with media type %s, digest %s", manifestDescriptor.Platform.OS, runtime.GOARCH, manifestDescriptor.Platform.OSVersion, manifestDescriptor.MediaType, manifestDescriptor.Digest.String())
if strings.EqualFold("windows", manifestDescriptor.Platform.OS) {

3
distribution/registry_unit_test.go
View file

@ -5,7 +5,6 @@ import (
"net/http"
"net/http/httptest"
"net/url"
"runtime"
"strings"
"testing"
@ -84,7 +83,7 @@ func testTokenPassThru(t *testing.T, ts *httptest.Server) {
logrus.Debug("About to pull")
// We expect it to fail, since we haven't mock'd the full registry exchange in our handler above
tag, _ := reference.WithTag(n, "tag_goes_here")
_ = p.pullV2Repository(ctx, tag, runtime.GOOS)
_ = p.pullV2Repository(ctx, tag, nil)
}
func TestTokenPassThru(t *testing.T) {