beenull/moby
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #37385 from tonistiigi/update-buildkit

Browse source 
vendor: update buildkit to 9acf51e491
This commit is contained in:
Tibor Vass 2018-07-03 17:33:54 -07:00 committed by GitHub
commit 4b3df087a2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 159 additions and 98 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
builder/builder-next/adapters/containerimage/pull.go
View file

@ -644,7 +644,7 @@ func showProgress(ctx context.Context, ongoing *jobs, cs content.Store, pw progr
// featured.
type jobs struct {
name string
added map[digest.Digest]job
added map[digest.Digest]*job
mu sync.Mutex
resolved bool
}
@ -658,7 +658,7 @@ type job struct {
func newJobs(name string) *jobs {
return &jobs{
name: name,
added: make(map[digest.Digest]job),
added: make(map[digest.Digest]*job),
}
}
@ -669,17 +669,17 @@ func (j *jobs) add(desc ocispec.Descriptor) {
if _, ok := j.added[desc.Digest]; ok {
return
}
j.added[desc.Digest] = job{
j.added[desc.Digest] = &job{
Descriptor: desc,
started: time.Now(),
}
}
func (j *jobs) jobs() []job {
func (j *jobs) jobs() []*job {
j.mu.Lock()
defer j.mu.Unlock()
descs := make([]job, 0, len(j.added))
descs := make([]*job, 0, len(j.added))
for _, j := range j.added {
descs = append(descs, j)
}

2
vendor.conf
View file

@ -26,7 +26,7 @@ github.com/imdario/mergo v0.3.5
golang.org/x/sync fd80eb99c8f653c847d294a001bdf2a3a6f768f5
# buildkit
github.com/moby/buildkit cce2080ddbe4698912f2290892b247c83627efa8
github.com/moby/buildkit 9acf51e49185b348608e0096b2903dd72907adcb
github.com/tonistiigi/fsutil 8abad97ee3969cdf5e9c367f46adba2c212b3ddb
github.com/grpc-ecosystem/grpc-opentracing 8e809c8a86450a29b90dcc9efbf062d0fe6d9746
github.com/opentracing/opentracing-go 1361b9cd60be79c4c3a7fa9841b3c132e40066a7

6
vendor/github.com/moby/buildkit/README.md generated vendored
View file

@ -138,11 +138,11 @@ docker inspect myimage
##### Building a Dockerfile using [external frontend](https://hub.docker.com/r/tonistiigi/dockerfile/tags/):
During development, an external version of the Dockerfile frontend is pushed to https://hub.docker.com/r/tonistiigi/dockerfile that can be used with the gateway frontend. The source for the external frontend is currently located in `./frontend/dockerfile/cmd/dockerfile-frontend` but will move out of this repository in the future ([#163](https://github.com/moby/buildkit/issues/163)).
During development, an external version of the Dockerfile frontend is pushed to https://hub.docker.com/r/tonistiigi/dockerfile that can be used with the gateway frontend. The source for the external frontend is currently located in `./frontend/dockerfile/cmd/dockerfile-frontend` but will move out of this repository in the future ([#163](https://github.com/moby/buildkit/issues/163)). For automatic build from master branch of this repository `tonistiigi/dockerfile:master` image can be used.
```
buildctl build --frontend=gateway.v0 --frontend-opt=source=tonistiigi/dockerfile:v0 --local context=. --local dockerfile=.
buildctl build --frontend gateway.v0 --frontend-opt=source=tonistiigi/dockerfile:v0 --frontend-opt=context=git://github.com/moby/moby --frontend-opt build-arg:APT_MIRROR=cdn-fastly.deb.debian.org
buildctl build --frontend=gateway.v0 --frontend-opt=source=tonistiigi/dockerfile --local context=. --local dockerfile=.
buildctl build --frontend gateway.v0 --frontend-opt=source=tonistiigi/dockerfile --frontend-opt=context=git://github.com/moby/moby --frontend-opt build-arg:APT_MIRROR=cdn-fastly.deb.debian.org
````
### Exporters

6
vendor/github.com/moby/buildkit/cache/remotecache/export.go generated vendored
View file

@ -9,7 +9,6 @@ import (
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/images"
"github.com/docker/distribution/manifest"
v1 "github.com/moby/buildkit/cache/remotecache/v1"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/solver"
@ -17,6 +16,7 @@ import (
"github.com/moby/buildkit/util/progress"
"github.com/moby/buildkit/util/push"
digest "github.com/opencontainers/go-digest"
specs "github.com/opencontainers/image-spec/specs-go"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
@ -46,7 +46,9 @@ func (ce *CacheExporter) Finalize(ctx context.Context, cc *v1.CacheChains, targe
// own type because oci type can't be pushed and docker type doesn't have annotations
type manifestList struct {
manifest.Versioned
specs.Versioned
MediaType string `json:"mediaType,omitempty"`
// Manifests references platform specific manifests.
Manifests []ocispec.Descriptor `json:"manifests"`

71
vendor/github.com/moby/buildkit/executor/oci/user.go generated vendored
View file

@ -2,27 +2,31 @@ package oci
import (
"context"
"errors"
"os"
"strconv"
"strings"
"github.com/containerd/containerd/containers"
containerdoci "github.com/containerd/containerd/oci"
"github.com/containerd/continuity/fs"
"github.com/opencontainers/runc/libcontainer/user"
"github.com/opencontainers/runtime-spec/specs-go"
)
func GetUser(ctx context.Context, root, username string) (uint32, uint32, error) {
func GetUser(ctx context.Context, root, username string) (uint32, uint32, []uint32, error) {
// fast path from uid/gid
if uid, gid, err := ParseUser(username); err == nil {
return uid, gid, nil
if uid, gid, err := ParseUIDGID(username); err == nil {
return uid, gid, nil, nil
}
passwdPath, err := user.GetPasswdPath()
if err != nil {
return 0, 0, err
return 0, 0, nil, err
}
groupPath, err := user.GetGroupPath()
if err != nil {
return 0, 0, err
return 0, 0, nil, err
}
passwdFile, err := openUserFile(root, passwdPath)
if err == nil {
@ -35,33 +39,29 @@ func GetUser(ctx context.Context, root, username string) (uint32, uint32, error)
execUser, err := user.GetExecUser(username, nil, passwdFile, groupFile)
if err != nil {
return 0, 0, err
return 0, 0, nil, err
}
return uint32(execUser.Uid), uint32(execUser.Gid), nil
var sgids []uint32
for _, g := range execUser.Sgids {
sgids = append(sgids, uint32(g))
}
return uint32(execUser.Uid), uint32(execUser.Gid), sgids, nil
}
func ParseUser(str string) (uid uint32, gid uint32, err error) {
// ParseUIDGID takes the fast path to parse UID and GID if and only if they are both provided
func ParseUIDGID(str string) (uid uint32, gid uint32, err error) {
if str == "" {
return 0, 0, nil
}
parts := strings.SplitN(str, ":", 2)
for i, v := range parts {
switch i {
case 0:
uid, err = parseUID(v)
if err != nil {
return 0, 0, err
}
if len(parts) == 1 {
gid = uid
}
case 1:
gid, err = parseUID(v)
if err != nil {
return 0, 0, err
}
}
if len(parts) == 1 {
return 0, 0, errors.New("groups ID is not provided")
}
if uid, err = parseUID(parts[0]); err != nil {
return 0, 0, err
}
if gid, err = parseUID(parts[1]); err != nil {
return 0, 0, err
}
return
}
@ -84,3 +84,24 @@ func parseUID(str string) (uint32, error) {
}
return uint32(uid), nil
}
// WithUIDGID allows the UID and GID for the Process to be set
// FIXME: This is a temporeray fix for the missing supplementary GIDs from containerd
// once the PR in containerd is merged we should remove this function.
func WithUIDGID(uid, gid uint32, sgids []uint32) containerdoci.SpecOpts {
return func(_ context.Context, _ containerdoci.Client, _ *containers.Container, s *containerdoci.Spec) error {
setProcess(s)
s.Process.User.UID = uid
s.Process.User.GID = gid
s.Process.User.AdditionalGids = sgids
return nil
}
}
// setProcess sets Process to empty if unset
// FIXME: Same on this one. Need to be removed after containerd fix merged
func setProcess(s *containerdoci.Spec) {
if s.Process == nil {
s.Process = &specs.Process{}
}
}

8
vendor/github.com/moby/buildkit/executor/runcexecutor/executor.go generated vendored
View file

@ -133,7 +133,7 @@ func (w *runcExecutor) Exec(ctx context.Context, meta executor.Meta, root cache.
}
defer mount.Unmount(rootFSPath, 0)
uid, gid, err := oci.GetUser(ctx, rootFSPath, meta.User)
uid, gid, sgids, err := oci.GetUser(ctx, rootFSPath, meta.User)
if err != nil {
return err
}
@ -143,7 +143,7 @@ func (w *runcExecutor) Exec(ctx context.Context, meta executor.Meta, root cache.
return err
}
defer f.Close()
opts := []containerdoci.SpecOpts{containerdoci.WithUIDGID(uid, gid)}
opts := []containerdoci.SpecOpts{oci.WithUIDGID(uid, gid, sgids)}
if system.SeccompSupported() {
opts = append(opts, seccomp.WithDefaultProfile())
}
@ -170,9 +170,7 @@ func (w *runcExecutor) Exec(ctx context.Context, meta executor.Meta, root cache.
}
if w.rootless {
specconv.ToRootless(spec, &specconv.RootlessOpts{
MapSubUIDGID: true,
})
specconv.ToRootless(spec, nil)
// TODO(AkihiroSuda): keep Cgroups enabled if /sys/fs/cgroup/cpuset/buildkit exists and writable
spec.Linux.CgroupsPath = ""
// TODO(AkihiroSuda): ToRootless removes netns, but we should readd netns here

127
vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert.go generated vendored
View file

@ -91,8 +91,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
metaResolver = imagemetaresolver.Default()
}
var allDispatchStates []*dispatchState
dispatchStatesByName := map[string]*dispatchState{}
allDispatchStates := newDispatchStates()
// set base state for every image
for _, st := range stages {
@ -100,6 +99,9 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
if err != nil {
return nil, nil, err
}
if name == "" {
return nil, nil, errors.Errorf("base name (%s) should not be blank", st.BaseName)
}
st.BaseName = name
ds := &dispatchState{
@ -121,13 +123,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
ds.platform = &p
}
if d, ok := dispatchStatesByName[st.BaseName]; ok {
ds.base = d
}
allDispatchStates = append(allDispatchStates, ds)
if st.Name != "" {
dispatchStatesByName[strings.ToLower(st.Name)] = ds
}
allDispatchStates.addState(ds)
if opt.IgnoreCache != nil {
if len(opt.IgnoreCache) == 0 {
ds.ignoreCache = true
@ -143,20 +139,20 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
var target *dispatchState
if opt.Target == "" {
target = allDispatchStates[len(allDispatchStates)-1]
target = allDispatchStates.lastTarget()
} else {
var ok bool
target, ok = dispatchStatesByName[strings.ToLower(opt.Target)]
target, ok = allDispatchStates.findStateByName(opt.Target)
if !ok {
return nil, nil, errors.Errorf("target stage %s could not be found", opt.Target)
}
}
// fill dependencies to stages so unreachable ones can avoid loading image configs
for _, d := range allDispatchStates {
for _, d := range allDispatchStates.states {
d.commands = make([]command, len(d.stage.Commands))
for i, cmd := range d.stage.Commands {
newCmd, err := toCommand(cmd, dispatchStatesByName, allDispatchStates)
newCmd, err := toCommand(cmd, allDispatchStates)
if err != nil {
return nil, nil, err
}
@ -165,7 +161,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
if src != nil {
d.deps[src] = struct{}{}
if src.unregistered {
allDispatchStates = append(allDispatchStates, src)
allDispatchStates.addState(src)
}
}
}
@ -173,7 +169,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
}
eg, ctx := errgroup.WithContext(ctx)
for i, d := range allDispatchStates {
for i, d := range allDispatchStates.states {
reachable := isReachable(target, d)
// resolve image config for every stage
if d.base == nil {
@ -239,7 +235,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
buildContext := &mutableOutput{}
ctxPaths := map[string]struct{}{}
for _, d := range allDispatchStates {
for _, d := range allDispatchStates.states {
if !isReachable(target, d) {
continue
}
@ -271,17 +267,16 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
}
opt := dispatchOpt{
allDispatchStates: allDispatchStates,
dispatchStatesByName: dispatchStatesByName,
metaArgs: metaArgs,
buildArgValues: opt.BuildArgs,
shlex: shlex,
sessionID: opt.SessionID,
buildContext: llb.NewState(buildContext),
proxyEnv: proxyEnv,
cacheIDNamespace: opt.CacheIDNamespace,
buildPlatforms: opt.BuildPlatforms,
targetPlatform: *opt.TargetPlatform,
allDispatchStates: allDispatchStates,
metaArgs: metaArgs,
buildArgValues: opt.BuildArgs,
shlex: shlex,
sessionID: opt.SessionID,
buildContext: llb.NewState(buildContext),
proxyEnv: proxyEnv,
cacheIDNamespace: opt.CacheIDNamespace,
buildPlatforms: opt.BuildPlatforms,
targetPlatform: *opt.TargetPlatform,
}
if err = dispatchOnBuild(d, d.image.Config.OnBuild, opt); err != nil {
@ -330,14 +325,14 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
return &st, &target.image, nil
}
func toCommand(ic instructions.Command, dispatchStatesByName map[string]*dispatchState, allDispatchStates []*dispatchState) (command, error) {
func toCommand(ic instructions.Command, allDispatchStates *dispatchStates) (command, error) {
cmd := command{Command: ic}
if c, ok := ic.(*instructions.CopyCommand); ok {
if c.From != "" {
var stn *dispatchState
index, err := strconv.Atoi(c.From)
if err != nil {
stn, ok = dispatchStatesByName[strings.ToLower(c.From)]
stn, ok = allDispatchStates.findStateByName(c.From)
if !ok {
stn = &dispatchState{
stage: instructions.Stage{BaseName: c.From},
@ -346,16 +341,16 @@ func toCommand(ic instructions.Command, dispatchStatesByName map[string]*dispatc
}
}
} else {
if index < 0 || index >= len(allDispatchStates) {
return command{}, errors.Errorf("invalid stage index %d", index)
stn, err = allDispatchStates.findStateByIndex(index)
if err != nil {
return command{}, err
}
stn = allDispatchStates[index]
}
cmd.sources = []*dispatchState{stn}
}
}
if ok := detectRunMount(&cmd, dispatchStatesByName, allDispatchStates); ok {
if ok := detectRunMount(&cmd, allDispatchStates); ok {
return cmd, nil
}
@ -363,17 +358,16 @@ func toCommand(ic instructions.Command, dispatchStatesByName map[string]*dispatc
}
type dispatchOpt struct {
allDispatchStates []*dispatchState
dispatchStatesByName map[string]*dispatchState
metaArgs []instructions.ArgCommand
buildArgValues map[string]string
shlex *shell.Lex
sessionID string
buildContext llb.State
proxyEnv *llb.ProxyEnv
cacheIDNamespace string
targetPlatform specs.Platform
buildPlatforms []specs.Platform
allDispatchStates *dispatchStates
metaArgs []instructions.ArgCommand
buildArgValues map[string]string
shlex *shell.Lex
sessionID string
buildContext llb.State
proxyEnv *llb.ProxyEnv
cacheIDNamespace string
targetPlatform specs.Platform
buildPlatforms []specs.Platform
}
func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
@ -456,6 +450,43 @@ type dispatchState struct {
unregistered bool
}
type dispatchStates struct {
states []*dispatchState
statesByName map[string]*dispatchState
}
func newDispatchStates() *dispatchStates {
return &dispatchStates{statesByName: map[string]*dispatchState{}}
}
func (dss *dispatchStates) addState(ds *dispatchState) {
dss.states = append(dss.states, ds)
if d, ok := dss.statesByName[ds.stage.BaseName]; ok {
ds.base = d
}
if ds.stage.Name != "" {
dss.statesByName[strings.ToLower(ds.stage.Name)] = ds
}
}
func (dss *dispatchStates) findStateByName(name string) (*dispatchState, bool) {
ds, ok := dss.statesByName[strings.ToLower(name)]
return ds, ok
}
func (dss *dispatchStates) findStateByIndex(index int) (*dispatchState, error) {
if index < 0 || index >= len(dss.states) {
return nil, errors.Errorf("invalid stage index %d", index)
}
return dss.states[index], nil
}
func (dss *dispatchStates) lastTarget() *dispatchState {
return dss.states[len(dss.states)-1]
}
type command struct {
instructions.Command
sources []*dispatchState
@ -474,7 +505,7 @@ func dispatchOnBuild(d *dispatchState, triggers []string, opt dispatchOpt) error
if err != nil {
return err
}
cmd, err := toCommand(ic, opt.dispatchStatesByName, opt.allDispatchStates)
cmd, err := toCommand(ic, opt.allDispatchStates)
if err != nil {
return err
}
@ -570,7 +601,11 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l
for i, src := range c.Sources() {
commitMessage.WriteString(" " + src)
if isAddCommand && (strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://")) {
if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
if !isAddCommand {
return errors.New("source can't be a URL for COPY")
}
// Resources from remote URLs are not decompressed.
// https://docs.docker.com/engine/reference/builder/#add
//

2
vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_norunmount.go generated vendored
View file

@ -7,7 +7,7 @@ import (
"github.com/moby/buildkit/frontend/dockerfile/instructions"
)
func detectRunMount(cmd *command, dispatchStatesByName map[string]*dispatchState, allDispatchStates []*dispatchState) bool {
func detectRunMount(cmd *command, allDispatchStates *dispatchStates) bool {
return false
}

5
vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_runmount.go generated vendored
View file

@ -5,14 +5,13 @@ package dockerfile2llb
import (
"path"
"path/filepath"
"strings"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/frontend/dockerfile/instructions"
"github.com/pkg/errors"
)
func detectRunMount(cmd *command, dispatchStatesByName map[string]*dispatchState, allDispatchStates []*dispatchState) bool {
func detectRunMount(cmd *command, allDispatchStates *dispatchStates) bool {
if c, ok := cmd.Command.(*instructions.RunCommand); ok {
mounts := instructions.GetMounts(c)
sources := make([]*dispatchState, len(mounts))
@ -24,7 +23,7 @@ func detectRunMount(cmd *command, dispatchStatesByName map[string]*dispatchState
if from == "" || mount.Type == instructions.MountTypeTmpfs {
continue
}
stn, ok := dispatchStatesByName[strings.ToLower(from)]
stn, ok := allDispatchStates.findStateByName(from)
if !ok {
stn = &dispatchState{
stage: instructions.Stage{BaseName: from},

2
vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/bflag.go generated vendored
View file

@ -72,7 +72,7 @@ func (bf *BFlags) AddString(name string, def string) *Flag {
return flag
}
// AddString adds a string flag to BFlags that can match multiple values
// AddStrings adds a string flag to BFlags that can match multiple values
func (bf *BFlags) AddStrings(name string) *Flag {
flag := bf.addFlag(name, stringsType)
if flag == nil {

2
vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands.go generated vendored
View file

@ -159,7 +159,7 @@ func (s SourcesAndDest) Dest() string {
// AddCommand : ADD foo /path
//
// Add the file 'foo' to '/path'. Tarball and Remote URL (git, http) handling
// Add the file 'foo' to '/path'. Tarball and Remote URL (http, https) handling
// exist here. If you do not wish to have this automatic handling, use COPY.
//
type AddCommand struct {

5
vendor/github.com/moby/buildkit/frontend/dockerfile/shell/equal_env_unix.go generated vendored
View file

@ -2,8 +2,9 @@
package shell
// EqualEnvKeys compare two strings and returns true if they are equal. On
// Windows this comparison is case insensitive.
// EqualEnvKeys compare two strings and returns true if they are equal.
// On Unix this comparison is case sensitive.
// On Windows this comparison is case insensitive.
func EqualEnvKeys(from, to string) bool {
return from == to
}

5
vendor/github.com/moby/buildkit/frontend/dockerfile/shell/equal_env_windows.go generated vendored
View file

@ -2,8 +2,9 @@ package shell
import "strings"
// EqualEnvKeys compare two strings and returns true if they are equal. On
// Windows this comparison is case insensitive.
// EqualEnvKeys compare two strings and returns true if they are equal.
// On Unix this comparison is case sensitive.
// On Windows this comparison is case insensitive.
func EqualEnvKeys(from, to string) bool {
return strings.ToUpper(from) == strings.ToUpper(to)
}

6
vendor/github.com/moby/buildkit/util/imageutil/config.go generated vendored
View file

@ -141,13 +141,17 @@ func DetectManifestMediaType(ra content.ReaderAt) (string, error) {
}
var mfst struct {
Config json.RawMessage `json:"config"`
MediaType string `json:"mediaType"`
Config json.RawMessage `json:"config"`
}
if err := json.Unmarshal(p, &mfst); err != nil {
return "", err
}
if mfst.MediaType != "" {
return mfst.MediaType, nil
}
if mfst.Config != nil {
return images.MediaTypeDockerSchema2Manifest, nil
}