beenull/moby
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

builder: add prune options to the API

Browse source 
Signed-off-by: Tibor Vass <tibor@docker.com>
This commit is contained in:
Tibor Vass 2018-08-15 21:24:37 +00:00
parent a005332346
commit 8ff7847d1c
9 changed files with 190 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
api/server/backend/build/backend.go
View file

@ -88,7 +88,7 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string
}
// PruneCache removes all cached build sources
func (b *Backend) PruneCache(ctx context.Context) (*types.BuildCachePruneReport, error) {
func (b *Backend) PruneCache(ctx context.Context, opts types.BuildCachePruneOptions) (*types.BuildCachePruneReport, error) {
eg, ctx := errgroup.WithContext(ctx)
var fsCacheSize uint64
@ -102,9 +102,10 @@ func (b *Backend) PruneCache(ctx context.Context) (*types.BuildCachePruneReport,
})
var buildCacheSize int64
var cacheIDs []string
eg.Go(func() error {
var err error
buildCacheSize, err = b.buildkit.Prune(ctx)
buildCacheSize, cacheIDs, err = b.buildkit.Prune(ctx, opts)
if err != nil {
return errors.Wrap(err, "failed to prune build cache")
}
@ -115,7 +116,7 @@ func (b *Backend) PruneCache(ctx context.Context) (*types.BuildCachePruneReport,
return nil, err
}
return &types.BuildCachePruneReport{SpaceReclaimed: fsCacheSize + uint64(buildCacheSize)}, nil
return &types.BuildCachePruneReport{SpaceReclaimed: fsCacheSize + uint64(buildCacheSize), CachesDeleted: cacheIDs}, nil
}
// Cancel cancels the build by ID

2
api/server/router/build/backend.go
View file

@ -14,7 +14,7 @@ type Backend interface {
Build(context.Context, backend.BuildConfig) (string, error)
// Prune build cache
PruneCache(context.Context) (*types.BuildCachePruneReport, error)
PruneCache(context.Context, types.BuildCachePruneOptions) (*types.BuildCachePruneReport, error)
Cancel(context.Context, string) error
}

22
api/server/router/build/build_routes.go
View file

@ -18,6 +18,7 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/backend"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/api/types/versions"
"github.com/docker/docker/errdefs"
"github.com/docker/docker/pkg/ioutils"
@ -161,7 +162,26 @@ func parseVersion(s string) (types.BuilderVersion, error) {
}
func (br *buildRouter) postPrune(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
report, err := br.backend.PruneCache(ctx)
if err := httputils.ParseForm(r); err != nil {
return err
}
filters, err := filters.FromJSON(r.Form.Get("filters"))
if err != nil {
return errors.Wrap(err, "could not parse filters")
}
ksfv := r.FormValue("keep-storage")
ks, err := strconv.Atoi(ksfv)
if err != nil {
return errors.Wrapf(err, "keep-storage is in bytes and expects an integer, got %v", ksfv)
}
opts := types.BuildCachePruneOptions{
All: httputils.BoolValue(r, "all"),
Filters: filters,
KeepStorage: int64(ks),
}
report, err := br.backend.PruneCache(ctx, opts)
if err != nil {
return err
}

57
api/swagger.yaml
View file

@ -1513,6 +1513,31 @@ definitions:
aux:
$ref: "#/definitions/ImageID"
BuildCache:
type: "object"
properties:
ID:
type: "string"
Parent:
type: "string"
Type:
type: "string"
Description:
type: "string"
InUse:
type: "boolean"
Shared:
type: "boolean"
Size:
type: "integer"
CreatedAt:
type: "integer"
LastUsedAt:
type: "integer"
x-nullable: true
UsageCount:
type: "integer"
ImageID:
type: "object"
description: "Image ID or Digest"
@ -6358,6 +6383,29 @@ paths:
produces:
- "application/json"
operationId: "BuildPrune"
parameters:
- name: "keep-storage"
in: "query"
description: "Amount of disk space in bytes to keep for cache"
type: "integer"
format: "int64"
- name: "all"
in: "query"
type: "boolean"
description: "Remove all types of build cache"
- name: "filters"
in: "query"
type: "string"
description: |
A JSON encoded value of the filters (a `map[string][]string`) to process on the list of build cache objects. Available filters:
- `unused-for=<duration>`: duration relative to daemon's time, during which build cache was not used, in Go's duration format (e.g., '24h')
- `id=<id>`
- `parent=<id>`
- `type=<string>`
- `description=<string>`
- `inuse`
- `shared`
- `private`
responses:
200:
description: "No error"
@ -6365,6 +6413,11 @@ paths:
type: "object"
title: "BuildPruneResponse"
properties:
CachesDeleted:
type: "array"
items:
description: "ID of build cache object"
type: "string"
SpaceReclaimed:
description: "Disk space reclaimed in bytes"
type: "integer"
@ -7199,6 +7252,10 @@ paths:
type: "array"
items:
$ref: "#/definitions/Volume"
BuildCache:
type: "array"
items:
$ref: "#/definitions/BuildCache"
example:
LayersSize: 1092588
Images:

22
api/types/types.go
View file

@ -543,6 +543,7 @@ type ImagesPruneReport struct {
// BuildCachePruneReport contains the response for Engine API:
// POST "/build/prune"
type BuildCachePruneReport struct {
CachesDeleted []string
SpaceReclaimed uint64
}
@ -592,14 +593,21 @@ type BuildResult struct {
// BuildCache contains information about a build cache record
type BuildCache struct {
ID string
Mutable bool
InUse bool
Size int64
ID string
Parent string
Type string
Description string
InUse bool
Shared bool
Size int64
CreatedAt time.Time
LastUsedAt *time.Time
UsageCount int
Parent string
Description string
}
// BuildCachePruneOptions hold parameters to prune the build cache
type BuildCachePruneOptions struct {
All bool
KeepStorage int64
Filters filters.Args
}

83
builder/builder-next/builder.go
View file

@ -29,6 +29,21 @@ import (
grpcmetadata "google.golang.org/grpc/metadata"
)
var errMultipleFilterValues = errors.New("filters expect only one value")
var cacheFields = map[string]bool{
"id": true,
"parent": true,
"type": true,
"description": true,
"inuse": true,
"shared": true,
"private": true,
// fields from buildkit that are not exposed
"mutable": false,
"immutable": false,
}
func init() {
llbsolver.AllowNetworkHostUnstable = true
}
@ -87,48 +102,94 @@ func (b *Builder) DiskUsage(ctx context.Context) ([]*types.BuildCache, error) {
var items []*types.BuildCache
for _, r := range duResp.Record {
items = append(items, &types.BuildCache{
ID: r.ID,
Mutable: r.Mutable,
InUse: r.InUse,
Size: r.Size_,
ID: r.ID,
Parent: r.Parent,
Type: r.RecordType,
Description: r.Description,
InUse: r.InUse,
Shared: r.Shared,
Size: r.Size_,
CreatedAt: r.CreatedAt,
LastUsedAt: r.LastUsedAt,
UsageCount: int(r.UsageCount),
Parent: r.Parent,
Description: r.Description,
})
}
return items, nil
}
// Prune clears all reclaimable build cache
func (b *Builder) Prune(ctx context.Context) (int64, error) {
func (b *Builder) Prune(ctx context.Context, opts types.BuildCachePruneOptions) (int64, []string, error) {
ch := make(chan *controlapi.UsageRecord)
eg, ctx := errgroup.WithContext(ctx)
validFilters := make(map[string]bool, 1+len(cacheFields))
validFilters["unused-for"] = true
for k, v := range cacheFields {
validFilters[k] = v
}
if err := opts.Filters.Validate(validFilters); err != nil {
return 0, nil, err
}
var unusedFor time.Duration
unusedForValues := opts.Filters.Get("unused-for")
switch len(unusedForValues) {
case 0:
case 1:
var err error
unusedFor, err = time.ParseDuration(unusedForValues[0])
if err != nil {
return 0, nil, errors.Wrap(err, "unused-for filter expects a duration (e.g., '24h')")
}
default:
return 0, nil, errMultipleFilterValues
}
bkFilter := make([]string, 0, opts.Filters.Len())
for cacheField := range cacheFields {
values := opts.Filters.Get(cacheField)
switch len(values) {
case 0:
bkFilter = append(bkFilter, cacheField)
case 1:
bkFilter = append(bkFilter, cacheField+"=="+values[0])
default:
return 0, nil, errMultipleFilterValues
}
}
eg.Go(func() error {
defer close(ch)
return b.controller.Prune(&controlapi.PruneRequest{}, &pruneProxy{
return b.controller.Prune(&controlapi.PruneRequest{
All: opts.All,
KeepDuration: int64(unusedFor),
KeepBytes: opts.KeepStorage,
Filter: bkFilter,
}, &pruneProxy{
streamProxy: streamProxy{ctx: ctx},
ch: ch,
})
})
var size int64
var cacheIDs []string
eg.Go(func() error {
for r := range ch {
size += r.Size_
cacheIDs = append(cacheIDs, r.ID)
}
return nil
})
if err := eg.Wait(); err != nil {
return 0, err
return 0, nil, err
}
return size, nil
return size, cacheIDs, nil
}
// Build executes a build request

19
client/build_prune.go
View file

@ -4,19 +4,34 @@ import (
"context"
"encoding/json"
"fmt"
"net/url"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/pkg/errors"
)
// BuildCachePrune requests the daemon to delete unused cache data
func (cli *Client) BuildCachePrune(ctx context.Context) (*types.BuildCachePruneReport, error) {
func (cli *Client) BuildCachePrune(ctx context.Context, opts types.BuildCachePruneOptions) (*types.BuildCachePruneReport, error) {
if err := cli.NewVersionError("1.31", "build prune"); err != nil {
return nil, err
}
report := types.BuildCachePruneReport{}
serverResp, err := cli.post(ctx, "/build/prune", nil, nil, nil)
query := url.Values{}
if opts.All {
query.Set("all", "1")
}
query.Set("keep-storage", fmt.Sprintf("%d", opts.KeepStorage))
filters, err := filters.ToJSON(opts.Filters)
if err != nil {
return nil, errors.Wrap(err, "prune could not marshal filters option")
}
query.Set("filters", filters)
serverResp, err := cli.post(ctx, "/build/prune", query, nil, nil)
if err != nil {
return nil, err
}

2
client/interface.go
View file

@ -86,7 +86,7 @@ type DistributionAPIClient interface {
// ImageAPIClient defines API client methods for the images
type ImageAPIClient interface {
ImageBuild(ctx context.Context, context io.Reader, options types.ImageBuildOptions) (types.ImageBuildResponse, error)
BuildCachePrune(ctx context.Context) (*types.BuildCachePruneReport, error)
BuildCachePrune(ctx context.Context, opts types.BuildCachePruneOptions) (*types.BuildCachePruneReport, error)
BuildCancel(ctx context.Context, id string) error
ImageCreate(ctx context.Context, parentReference string, options types.ImageCreateOptions) (io.ReadCloser, error)
ImageHistory(ctx context.Context, image string) ([]image.HistoryResponseItem, error)

3
integration/build/build_session_test.go
View file

@ -7,6 +7,7 @@ import (
"strings"
"testing"
"github.com/docker/docker/api/types"
dclient "github.com/docker/docker/client"
"github.com/docker/docker/internal/test/fakecontext"
"github.com/docker/docker/internal/test/request"
@ -76,7 +77,7 @@ func TestBuildWithSession(t *testing.T) {
assert.Check(t, is.Contains(string(outBytes), "Successfully built"))
assert.Check(t, is.Equal(strings.Count(string(outBytes), "Using cache"), 4))
_, err = client.BuildCachePrune(context.TODO())
_, err = client.BuildCachePrune(context.TODO(), types.BuildCachePruneOptions{All: true})
assert.Check(t, err)
du, err = client.DiskUsage(context.TODO())