From 47ad2f3dd61d99ce1a24ad750252c29a27df249d Mon Sep 17 00:00:00 2001 From: Roman Volosatovs Date: Wed, 23 Jun 2021 15:26:54 +0200 Subject: [PATCH] API,daemon: support `type` URL parameter to /system/df Let clients choose object types to compute disk usage of. Signed-off-by: Roman Volosatovs Co-authored-by: Sebastiaan van Stijn --- api/server/router/system/backend.go | 14 +- api/server/router/system/system_routes.go | 85 +++++-- api/swagger.yaml | 10 + api/types/types.go | 21 ++ client/disk_usage.go | 19 +- client/disk_usage_test.go | 4 +- client/interface.go | 2 +- daemon/disk_usage.go | 72 +++--- docs/api/version-history.md | 4 + integration/build/build_session_test.go | 6 +- integration/system/disk_usage_test.go | 274 ++++++++++++++++++++++ 11 files changed, 446 insertions(+), 65 deletions(-) create mode 100644 integration/system/disk_usage_test.go diff --git a/api/server/router/system/backend.go b/api/server/router/system/backend.go index 2a9ffd6d1d..1d704348c4 100644 --- a/api/server/router/system/backend.go +++ b/api/server/router/system/backend.go @@ -10,12 +10,24 @@ import ( "github.com/docker/docker/api/types/swarm" ) +// DiskUsageOptions holds parameters for system disk usage query. +type DiskUsageOptions struct { + // Containers controls whether container disk usage should be computed. + Containers bool + + // Images controls whether image disk usage should be computed. + Images bool + + // Volumes controls whether volume disk usage should be computed. + Volumes bool +} + // Backend is the methods that need to be implemented to provide // system specific functionality. type Backend interface { SystemInfo() *types.Info SystemVersion() types.Version - SystemDiskUsage(ctx context.Context) (*types.DiskUsage, error) + SystemDiskUsage(ctx context.Context, opts DiskUsageOptions) (*types.DiskUsage, error) SubscribeToEvents(since, until time.Time, ef filters.Args) ([]events.Message, chan interface{}) UnsubscribeFromEvents(chan interface{}) AuthenticateToRegistry(ctx context.Context, authConfig *types.AuthConfig) (string, string, error) diff --git a/api/server/router/system/system_routes.go b/api/server/router/system/system_routes.go index f116364f5b..3d2efb6052 100644 --- a/api/server/router/system/system_routes.go +++ b/api/server/router/system/system_routes.go @@ -16,7 +16,7 @@ import ( timetypes "github.com/docker/docker/api/types/time" "github.com/docker/docker/api/types/versions" "github.com/docker/docker/pkg/ioutils" - pkgerrors "github.com/pkg/errors" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" ) @@ -90,44 +90,83 @@ func (s *systemRouter) getVersion(ctx context.Context, w http.ResponseWriter, r } func (s *systemRouter) getDiskUsage(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if err := httputils.ParseForm(r); err != nil { + return err + } + + var getContainers, getImages, getVolumes, getBuildCache bool + if typeStrs, ok := r.Form["type"]; !ok { + getContainers, getImages, getVolumes, getBuildCache = true, true, true, true + } else { + for _, typ := range typeStrs { + switch types.DiskUsageObject(typ) { + case types.ContainerObject: + getContainers = true + case types.ImageObject: + getImages = true + case types.VolumeObject: + getVolumes = true + case types.BuildCacheObject: + getBuildCache = true + default: + return invalidRequestError{Err: fmt.Errorf("unknown object type: %s", typ)} + } + } + } + eg, ctx := errgroup.WithContext(ctx) - var du *types.DiskUsage - eg.Go(func() error { - var err error - du, err = s.backend.SystemDiskUsage(ctx) - return err - }) + var systemDiskUsage *types.DiskUsage + if getContainers || getImages || getVolumes { + eg.Go(func() error { + var err error + systemDiskUsage, err = s.backend.SystemDiskUsage(ctx, DiskUsageOptions{ + Containers: getContainers, + Images: getImages, + Volumes: getVolumes, + }) + return err + }) + } var buildCache []*types.BuildCache - eg.Go(func() error { - var err error - buildCache, err = s.builder.DiskUsage(ctx) - if err != nil { - return pkgerrors.Wrap(err, "error getting build cache usage") - } - return nil - }) + if getBuildCache { + eg.Go(func() error { + var err error + buildCache, err = s.builder.DiskUsage(ctx) + if err != nil { + return errors.Wrap(err, "error getting build cache usage") + } + if buildCache == nil { + // Ensure empty `BuildCache` field is represented as empty JSON array(`[]`) + // instead of `null` to be consistent with `Images`, `Containers` etc. + buildCache = []*types.BuildCache{} + } + return nil + }) + } if err := eg.Wait(); err != nil { return err } + var builderSize int64 if versions.LessThan(httputils.VersionFromContext(ctx), "1.42") { - var builderSize int64 for _, b := range buildCache { builderSize += b.Size } - du.BuilderSize = builderSize } - du.BuildCache = buildCache - if buildCache == nil { - // Ensure empty `BuildCache` field is represented as empty JSON array(`[]`) - // instead of `null` to be consistent with `Images`, `Containers` etc. - du.BuildCache = []*types.BuildCache{} + du := types.DiskUsage{ + BuildCache: buildCache, + BuilderSize: builderSize, + } + if systemDiskUsage != nil { + du.LayersSize = systemDiskUsage.LayersSize + du.Images = systemDiskUsage.Images + du.Containers = systemDiskUsage.Containers + du.Volumes = systemDiskUsage.Volumes } - return httputils.WriteJSON(w, http.StatusOK, du) } diff --git a/api/swagger.yaml b/api/swagger.yaml index b4dbbc1f68..bdaa378eb6 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -8371,6 +8371,16 @@ paths: description: "server error" schema: $ref: "#/definitions/ErrorResponse" + parameters: + - name: "type" + in: "query" + description: | + Object types, for which to compute and return data. + type: "array" + collectionFormat: multi + items: + type: "string" + enum: ["container", "image", "volume", "build-cache"] tags: ["System"] /images/{name}/get: get: diff --git a/api/types/types.go b/api/types/types.go index d6017898d0..a186550d72 100644 --- a/api/types/types.go +++ b/api/types/types.go @@ -535,6 +535,27 @@ type ShimConfig struct { Opts interface{} } +// DiskUsageObject represents an object type used for disk usage query filtering. +type DiskUsageObject string + +const ( + // ContainerObject represents a container DiskUsageObject. + ContainerObject DiskUsageObject = "container" + // ImageObject represents an image DiskUsageObject. + ImageObject DiskUsageObject = "image" + // VolumeObject represents a volume DiskUsageObject. + VolumeObject DiskUsageObject = "volume" + // BuildCacheObject represents a build-cache DiskUsageObject. + BuildCacheObject DiskUsageObject = "build-cache" +) + +// DiskUsageOptions holds parameters for system disk usage query. +type DiskUsageOptions struct { + // Types specifies what object types to include in the response. If empty, + // all object types are returned. + Types []DiskUsageObject +} + // DiskUsage contains response of Engine API: // GET "/system/df" type DiskUsage struct { diff --git a/client/disk_usage.go b/client/disk_usage.go index 354cd36939..ba0d92e9e6 100644 --- a/client/disk_usage.go +++ b/client/disk_usage.go @@ -4,23 +4,30 @@ import ( "context" "encoding/json" "fmt" + "net/url" "github.com/docker/docker/api/types" ) // DiskUsage requests the current data usage from the daemon -func (cli *Client) DiskUsage(ctx context.Context) (types.DiskUsage, error) { - var du types.DiskUsage +func (cli *Client) DiskUsage(ctx context.Context, options types.DiskUsageOptions) (types.DiskUsage, error) { + var query url.Values + if len(options.Types) > 0 { + query = url.Values{} + for _, t := range options.Types { + query.Add("type", string(t)) + } + } - serverResp, err := cli.get(ctx, "/system/df", nil, nil) + serverResp, err := cli.get(ctx, "/system/df", query, nil) defer ensureReaderClosed(serverResp) if err != nil { - return du, err + return types.DiskUsage{}, err } + var du types.DiskUsage if err := json.NewDecoder(serverResp.body).Decode(&du); err != nil { - return du, fmt.Errorf("Error retrieving disk usage: %v", err) + return types.DiskUsage{}, fmt.Errorf("Error retrieving disk usage: %v", err) } - return du, nil } diff --git a/client/disk_usage_test.go b/client/disk_usage_test.go index 4166e8e2bd..bfa53ef4a1 100644 --- a/client/disk_usage_test.go +++ b/client/disk_usage_test.go @@ -18,7 +18,7 @@ func TestDiskUsageError(t *testing.T) { client := &Client{ client: newMockClient(errorMock(http.StatusInternalServerError, "Server error")), } - _, err := client.DiskUsage(context.Background()) + _, err := client.DiskUsage(context.Background(), types.DiskUsageOptions{}) if !errdefs.IsSystem(err) { t.Fatalf("expected a Server Error, got %[1]T: %[1]v", err) } @@ -50,7 +50,7 @@ func TestDiskUsage(t *testing.T) { }, nil }), } - if _, err := client.DiskUsage(context.Background()); err != nil { + if _, err := client.DiskUsage(context.Background(), types.DiskUsageOptions{}); err != nil { t.Fatal(err) } } diff --git a/client/interface.go b/client/interface.go index aabad4a911..7277d1bbdd 100644 --- a/client/interface.go +++ b/client/interface.go @@ -168,7 +168,7 @@ type SystemAPIClient interface { Events(ctx context.Context, options types.EventsOptions) (<-chan events.Message, <-chan error) Info(ctx context.Context) (types.Info, error) RegistryLogin(ctx context.Context, auth types.AuthConfig) (registry.AuthenticateOKBody, error) - DiskUsage(ctx context.Context) (types.DiskUsage, error) + DiskUsage(ctx context.Context, options types.DiskUsageOptions) (types.DiskUsage, error) Ping(ctx context.Context) (types.Ping, error) } diff --git a/daemon/disk_usage.go b/daemon/disk_usage.go index 9e59510aa1..519b96c099 100644 --- a/daemon/disk_usage.go +++ b/daemon/disk_usage.go @@ -5,50 +5,64 @@ import ( "fmt" "sync/atomic" + "github.com/docker/docker/api/server/router/system" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" ) // SystemDiskUsage returns information about the daemon data disk usage -func (daemon *Daemon) SystemDiskUsage(ctx context.Context) (*types.DiskUsage, error) { +func (daemon *Daemon) SystemDiskUsage(ctx context.Context, opts system.DiskUsageOptions) (*types.DiskUsage, error) { if !atomic.CompareAndSwapInt32(&daemon.diskUsageRunning, 0, 1) { return nil, fmt.Errorf("a disk usage operation is already running") } defer atomic.StoreInt32(&daemon.diskUsageRunning, 0) - // Retrieve container list - allContainers, err := daemon.Containers(&types.ContainerListOptions{ - Size: true, - All: true, - }) - if err != nil { - return nil, fmt.Errorf("failed to retrieve container list: %v", err) + var err error + + var containers []*types.Container + if opts.Containers { + // Retrieve container list + containers, err = daemon.Containers(&types.ContainerListOptions{ + Size: true, + All: true, + }) + if err != nil { + return nil, fmt.Errorf("failed to retrieve container list: %v", err) + } } - // Get all top images with extra attributes - allImages, err := daemon.imageService.Images(ctx, types.ImageListOptions{ - Filters: filters.NewArgs(), - SharedSize: true, - ContainerCount: true, - }) - if err != nil { - return nil, fmt.Errorf("failed to retrieve image list: %v", err) + var ( + images []*types.ImageSummary + layersSize int64 + ) + if opts.Images { + // Get all top images with extra attributes + images, err = daemon.imageService.Images(ctx, types.ImageListOptions{ + Filters: filters.NewArgs(), + SharedSize: true, + ContainerCount: true, + }) + if err != nil { + return nil, fmt.Errorf("failed to retrieve image list: %v", err) + } + + layersSize, err = daemon.imageService.LayerDiskUsage(ctx) + if err != nil { + return nil, err + } } - localVolumes, err := daemon.volumes.LocalVolumesSize(ctx) - if err != nil { - return nil, err + var volumes []*types.Volume + if opts.Volumes { + volumes, err = daemon.volumes.LocalVolumesSize(ctx) + if err != nil { + return nil, err + } } - - allLayersSize, err := daemon.imageService.LayerDiskUsage(ctx) - if err != nil { - return nil, err - } - return &types.DiskUsage{ - LayersSize: allLayersSize, - Containers: allContainers, - Volumes: localVolumes, - Images: allImages, + LayersSize: layersSize, + Containers: containers, + Volumes: volumes, + Images: images, }, nil } diff --git a/docs/api/version-history.md b/docs/api/version-history.md index 15a759a235..a36764d2de 100644 --- a/docs/api/version-history.md +++ b/docs/api/version-history.md @@ -24,6 +24,10 @@ keywords: "API, Docker, rcli, REST, documentation" * `GET /images/json` now accepts query parameter `shared-size`. When set `true`, images returned will include `SharedSize`, which provides the size on disk shared with other images present on the system. +* `GET /system/df` now accepts query parameter `type`. When set, + computes and returns data only for the specified object type. + The parameter can be specified multiple times to select several object types. + Supported values are: `container`, `image`, `volume`, `build-cache`. ## v1.41 API changes diff --git a/integration/build/build_session_test.go b/integration/build/build_session_test.go index 7c626416a1..2192481656 100644 --- a/integration/build/build_session_test.go +++ b/integration/build/build_session_test.go @@ -53,14 +53,14 @@ func TestBuildWithSession(t *testing.T) { assert.Check(t, is.Equal(strings.Count(out, "Using cache"), 2)) assert.Check(t, is.Contains(out, "contentcontent")) - du, err := client.DiskUsage(context.TODO()) + du, err := client.DiskUsage(context.TODO(), types.DiskUsageOptions{}) assert.Check(t, err) assert.Check(t, du.BuilderSize > 10) out = testBuildWithSession(t, client, client.DaemonHost(), fctx.Dir, dockerfile) assert.Check(t, is.Equal(strings.Count(out, "Using cache"), 4)) - du2, err := client.DiskUsage(context.TODO()) + du2, err := client.DiskUsage(context.TODO(), types.DiskUsageOptions{}) assert.Check(t, err) assert.Check(t, is.Equal(du.BuilderSize, du2.BuilderSize)) @@ -84,7 +84,7 @@ func TestBuildWithSession(t *testing.T) { _, err = client.BuildCachePrune(context.TODO(), types.BuildCachePruneOptions{All: true}) assert.Check(t, err) - du, err = client.DiskUsage(context.TODO()) + du, err = client.DiskUsage(context.TODO(), types.DiskUsageOptions{}) assert.Check(t, err) assert.Check(t, is.Equal(du.BuilderSize, int64(0))) } diff --git a/integration/system/disk_usage_test.go b/integration/system/disk_usage_test.go new file mode 100644 index 0000000000..6c40a9b2fd --- /dev/null +++ b/integration/system/disk_usage_test.go @@ -0,0 +1,274 @@ +package system // import "github.com/docker/docker/integration/system" + +import ( + "context" + "testing" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/integration/internal/container" + "github.com/docker/docker/testutil/daemon" + "gotest.tools/v3/assert" + "gotest.tools/v3/skip" +) + +func TestDiskUsage(t *testing.T) { + skip.If(t, testEnv.OSType == "windows") // d.Start fails on Windows with `protocol not available` + + t.Parallel() + + d := daemon.New(t) + defer d.Cleanup(t) + d.Start(t, "--iptables=false") + defer d.Stop(t) + client := d.NewClientT(t) + + ctx := context.Background() + + var stepDU types.DiskUsage + for _, step := range []struct { + doc string + next func(t *testing.T, prev types.DiskUsage) types.DiskUsage + }{ + { + doc: "empty", + next: func(t *testing.T, _ types.DiskUsage) types.DiskUsage { + du, err := client.DiskUsage(ctx, types.DiskUsageOptions{}) + assert.NilError(t, err) + assert.DeepEqual(t, du, types.DiskUsage{ + Images: []*types.ImageSummary{}, + Containers: []*types.Container{}, + Volumes: []*types.Volume{}, + BuildCache: []*types.BuildCache{}, + }) + return du + }, + }, + { + doc: "after LoadBusybox", + next: func(t *testing.T, _ types.DiskUsage) types.DiskUsage { + d.LoadBusybox(t) + + du, err := client.DiskUsage(ctx, types.DiskUsageOptions{}) + assert.NilError(t, err) + assert.Assert(t, du.LayersSize > 0) + assert.Equal(t, len(du.Images), 1) + assert.DeepEqual(t, du, types.DiskUsage{ + LayersSize: du.LayersSize, + Images: []*types.ImageSummary{ + { + Created: du.Images[0].Created, + ID: du.Images[0].ID, + RepoTags: []string{"busybox:latest"}, + Size: du.LayersSize, + VirtualSize: du.LayersSize, + }, + }, + Containers: []*types.Container{}, + Volumes: []*types.Volume{}, + BuildCache: []*types.BuildCache{}, + }) + return du + }, + }, + { + doc: "after container.Run", + next: func(t *testing.T, prev types.DiskUsage) types.DiskUsage { + cID := container.Run(ctx, t, client) + + du, err := client.DiskUsage(ctx, types.DiskUsageOptions{}) + assert.NilError(t, err) + assert.Equal(t, len(du.Containers), 1) + assert.Equal(t, len(du.Containers[0].Names), 1) + assert.Assert(t, du.Containers[0].Created >= prev.Images[0].Created) + assert.DeepEqual(t, du, types.DiskUsage{ + LayersSize: prev.LayersSize, + Images: []*types.ImageSummary{ + func() *types.ImageSummary { + sum := *prev.Images[0] + sum.Containers++ + return &sum + }(), + }, + Containers: []*types.Container{ + { + ID: cID, + Names: du.Containers[0].Names, + Image: "busybox", + ImageID: prev.Images[0].ID, + Command: du.Containers[0].Command, // not relevant for the test + Created: du.Containers[0].Created, + Ports: du.Containers[0].Ports, // not relevant for the test + SizeRootFs: prev.Images[0].Size, + Labels: du.Containers[0].Labels, // not relevant for the test + State: du.Containers[0].State, // not relevant for the test + Status: du.Containers[0].Status, // not relevant for the test + HostConfig: du.Containers[0].HostConfig, // not relevant for the test + NetworkSettings: du.Containers[0].NetworkSettings, // not relevant for the test + Mounts: du.Containers[0].Mounts, // not relevant for the test + }, + }, + Volumes: []*types.Volume{}, + BuildCache: []*types.BuildCache{}, + }) + return du + }, + }, + } { + t.Run(step.doc, func(t *testing.T) { + stepDU = step.next(t, stepDU) + + for _, tc := range []struct { + doc string + options types.DiskUsageOptions + expected types.DiskUsage + }{ + { + doc: "container types", + options: types.DiskUsageOptions{ + Types: []types.DiskUsageObject{ + types.ContainerObject, + }, + }, + expected: types.DiskUsage{ + Containers: stepDU.Containers, + }, + }, + { + doc: "image types", + options: types.DiskUsageOptions{ + Types: []types.DiskUsageObject{ + types.ImageObject, + }, + }, + expected: types.DiskUsage{ + LayersSize: stepDU.LayersSize, + Images: stepDU.Images, + }, + }, + { + doc: "volume types", + options: types.DiskUsageOptions{ + Types: []types.DiskUsageObject{ + types.VolumeObject, + }, + }, + expected: types.DiskUsage{ + Volumes: stepDU.Volumes, + }, + }, + { + doc: "build-cache types", + options: types.DiskUsageOptions{ + Types: []types.DiskUsageObject{ + types.BuildCacheObject, + }, + }, + expected: types.DiskUsage{ + BuildCache: stepDU.BuildCache, + }, + }, + { + doc: "container, volume types", + options: types.DiskUsageOptions{ + Types: []types.DiskUsageObject{ + types.ContainerObject, + types.VolumeObject, + }, + }, + expected: types.DiskUsage{ + Containers: stepDU.Containers, + Volumes: stepDU.Volumes, + }, + }, + { + doc: "image, build-cache types", + options: types.DiskUsageOptions{ + Types: []types.DiskUsageObject{ + types.ImageObject, + types.BuildCacheObject, + }, + }, + expected: types.DiskUsage{ + LayersSize: stepDU.LayersSize, + Images: stepDU.Images, + BuildCache: stepDU.BuildCache, + }, + }, + { + doc: "container, volume, build-cache types", + options: types.DiskUsageOptions{ + Types: []types.DiskUsageObject{ + types.ContainerObject, + types.VolumeObject, + types.BuildCacheObject, + }, + }, + expected: types.DiskUsage{ + Containers: stepDU.Containers, + Volumes: stepDU.Volumes, + BuildCache: stepDU.BuildCache, + }, + }, + { + doc: "image, volume, build-cache types", + options: types.DiskUsageOptions{ + Types: []types.DiskUsageObject{ + types.ImageObject, + types.VolumeObject, + types.BuildCacheObject, + }, + }, + expected: types.DiskUsage{ + LayersSize: stepDU.LayersSize, + Images: stepDU.Images, + Volumes: stepDU.Volumes, + BuildCache: stepDU.BuildCache, + }, + }, + { + doc: "container, image, volume types", + options: types.DiskUsageOptions{ + Types: []types.DiskUsageObject{ + types.ContainerObject, + types.ImageObject, + types.VolumeObject, + }, + }, + expected: types.DiskUsage{ + LayersSize: stepDU.LayersSize, + Containers: stepDU.Containers, + Images: stepDU.Images, + Volumes: stepDU.Volumes, + }, + }, + { + doc: "container, image, volume, build-cache types", + options: types.DiskUsageOptions{ + Types: []types.DiskUsageObject{ + types.ContainerObject, + types.ImageObject, + types.VolumeObject, + types.BuildCacheObject, + }, + }, + expected: types.DiskUsage{ + LayersSize: stepDU.LayersSize, + Containers: stepDU.Containers, + Images: stepDU.Images, + Volumes: stepDU.Volumes, + BuildCache: stepDU.BuildCache, + }, + }, + } { + tc := tc + t.Run(tc.doc, func(t *testing.T) { + // TODO: Run in parallel once https://github.com/moby/moby/pull/42560 is merged. + + du, err := client.DiskUsage(ctx, tc.options) + assert.NilError(t, err) + assert.DeepEqual(t, du, tc.expected) + }) + } + }) + } +}