API,daemon: support type URL parameter to /system/df

Let clients choose object types to compute disk usage of.

Signed-off-by: Roman Volosatovs <roman.volosatovs@docker.com>
Co-authored-by: Sebastiaan van Stijn <github@gone.nl>

api/server/router/system/backend.go

@@ -10,12 +10,24 @@ import (
 	"github.com/docker/docker/api/types/swarm"
 )
 
+// DiskUsageOptions holds parameters for system disk usage query.
+type DiskUsageOptions struct {
+	// Containers controls whether container disk usage should be computed.
+	Containers bool
+
+	// Images controls whether image disk usage should be computed.
+	Images bool
+
+	// Volumes controls whether volume disk usage should be computed.
+	Volumes bool
+}
+
 // Backend is the methods that need to be implemented to provide
 // system specific functionality.
 type Backend interface {
 	SystemInfo() *types.Info
 	SystemVersion() types.Version
-	SystemDiskUsage(ctx context.Context) (*types.DiskUsage, error)
+	SystemDiskUsage(ctx context.Context, opts DiskUsageOptions) (*types.DiskUsage, error)
 	SubscribeToEvents(since, until time.Time, ef filters.Args) ([]events.Message, chan interface{})
 	UnsubscribeFromEvents(chan interface{})
 	AuthenticateToRegistry(ctx context.Context, authConfig *types.AuthConfig) (string, string, error)

api/server/router/system/system_routes.go

@@ -16,7 +16,7 @@ import (
 	timetypes "github.com/docker/docker/api/types/time"
 	"github.com/docker/docker/api/types/versions"
 	"github.com/docker/docker/pkg/ioutils"
-	pkgerrors "github.com/pkg/errors"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
 )
@@ -90,44 +90,83 @@ func (s *systemRouter) getVersion(ctx context.Context, w http.ResponseWriter, r
 }
 
 func (s *systemRouter) getDiskUsage(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
+	if err := httputils.ParseForm(r); err != nil {
+		return err
+	}
+
+	var getContainers, getImages, getVolumes, getBuildCache bool
+	if typeStrs, ok := r.Form["type"]; !ok {
+		getContainers, getImages, getVolumes, getBuildCache = true, true, true, true
+	} else {
+		for _, typ := range typeStrs {
+			switch types.DiskUsageObject(typ) {
+			case types.ContainerObject:
+				getContainers = true
+			case types.ImageObject:
+				getImages = true
+			case types.VolumeObject:
+				getVolumes = true
+			case types.BuildCacheObject:
+				getBuildCache = true
+			default:
+				return invalidRequestError{Err: fmt.Errorf("unknown object type: %s", typ)}
+			}
+		}
+	}
+
 	eg, ctx := errgroup.WithContext(ctx)
 
-	var du *types.DiskUsage
-	eg.Go(func() error {
-		var err error
-		du, err = s.backend.SystemDiskUsage(ctx)
-		return err
-	})
+	var systemDiskUsage *types.DiskUsage
+	if getContainers || getImages || getVolumes {
+		eg.Go(func() error {
+			var err error
+			systemDiskUsage, err = s.backend.SystemDiskUsage(ctx, DiskUsageOptions{
+				Containers: getContainers,
+				Images:     getImages,
+				Volumes:    getVolumes,
+			})
+			return err
+		})
+	}
 
 	var buildCache []*types.BuildCache
-	eg.Go(func() error {
-		var err error
-		buildCache, err = s.builder.DiskUsage(ctx)
-		if err != nil {
-			return pkgerrors.Wrap(err, "error getting build cache usage")
-		}
-		return nil
-	})
+	if getBuildCache {
+		eg.Go(func() error {
+			var err error
+			buildCache, err = s.builder.DiskUsage(ctx)
+			if err != nil {
+				return errors.Wrap(err, "error getting build cache usage")
+			}
+			if buildCache == nil {
+				// Ensure empty `BuildCache` field is represented as empty JSON array(`[]`)
+				// instead of `null` to be consistent with `Images`, `Containers` etc.
+				buildCache = []*types.BuildCache{}
+			}
+			return nil
+		})
+	}
 
 	if err := eg.Wait(); err != nil {
 		return err
 	}
 
+	var builderSize int64
 	if versions.LessThan(httputils.VersionFromContext(ctx), "1.42") {
-		var builderSize int64
 		for _, b := range buildCache {
 			builderSize += b.Size
 		}
-		du.BuilderSize = builderSize
 	}
 
-	du.BuildCache = buildCache
-	if buildCache == nil {
-		// Ensure empty `BuildCache` field is represented as empty JSON array(`[]`)
-		// instead of `null` to be consistent with `Images`, `Containers` etc.
-		du.BuildCache = []*types.BuildCache{}
+	du := types.DiskUsage{
+		BuildCache:  buildCache,
+		BuilderSize: builderSize,
+	}
+	if systemDiskUsage != nil {
+		du.LayersSize = systemDiskUsage.LayersSize
+		du.Images = systemDiskUsage.Images
+		du.Containers = systemDiskUsage.Containers
+		du.Volumes = systemDiskUsage.Volumes
 	}
-
 	return httputils.WriteJSON(w, http.StatusOK, du)
 }
 

api/swagger.yaml

@@ -8371,6 +8371,16 @@ paths:
           description: "server error"
           schema:
             $ref: "#/definitions/ErrorResponse"
+      parameters:
+        - name: "type"
+          in: "query"
+          description: |
+            Object types, for which to compute and return data.
+          type: "array"
+          collectionFormat: multi
+          items:
+            type: "string"
+            enum: ["container", "image", "volume", "build-cache"]
       tags: ["System"]
   /images/{name}/get:
     get:

api/types/types.go

@@ -535,6 +535,27 @@ type ShimConfig struct {
 	Opts   interface{}
 }
 
+// DiskUsageObject represents an object type used for disk usage query filtering.
+type DiskUsageObject string
+
+const (
+	// ContainerObject represents a container DiskUsageObject.
+	ContainerObject DiskUsageObject = "container"
+	// ImageObject represents an image DiskUsageObject.
+	ImageObject DiskUsageObject = "image"
+	// VolumeObject represents a volume DiskUsageObject.
+	VolumeObject DiskUsageObject = "volume"
+	// BuildCacheObject represents a build-cache DiskUsageObject.
+	BuildCacheObject DiskUsageObject = "build-cache"
+)
+
+// DiskUsageOptions holds parameters for system disk usage query.
+type DiskUsageOptions struct {
+	// Types specifies what object types to include in the response. If empty,
+	// all object types are returned.
+	Types []DiskUsageObject
+}
+
 // DiskUsage contains response of Engine API:
 // GET "/system/df"
 type DiskUsage struct {

client/disk_usage.go

@@ -4,23 +4,30 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/url"
 
 	"github.com/docker/docker/api/types"
 )
 
 // DiskUsage requests the current data usage from the daemon
-func (cli *Client) DiskUsage(ctx context.Context) (types.DiskUsage, error) {
-	var du types.DiskUsage
+func (cli *Client) DiskUsage(ctx context.Context, options types.DiskUsageOptions) (types.DiskUsage, error) {
+	var query url.Values
+	if len(options.Types) > 0 {
+		query = url.Values{}
+		for _, t := range options.Types {
+			query.Add("type", string(t))
+		}
+	}
 
-	serverResp, err := cli.get(ctx, "/system/df", nil, nil)
+	serverResp, err := cli.get(ctx, "/system/df", query, nil)
 	defer ensureReaderClosed(serverResp)
 	if err != nil {
-		return du, err
+		return types.DiskUsage{}, err
 	}
 
+	var du types.DiskUsage
 	if err := json.NewDecoder(serverResp.body).Decode(&du); err != nil {
-		return du, fmt.Errorf("Error retrieving disk usage: %v", err)
+		return types.DiskUsage{}, fmt.Errorf("Error retrieving disk usage: %v", err)
 	}
-
 	return du, nil
 }

client/disk_usage_test.go

@@ -18,7 +18,7 @@ func TestDiskUsageError(t *testing.T) {
 	client := &Client{
 		client: newMockClient(errorMock(http.StatusInternalServerError, "Server error")),
 	}
-	_, err := client.DiskUsage(context.Background())
+	_, err := client.DiskUsage(context.Background(), types.DiskUsageOptions{})
 	if !errdefs.IsSystem(err) {
 		t.Fatalf("expected a Server Error, got %[1]T: %[1]v", err)
 	}
@@ -50,7 +50,7 @@ func TestDiskUsage(t *testing.T) {
 			}, nil
 		}),
 	}
-	if _, err := client.DiskUsage(context.Background()); err != nil {
+	if _, err := client.DiskUsage(context.Background(), types.DiskUsageOptions{}); err != nil {
 		t.Fatal(err)
 	}
 }

client/interface.go

@@ -168,7 +168,7 @@ type SystemAPIClient interface {
 	Events(ctx context.Context, options types.EventsOptions) (<-chan events.Message, <-chan error)
 	Info(ctx context.Context) (types.Info, error)
 	RegistryLogin(ctx context.Context, auth types.AuthConfig) (registry.AuthenticateOKBody, error)
-	DiskUsage(ctx context.Context) (types.DiskUsage, error)
+	DiskUsage(ctx context.Context, options types.DiskUsageOptions) (types.DiskUsage, error)
 	Ping(ctx context.Context) (types.Ping, error)
 }
 

daemon/disk_usage.go

@@ -5,50 +5,64 @@ import (
 	"fmt"
 	"sync/atomic"
 
+	"github.com/docker/docker/api/server/router/system"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 )
 
 // SystemDiskUsage returns information about the daemon data disk usage
-func (daemon *Daemon) SystemDiskUsage(ctx context.Context) (*types.DiskUsage, error) {
+func (daemon *Daemon) SystemDiskUsage(ctx context.Context, opts system.DiskUsageOptions) (*types.DiskUsage, error) {
 	if !atomic.CompareAndSwapInt32(&daemon.diskUsageRunning, 0, 1) {
 		return nil, fmt.Errorf("a disk usage operation is already running")
 	}
 	defer atomic.StoreInt32(&daemon.diskUsageRunning, 0)
 
-	// Retrieve container list
-	allContainers, err := daemon.Containers(&types.ContainerListOptions{
-		Size: true,
-		All:  true,
-	})
-	if err != nil {
-		return nil, fmt.Errorf("failed to retrieve container list: %v", err)
+	var err error
+
+	var containers []*types.Container
+	if opts.Containers {
+		// Retrieve container list
+		containers, err = daemon.Containers(&types.ContainerListOptions{
+			Size: true,
+			All:  true,
+		})
+		if err != nil {
+			return nil, fmt.Errorf("failed to retrieve container list: %v", err)
+		}
 	}
 
-	// Get all top images with extra attributes
-	allImages, err := daemon.imageService.Images(ctx, types.ImageListOptions{
-		Filters:        filters.NewArgs(),
-		SharedSize:     true,
-		ContainerCount: true,
-	})
-	if err != nil {
-		return nil, fmt.Errorf("failed to retrieve image list: %v", err)
+	var (
+		images     []*types.ImageSummary
+		layersSize int64
+	)
+	if opts.Images {
+		// Get all top images with extra attributes
+		images, err = daemon.imageService.Images(ctx, types.ImageListOptions{
+			Filters:        filters.NewArgs(),
+			SharedSize:     true,
+			ContainerCount: true,
+		})
+		if err != nil {
+			return nil, fmt.Errorf("failed to retrieve image list: %v", err)
+		}
+
+		layersSize, err = daemon.imageService.LayerDiskUsage(ctx)
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	localVolumes, err := daemon.volumes.LocalVolumesSize(ctx)
-	if err != nil {
-		return nil, err
+	var volumes []*types.Volume
+	if opts.Volumes {
+		volumes, err = daemon.volumes.LocalVolumesSize(ctx)
+		if err != nil {
+			return nil, err
+		}
 	}
-
-	allLayersSize, err := daemon.imageService.LayerDiskUsage(ctx)
-	if err != nil {
-		return nil, err
-	}
-
 	return &types.DiskUsage{
-		LayersSize: allLayersSize,
-		Containers: allContainers,
-		Volumes:    localVolumes,
-		Images:     allImages,
+		LayersSize: layersSize,
+		Containers: containers,
+		Volumes:    volumes,
+		Images:     images,
 	}, nil
 }

docs/api/version-history.md

@@ -24,6 +24,10 @@ keywords: "API, Docker, rcli, REST, documentation"
 * `GET /images/json` now accepts query parameter `shared-size`. When set `true`,
   images returned will include `SharedSize`, which provides the size on disk shared
   with other images present on the system.
+* `GET /system/df` now accepts query parameter `type`. When set,
+  computes and returns data only for the specified object type.
+  The parameter can be specified multiple times to select several object types.
+  Supported values are: `container`, `image`, `volume`, `build-cache`.
 
 ## v1.41 API changes
 

integration/build/build_session_test.go

@@ -53,14 +53,14 @@ func TestBuildWithSession(t *testing.T) {
 	assert.Check(t, is.Equal(strings.Count(out, "Using cache"), 2))
 	assert.Check(t, is.Contains(out, "contentcontent"))
 
-	du, err := client.DiskUsage(context.TODO())
+	du, err := client.DiskUsage(context.TODO(), types.DiskUsageOptions{})
 	assert.Check(t, err)
 	assert.Check(t, du.BuilderSize > 10)
 
 	out = testBuildWithSession(t, client, client.DaemonHost(), fctx.Dir, dockerfile)
 	assert.Check(t, is.Equal(strings.Count(out, "Using cache"), 4))
 
-	du2, err := client.DiskUsage(context.TODO())
+	du2, err := client.DiskUsage(context.TODO(), types.DiskUsageOptions{})
 	assert.Check(t, err)
 	assert.Check(t, is.Equal(du.BuilderSize, du2.BuilderSize))
 
@@ -84,7 +84,7 @@ func TestBuildWithSession(t *testing.T) {
 	_, err = client.BuildCachePrune(context.TODO(), types.BuildCachePruneOptions{All: true})
 	assert.Check(t, err)
 
-	du, err = client.DiskUsage(context.TODO())
+	du, err = client.DiskUsage(context.TODO(), types.DiskUsageOptions{})
 	assert.Check(t, err)
 	assert.Check(t, is.Equal(du.BuilderSize, int64(0)))
 }

integration/system/disk_usage_test.go

@@ -0,0 +1,274 @@
+package system // import "github.com/docker/docker/integration/system"
+
+import (
+	"context"
+	"testing"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/integration/internal/container"
+	"github.com/docker/docker/testutil/daemon"
+	"gotest.tools/v3/assert"
+	"gotest.tools/v3/skip"
+)
+
+func TestDiskUsage(t *testing.T) {
+	skip.If(t, testEnv.OSType == "windows") // d.Start fails on Windows with `protocol not available`
+
+	t.Parallel()
+
+	d := daemon.New(t)
+	defer d.Cleanup(t)
+	d.Start(t, "--iptables=false")
+	defer d.Stop(t)
+	client := d.NewClientT(t)
+
+	ctx := context.Background()
+
+	var stepDU types.DiskUsage
+	for _, step := range []struct {
+		doc  string
+		next func(t *testing.T, prev types.DiskUsage) types.DiskUsage
+	}{
+		{
+			doc: "empty",
+			next: func(t *testing.T, _ types.DiskUsage) types.DiskUsage {
+				du, err := client.DiskUsage(ctx, types.DiskUsageOptions{})
+				assert.NilError(t, err)
+				assert.DeepEqual(t, du, types.DiskUsage{
+					Images:     []*types.ImageSummary{},
+					Containers: []*types.Container{},
+					Volumes:    []*types.Volume{},
+					BuildCache: []*types.BuildCache{},
+				})
+				return du
+			},
+		},
+		{
+			doc: "after LoadBusybox",
+			next: func(t *testing.T, _ types.DiskUsage) types.DiskUsage {
+				d.LoadBusybox(t)
+
+				du, err := client.DiskUsage(ctx, types.DiskUsageOptions{})
+				assert.NilError(t, err)
+				assert.Assert(t, du.LayersSize > 0)
+				assert.Equal(t, len(du.Images), 1)
+				assert.DeepEqual(t, du, types.DiskUsage{
+					LayersSize: du.LayersSize,
+					Images: []*types.ImageSummary{
+						{
+							Created:     du.Images[0].Created,
+							ID:          du.Images[0].ID,
+							RepoTags:    []string{"busybox:latest"},
+							Size:        du.LayersSize,
+							VirtualSize: du.LayersSize,
+						},
+					},
+					Containers: []*types.Container{},
+					Volumes:    []*types.Volume{},
+					BuildCache: []*types.BuildCache{},
+				})
+				return du
+			},
+		},
+		{
+			doc: "after container.Run",
+			next: func(t *testing.T, prev types.DiskUsage) types.DiskUsage {
+				cID := container.Run(ctx, t, client)
+
+				du, err := client.DiskUsage(ctx, types.DiskUsageOptions{})
+				assert.NilError(t, err)
+				assert.Equal(t, len(du.Containers), 1)
+				assert.Equal(t, len(du.Containers[0].Names), 1)
+				assert.Assert(t, du.Containers[0].Created >= prev.Images[0].Created)
+				assert.DeepEqual(t, du, types.DiskUsage{
+					LayersSize: prev.LayersSize,
+					Images: []*types.ImageSummary{
+						func() *types.ImageSummary {
+							sum := *prev.Images[0]
+							sum.Containers++
+							return &sum
+						}(),
+					},
+					Containers: []*types.Container{
+						{
+							ID:              cID,
+							Names:           du.Containers[0].Names,
+							Image:           "busybox",
+							ImageID:         prev.Images[0].ID,
+							Command:         du.Containers[0].Command, // not relevant for the test
+							Created:         du.Containers[0].Created,
+							Ports:           du.Containers[0].Ports, // not relevant for the test
+							SizeRootFs:      prev.Images[0].Size,
+							Labels:          du.Containers[0].Labels,          // not relevant for the test
+							State:           du.Containers[0].State,           // not relevant for the test
+							Status:          du.Containers[0].Status,          // not relevant for the test
+							HostConfig:      du.Containers[0].HostConfig,      // not relevant for the test
+							NetworkSettings: du.Containers[0].NetworkSettings, // not relevant for the test
+							Mounts:          du.Containers[0].Mounts,          // not relevant for the test
+						},
+					},
+					Volumes:    []*types.Volume{},
+					BuildCache: []*types.BuildCache{},
+				})
+				return du
+			},
+		},
+	} {
+		t.Run(step.doc, func(t *testing.T) {
+			stepDU = step.next(t, stepDU)
+
+			for _, tc := range []struct {
+				doc      string
+				options  types.DiskUsageOptions
+				expected types.DiskUsage
+			}{
+				{
+					doc: "container types",
+					options: types.DiskUsageOptions{
+						Types: []types.DiskUsageObject{
+							types.ContainerObject,
+						},
+					},
+					expected: types.DiskUsage{
+						Containers: stepDU.Containers,
+					},
+				},
+				{
+					doc: "image types",
+					options: types.DiskUsageOptions{
+						Types: []types.DiskUsageObject{
+							types.ImageObject,
+						},
+					},
+					expected: types.DiskUsage{
+						LayersSize: stepDU.LayersSize,
+						Images:     stepDU.Images,
+					},
+				},
+				{
+					doc: "volume types",
+					options: types.DiskUsageOptions{
+						Types: []types.DiskUsageObject{
+							types.VolumeObject,
+						},
+					},
+					expected: types.DiskUsage{
+						Volumes: stepDU.Volumes,
+					},
+				},
+				{
+					doc: "build-cache types",
+					options: types.DiskUsageOptions{
+						Types: []types.DiskUsageObject{
+							types.BuildCacheObject,
+						},
+					},
+					expected: types.DiskUsage{
+						BuildCache: stepDU.BuildCache,
+					},
+				},
+				{
+					doc: "container, volume types",
+					options: types.DiskUsageOptions{
+						Types: []types.DiskUsageObject{
+							types.ContainerObject,
+							types.VolumeObject,
+						},
+					},
+					expected: types.DiskUsage{
+						Containers: stepDU.Containers,
+						Volumes:    stepDU.Volumes,
+					},
+				},
+				{
+					doc: "image, build-cache types",
+					options: types.DiskUsageOptions{
+						Types: []types.DiskUsageObject{
+							types.ImageObject,
+							types.BuildCacheObject,
+						},
+					},
+					expected: types.DiskUsage{
+						LayersSize: stepDU.LayersSize,
+						Images:     stepDU.Images,
+						BuildCache: stepDU.BuildCache,
+					},
+				},
+				{
+					doc: "container, volume, build-cache types",
+					options: types.DiskUsageOptions{
+						Types: []types.DiskUsageObject{
+							types.ContainerObject,
+							types.VolumeObject,
+							types.BuildCacheObject,
+						},
+					},
+					expected: types.DiskUsage{
+						Containers: stepDU.Containers,
+						Volumes:    stepDU.Volumes,
+						BuildCache: stepDU.BuildCache,
+					},
+				},
+				{
+					doc: "image, volume, build-cache types",
+					options: types.DiskUsageOptions{
+						Types: []types.DiskUsageObject{
+							types.ImageObject,
+							types.VolumeObject,
+							types.BuildCacheObject,
+						},
+					},
+					expected: types.DiskUsage{
+						LayersSize: stepDU.LayersSize,
+						Images:     stepDU.Images,
+						Volumes:    stepDU.Volumes,
+						BuildCache: stepDU.BuildCache,
+					},
+				},
+				{
+					doc: "container, image, volume types",
+					options: types.DiskUsageOptions{
+						Types: []types.DiskUsageObject{
+							types.ContainerObject,
+							types.ImageObject,
+							types.VolumeObject,
+						},
+					},
+					expected: types.DiskUsage{
+						LayersSize: stepDU.LayersSize,
+						Containers: stepDU.Containers,
+						Images:     stepDU.Images,
+						Volumes:    stepDU.Volumes,
+					},
+				},
+				{
+					doc: "container, image, volume, build-cache types",
+					options: types.DiskUsageOptions{
+						Types: []types.DiskUsageObject{
+							types.ContainerObject,
+							types.ImageObject,
+							types.VolumeObject,
+							types.BuildCacheObject,
+						},
+					},
+					expected: types.DiskUsage{
+						LayersSize: stepDU.LayersSize,
+						Containers: stepDU.Containers,
+						Images:     stepDU.Images,
+						Volumes:    stepDU.Volumes,
+						BuildCache: stepDU.BuildCache,
+					},
+				},
+			} {
+				tc := tc
+				t.Run(tc.doc, func(t *testing.T) {
+					// TODO: Run in parallel once https://github.com/moby/moby/pull/42560 is merged.
+
+					du, err := client.DiskUsage(ctx, tc.options)
+					assert.NilError(t, err)
+					assert.DeepEqual(t, du, tc.expected)
+				})
+			}
+		})
+	}
+}