Merge pull request #35177 from sargun/add-quota-tests

Add tests to project quotas and detection mechanism

daemon/graphdriver/quota/projectquota.go

@@ -47,6 +47,8 @@ struct fsxattr {
 #ifndef Q_XGETPQUOTA
 #define Q_XGETPQUOTA QCMD(Q_XGETQUOTA, PRJQUOTA)
 #endif
+
+const int Q_XGETQSTAT_PRJQUOTA = QCMD(Q_XGETQSTAT, PRJQUOTA);
 */
 import "C"
 import (
@@ -56,10 +58,15 @@ import (
 	"path/filepath"
 	"unsafe"
 
+	"errors"
+
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 )
 
+// ErrQuotaNotSupported indicates if were found the FS does not have projects quotas available
+var ErrQuotaNotSupported = errors.New("Filesystem does not support or has not enabled quotas")
+
 // Quota limit params - currently we only control blocks hard limit
 type Quota struct {
 	Size uint64
@@ -97,21 +104,31 @@ type Control struct {
 //
 func NewControl(basePath string) (*Control, error) {
 	//
-	// Get project id of parent dir as minimal id to be used by driver
+	// create backing filesystem device node
 	//
-	minProjectID, err := getProjectID(basePath)
+	backingFsBlockDev, err := makeBackingFsDev(basePath)
 	if err != nil {
 		return nil, err
 	}
-	minProjectID++
+
+	// check if we can call quotactl with project quotas
+	// as a mechanism to determine (early) if we have support
+	hasQuotaSupport, err := hasQuotaSupport(backingFsBlockDev)
+	if err != nil {
+		return nil, err
+	}
+	if !hasQuotaSupport {
+		return nil, ErrQuotaNotSupported
+	}
 
 	//
-	// create backing filesystem device node
+	// Get project id of parent dir as minimal id to be used by driver
 	//
-	backingFsBlockDev, err := makeBackingFsDev(basePath)
+	minProjectID, err := getProjectID(basePath)
 	if err != nil {
 		return nil, err
 	}
+	minProjectID++
 
 	//
 	// Test if filesystem supports project quotas by trying to set
@@ -335,3 +352,23 @@ func makeBackingFsDev(home string) (string, error) {
 
 	return backingFsBlockDev, nil
 }
+
+func hasQuotaSupport(backingFsBlockDev string) (bool, error) {
+	var cs = C.CString(backingFsBlockDev)
+	defer free(cs)
+	var qstat C.fs_quota_stat_t
+
+	_, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, uintptr(C.Q_XGETQSTAT_PRJQUOTA), uintptr(unsafe.Pointer(cs)), 0, uintptr(unsafe.Pointer(&qstat)), 0, 0)
+	if errno == 0 && qstat.qs_flags&C.FS_QUOTA_PDQ_ENFD > 0 && qstat.qs_flags&C.FS_QUOTA_PDQ_ACCT > 0 {
+		return true, nil
+	}
+
+	switch errno {
+	// These are the known fatal errors, consider all other errors (ENOTTY, etc.. not supporting quota)
+	case unix.EFAULT, unix.ENOENT, unix.ENOTBLK, unix.EPERM:
+	default:
+		return false, nil
+	}
+
+	return false, errno
+}

daemon/graphdriver/quota/projectquota_test.go

@@ -0,0 +1,161 @@
+// +build linux
+
+package quota
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
+)
+
+// 10MB
+const testQuotaSize = 10 * 1024 * 1024
+const imageSize = 64 * 1024 * 1024
+
+func TestBlockDev(t *testing.T) {
+	mkfs, err := exec.LookPath("mkfs.xfs")
+	if err != nil {
+		t.Fatal("mkfs.xfs not installed")
+	}
+
+	// create a sparse image
+	imageFile, err := ioutil.TempFile("", "xfs-image")
+	if err != nil {
+		t.Fatal(err)
+	}
+	imageFileName := imageFile.Name()
+	defer os.Remove(imageFileName)
+	if _, err = imageFile.Seek(imageSize-1, 0); err != nil {
+		t.Fatal(err)
+	}
+	if _, err = imageFile.Write([]byte{0}); err != nil {
+		t.Fatal(err)
+	}
+	if err = imageFile.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	// The reason for disabling these options is sometimes people run with a newer userspace
+	// than kernelspace
+	out, err := exec.Command(mkfs, "-m", "crc=0,finobt=0", imageFileName).CombinedOutput()
+	if len(out) > 0 {
+		t.Log(string(out))
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	runTest(t, "testBlockDevQuotaDisabled", wrapMountTest(imageFileName, false, testBlockDevQuotaDisabled))
+	runTest(t, "testBlockDevQuotaEnabled", wrapMountTest(imageFileName, true, testBlockDevQuotaEnabled))
+	runTest(t, "testSmallerThanQuota", wrapMountTest(imageFileName, true, wrapQuotaTest(testSmallerThanQuota)))
+	runTest(t, "testBiggerThanQuota", wrapMountTest(imageFileName, true, wrapQuotaTest(testBiggerThanQuota)))
+	runTest(t, "testRetrieveQuota", wrapMountTest(imageFileName, true, wrapQuotaTest(testRetrieveQuota)))
+}
+
+func runTest(t *testing.T, testName string, testFunc func(*testing.T)) {
+	if success := t.Run(testName, testFunc); !success {
+		out, _ := exec.Command("dmesg").CombinedOutput()
+		t.Log(string(out))
+	}
+}
+
+func wrapMountTest(imageFileName string, enableQuota bool, testFunc func(t *testing.T, mountPoint, backingFsDev string)) func(*testing.T) {
+	return func(t *testing.T) {
+		mountOptions := "loop"
+
+		if enableQuota {
+			mountOptions = mountOptions + ",prjquota"
+		}
+
+		// create a mountPoint
+		mountPoint, err := ioutil.TempDir("", "xfs-mountPoint")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer os.RemoveAll(mountPoint)
+
+		out, err := exec.Command("mount", "-o", mountOptions, imageFileName, mountPoint).CombinedOutput()
+		if len(out) > 0 {
+			t.Log(string(out))
+		}
+		if err != nil {
+			t.Fatal("mount failed")
+		}
+
+		defer func() {
+			if err := unix.Unmount(mountPoint, 0); err != nil {
+				t.Fatal(err)
+			}
+		}()
+
+		backingFsDev, err := makeBackingFsDev(mountPoint)
+		require.NoError(t, err)
+
+		testFunc(t, mountPoint, backingFsDev)
+	}
+}
+
+func testBlockDevQuotaDisabled(t *testing.T, mountPoint, backingFsDev string) {
+	hasSupport, err := hasQuotaSupport(backingFsDev)
+	require.NoError(t, err)
+	assert.False(t, hasSupport)
+}
+
+func testBlockDevQuotaEnabled(t *testing.T, mountPoint, backingFsDev string) {
+	hasSupport, err := hasQuotaSupport(backingFsDev)
+	require.NoError(t, err)
+	assert.True(t, hasSupport)
+}
+
+func wrapQuotaTest(testFunc func(t *testing.T, ctrl *Control, mountPoint, testDir, testSubDir string)) func(t *testing.T, mountPoint, backingFsDev string) {
+	return func(t *testing.T, mountPoint, backingFsDev string) {
+		testDir, err := ioutil.TempDir(mountPoint, "per-test")
+		require.NoError(t, err)
+		defer os.RemoveAll(testDir)
+
+		ctrl, err := NewControl(testDir)
+		require.NoError(t, err)
+
+		testSubDir, err := ioutil.TempDir(testDir, "quota-test")
+		require.NoError(t, err)
+		testFunc(t, ctrl, mountPoint, testDir, testSubDir)
+	}
+
+}
+
+func testSmallerThanQuota(t *testing.T, ctrl *Control, homeDir, testDir, testSubDir string) {
+	require.NoError(t, ctrl.SetQuota(testSubDir, Quota{testQuotaSize}))
+	smallerThanQuotaFile := filepath.Join(testSubDir, "smaller-than-quota")
+	require.NoError(t, ioutil.WriteFile(smallerThanQuotaFile, make([]byte, testQuotaSize/2), 0644))
+	require.NoError(t, os.Remove(smallerThanQuotaFile))
+}
+
+func testBiggerThanQuota(t *testing.T, ctrl *Control, homeDir, testDir, testSubDir string) {
+	// Make sure the quota is being enforced
+	// TODO: When we implement this under EXT4, we need to shed CAP_SYS_RESOURCE, otherwise
+	// we're able to violate quota without issue
+	require.NoError(t, ctrl.SetQuota(testSubDir, Quota{testQuotaSize}))
+
+	biggerThanQuotaFile := filepath.Join(testSubDir, "bigger-than-quota")
+	err := ioutil.WriteFile(biggerThanQuotaFile, make([]byte, testQuotaSize+1), 0644)
+	require.Error(t, err)
+	if err == io.ErrShortWrite {
+		require.NoError(t, os.Remove(biggerThanQuotaFile))
+	}
+}
+
+func testRetrieveQuota(t *testing.T, ctrl *Control, homeDir, testDir, testSubDir string) {
+	// Validate that we can retrieve quota
+	require.NoError(t, ctrl.SetQuota(testSubDir, Quota{testQuotaSize}))
+
+	var q Quota
+	require.NoError(t, ctrl.GetQuota(testSubDir, &q))
+	assert.EqualValues(t, testQuotaSize, q.Size)
+}