Merge pull request #21380 from sainath14/improve_help_text_test_time

Improve the time taken by DockerSuite.TestHelpTextVerify

integration-cli/docker_cli_help_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"fmt"
 	"os/exec"
 	"runtime"
 	"strings"
@@ -13,6 +14,7 @@ import (
 
 func (s *DockerSuite) TestHelpTextVerify(c *check.C) {
 	testRequires(c, DaemonIsLinux)
+
 	// Make sure main help text fits within 80 chars and that
 	// on non-windows system we use ~ when possible (to shorten things).
 	// Test for HOME set to its default value and set to "/" on linux
@@ -40,6 +42,7 @@ func (s *DockerSuite) TestHelpTextVerify(c *check.C) {
 	}
 
 	for _, home := range homes {
+
 		// Dup baseEnvs and add our new HOME value
 		newEnvs := make([]string, len(baseEnvs)+1)
 		copy(newEnvs, baseEnvs)
@@ -117,114 +120,22 @@ func (s *DockerSuite) TestHelpTextVerify(c *check.C) {
 		cmdsToTest = append(cmdsToTest, "volume ls")
 		cmdsToTest = append(cmdsToTest, "volume rm")
 
-		for _, cmd := range cmdsToTest {
-			var stderr string
-
-			args := strings.Split(cmd+" --help", " ")
-
-			// Check the full usage text
-			helpCmd := exec.Command(dockerBinary, args...)
-			helpCmd.Env = newEnvs
-			out, stderr, _, err = runCommandWithStdoutStderr(helpCmd)
-			c.Assert(len(stderr), checker.Equals, 0, check.Commentf("Error on %q help. non-empty stderr:%q", cmd, stderr))
-			c.Assert(out, checker.Not(checker.HasSuffix), "\n\n", check.Commentf("Should not have blank line on %q\n", cmd))
-			c.Assert(out, checker.Contains, "--help", check.Commentf("All commands should mention '--help'. Command '%v' did not.\n", cmd))
-
-			c.Assert(err, checker.IsNil, check.Commentf(out))
-
-			// Check each line for lots of stuff
-			lines := strings.Split(out, "\n")
-			for _, line := range lines {
-				c.Assert(len(line), checker.LessOrEqualThan, 107, check.Commentf("Help for %q is too long:\n%s", cmd, line))
-
-				if scanForHome && strings.Contains(line, `"`+home) {
-					c.Fatalf("Help for %q should use ~ instead of %q on:\n%s",
-						cmd, home, line)
-				}
-				i := strings.Index(line, "~")
-				if i >= 0 && i != len(line)-1 && line[i+1] != '/' {
-					c.Fatalf("Help for %q should not have used ~:\n%s", cmd, line)
-				}
-
-				// If a line starts with 4 spaces then assume someone
-				// added a multi-line description for an option and we need
-				// to flag it
-				c.Assert(line, checker.Not(checker.HasPrefix), "    ", check.Commentf("Help for %q should not have a multi-line option", cmd))
-
-				// Options should NOT end with a period
-				if strings.HasPrefix(line, "  -") && strings.HasSuffix(line, ".") {
-					c.Fatalf("Help for %q should not end with a period: %s", cmd, line)
-				}
-
-				// Options should NOT end with a space
-				c.Assert(line, checker.Not(checker.HasSuffix), " ", check.Commentf("Help for %q should not end with a space", cmd))
-
-			}
-
-			// For each command make sure we generate an error
-			// if we give a bad arg
-			args = strings.Split(cmd+" --badArg", " ")
-
-			out, _, err = dockerCmdWithError(args...)
-			c.Assert(err, checker.NotNil, check.Commentf(out))
-			// Be really picky
-			c.Assert(stderr, checker.Not(checker.HasSuffix), "\n\n", check.Commentf("Should not have a blank line at the end of 'docker rm'\n"))
-
-			// Now make sure that each command will print a short-usage
-			// (not a full usage - meaning no opts section) if we
-			// are missing a required arg or pass in a bad arg
-
-			// These commands will never print a short-usage so don't test
-			noShortUsage := map[string]string{
-				"images":  "",
-				"login":   "",
-				"logout":  "",
-				"network": "",
-				"stats":   "",
-			}
+		// Divide the list of commands into go routines and  run the func testcommand on the commands in parallel
+		// to save runtime of test
 
-			if _, ok := noShortUsage[cmd]; !ok {
-				// For each command run it w/o any args. It will either return
-				// valid output or print a short-usage
-				var dCmd *exec.Cmd
-				var stdout, stderr string
-				var args []string
-
-				// skipNoArgs are ones that we don't want to try w/o
-				// any args. Either because it'll hang the test or
-				// lead to incorrect test result (like false negative).
-				// Whatever the reason, skip trying to run w/o args and
-				// jump to trying with a bogus arg.
-				skipNoArgs := map[string]struct{}{
-					"daemon": {},
-					"events": {},
-					"load":   {},
-				}
+		errChan := make(chan error)
 
-				ec := 0
-				if _, ok := skipNoArgs[cmd]; !ok {
-					args = strings.Split(cmd, " ")
-					dCmd = exec.Command(dockerBinary, args...)
-					stdout, stderr, ec, err = runCommandWithStdoutStderr(dCmd)
-				}
-
-				// If its ok w/o any args then try again with an arg
-				if ec == 0 {
-					args = strings.Split(cmd+" badArg", " ")
-					dCmd = exec.Command(dockerBinary, args...)
-					stdout, stderr, ec, err = runCommandWithStdoutStderr(dCmd)
-				}
+		for index := 0; index < len(cmdsToTest); index++ {
+			go func(index int) {
+				errChan <- testCommand(cmdsToTest[index], newEnvs, scanForHome, home)
+			}(index)
+		}
 
-				if len(stdout) != 0 || len(stderr) == 0 || ec == 0 || err == nil {
-					c.Fatalf("Bad output from %q\nstdout:%q\nstderr:%q\nec:%d\nerr:%q", args, stdout, stderr, ec, err)
-				}
-				// Should have just short usage
-				c.Assert(stderr, checker.Contains, "\nUsage:\t", check.Commentf("Missing short usage on %q\n", args))
-				// But shouldn't have full usage
-				c.Assert(stderr, checker.Not(checker.Contains), "--help=false", check.Commentf("Should not have full usage on %q\n", args))
-				c.Assert(stderr, checker.Not(checker.HasSuffix), "\n\n", check.Commentf("Should not have a blank line on %q\n", args))
+		for index := 0; index < len(cmdsToTest); index++ {
+			err := <-errChan
+			if err != nil {
+				c.Fatal(err)
 			}
-
 		}
 
 		// Number of commands for standard release and experimental release
@@ -296,3 +207,136 @@ func (s *DockerSuite) TestHelpExitCodesHelpOutput(c *check.C) {
 	c.Assert(stdout, checker.Equals, "")
 	c.Assert(stderr, checker.Equals, "docker: 'BadCmd' is not a docker command.\nSee 'docker --help'.\n", check.Commentf("Unexcepted output for 'docker badCmd'\n"))
 }
+
+func testCommand(cmd string, newEnvs []string, scanForHome bool, home string) error {
+
+	args := strings.Split(cmd+" --help", " ")
+
+	// Check the full usage text
+	helpCmd := exec.Command(dockerBinary, args...)
+	helpCmd.Env = newEnvs
+	out, stderr, _, err := runCommandWithStdoutStderr(helpCmd)
+	if len(stderr) != 0 {
+		return fmt.Errorf("Error on %q help. non-empty stderr:%q\n", cmd, stderr)
+	}
+	if strings.HasSuffix(out, "\n\n") {
+		return fmt.Errorf("Should not have blank line on %q\n", cmd)
+	}
+	if !strings.Contains(out, "--help") {
+		return fmt.Errorf("All commands should mention '--help'. Command '%v' did not.\n", cmd)
+	}
+
+	if err != nil {
+		return fmt.Errorf(out)
+	}
+
+	// Check each line for lots of stuff
+	lines := strings.Split(out, "\n")
+	for _, line := range lines {
+		if len(line) > 107 {
+			return fmt.Errorf("Help for %q is too long:\n%s\n", cmd, line)
+		}
+
+		if scanForHome && strings.Contains(line, `"`+home) {
+			return fmt.Errorf("Help for %q should use ~ instead of %q on:\n%s\n",
+				cmd, home, line)
+		}
+		i := strings.Index(line, "~")
+		if i >= 0 && i != len(line)-1 && line[i+1] != '/' {
+			return fmt.Errorf("Help for %q should not have used ~:\n%s", cmd, line)
+		}
+
+		// If a line starts with 4 spaces then assume someone
+		// added a multi-line description for an option and we need
+		// to flag it
+		if strings.HasPrefix(line, "    ") {
+			return fmt.Errorf("Help for %q should not have a multi-line option", cmd)
+		}
+
+		// Options should NOT end with a period
+		if strings.HasPrefix(line, "  -") && strings.HasSuffix(line, ".") {
+			return fmt.Errorf("Help for %q should not end with a period: %s", cmd, line)
+		}
+
+		// Options should NOT end with a space
+		if strings.HasSuffix(line, " ") {
+			return fmt.Errorf("Help for %q should not end with a space", cmd)
+		}
+
+	}
+
+	// For each command make sure we generate an error
+	// if we give a bad arg
+	args = strings.Split(cmd+" --badArg", " ")
+
+	out, _, err = dockerCmdWithError(args...)
+	if err == nil {
+		return fmt.Errorf(out)
+	}
+
+	// Be really picky
+	if strings.HasSuffix(stderr, "\n\n") {
+		return fmt.Errorf("Should not have a blank line at the end of 'docker rm'\n")
+	}
+
+	// Now make sure that each command will print a short-usage
+	// (not a full usage - meaning no opts section) if we
+	// are missing a required arg or pass in a bad arg
+
+	// These commands will never print a short-usage so don't test
+	noShortUsage := map[string]string{
+		"images":  "",
+		"login":   "",
+		"logout":  "",
+		"network": "",
+		"stats":   "",
+	}
+
+	if _, ok := noShortUsage[cmd]; !ok {
+		// For each command run it w/o any args. It will either return
+		// valid output or print a short-usage
+		var dCmd *exec.Cmd
+
+		// skipNoArgs are ones that we don't want to try w/o
+		// any args. Either because it'll hang the test or
+		// lead to incorrect test result (like false negative).
+		// Whatever the reason, skip trying to run w/o args and
+		// jump to trying with a bogus arg.
+		skipNoArgs := map[string]struct{}{
+			"daemon": {},
+			"events": {},
+			"load":   {},
+		}
+
+		ec := 0
+		if _, ok := skipNoArgs[cmd]; !ok {
+			args = strings.Split(cmd, " ")
+			dCmd = exec.Command(dockerBinary, args...)
+			out, stderr, ec, err = runCommandWithStdoutStderr(dCmd)
+		}
+
+		// If its ok w/o any args then try again with an arg
+		if ec == 0 {
+			args = strings.Split(cmd+" badArg", " ")
+			dCmd = exec.Command(dockerBinary, args...)
+			out, stderr, ec, err = runCommandWithStdoutStderr(dCmd)
+		}
+
+		if len(out) != 0 || len(stderr) == 0 || ec == 0 || err == nil {
+			return fmt.Errorf("Bad output from %q\nstdout:%q\nstderr:%q\nec:%d\nerr:%q\n", args, out, stderr, ec, err)
+		}
+		// Should have just short usage
+		if !strings.Contains(stderr, "\nUsage:\t") {
+			return fmt.Errorf("Missing short usage on %q\n", args)
+		}
+		// But shouldn't have full usage
+		if strings.Contains(stderr, "--help=false") {
+			return fmt.Errorf("Should not have full usage on %q\n", args)
+		}
+		if strings.HasSuffix(stderr, "\n\n") {
+			return fmt.Errorf("Should not have a blank line on %q\n", args)
+		}
+	}
+
+	return nil
+}