Improve the time taken by DockerSuite.TestHelpTextVerify. This test runs docker <command> --help on all commands supported and also check the output

when it is passed with bad arguments and no arguments. This patch would divide the total number of commands into five sets and runs them in parallel.
Test time is improved from around 9 seconds to around 3 seconds

Signed-off-by: Sainath Grandhi <sainath.grandhi@intel.com>

integration-cli/docker_cli_help_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"fmt"
 	"os/exec"
 	"runtime"
 	"strings"
@@ -13,6 +14,7 @@ import (
 
 func (s *DockerSuite) TestHelpTextVerify(c *check.C) {
 	testRequires(c, DaemonIsLinux)
+
 	// Make sure main help text fits within 80 chars and that
 	// on non-windows system we use ~ when possible (to shorten things).
 	// Test for HOME set to its default value and set to "/" on linux
@@ -40,6 +42,7 @@ func (s *DockerSuite) TestHelpTextVerify(c *check.C) {
 	}
 
 	for _, home := range homes {
+
 		// Dup baseEnvs and add our new HOME value
 		newEnvs := make([]string, len(baseEnvs)+1)
 		copy(newEnvs, baseEnvs)
@@ -117,114 +120,22 @@ func (s *DockerSuite) TestHelpTextVerify(c *check.C) {
 		cmdsToTest = append(cmdsToTest, "volume ls")
 		cmdsToTest = append(cmdsToTest, "volume rm")
 
-		for _, cmd := range cmdsToTest {
-			var stderr string
-
-			args := strings.Split(cmd+" --help", " ")
-
-			// Check the full usage text
-			helpCmd := exec.Command(dockerBinary, args...)
-			helpCmd.Env = newEnvs
-			out, stderr, _, err = runCommandWithStdoutStderr(helpCmd)
-			c.Assert(len(stderr), checker.Equals, 0, check.Commentf("Error on %q help. non-empty stderr:%q", cmd, stderr))
-			c.Assert(out, checker.Not(checker.HasSuffix), "\n\n", check.Commentf("Should not have blank line on %q\n", cmd))
-			c.Assert(out, checker.Contains, "--help", check.Commentf("All commands should mention '--help'. Command '%v' did not.\n", cmd))
-
-			c.Assert(err, checker.IsNil, check.Commentf(out))
-
-			// Check each line for lots of stuff
-			lines := strings.Split(out, "\n")
-			for _, line := range lines {
-				c.Assert(len(line), checker.LessOrEqualThan, 107, check.Commentf("Help for %q is too long:\n%s", cmd, line))
-
-				if scanForHome && strings.Contains(line, `"`+home) {
-					c.Fatalf("Help for %q should use ~ instead of %q on:\n%s",
-						cmd, home, line)
-				}
-				i := strings.Index(line, "~")
-				if i >= 0 && i != len(line)-1 && line[i+1] != '/' {
-					c.Fatalf("Help for %q should not have used ~:\n%s", cmd, line)
-				}
-
-				// If a line starts with 4 spaces then assume someone
-				// added a multi-line description for an option and we need
-				// to flag it
-				c.Assert(line, checker.Not(checker.HasPrefix), "    ", check.Commentf("Help for %q should not have a multi-line option", cmd))
-
-				// Options should NOT end with a period
-				if strings.HasPrefix(line, "  -") && strings.HasSuffix(line, ".") {
-					c.Fatalf("Help for %q should not end with a period: %s", cmd, line)
-				}
-
-				// Options should NOT end with a space
-				c.Assert(line, checker.Not(checker.HasSuffix), " ", check.Commentf("Help for %q should not end with a space", cmd))
-
-			}
-
-			// For each command make sure we generate an error
-			// if we give a bad arg
-			args = strings.Split(cmd+" --badArg", " ")
-
-			out, _, err = dockerCmdWithError(args...)
-			c.Assert(err, checker.NotNil, check.Commentf(out))
-			// Be really picky
-			c.Assert(stderr, checker.Not(checker.HasSuffix), "\n\n", check.Commentf("Should not have a blank line at the end of 'docker rm'\n"))
-
-			// Now make sure that each command will print a short-usage
-			// (not a full usage - meaning no opts section) if we
-			// are missing a required arg or pass in a bad arg
-
-			// These commands will never print a short-usage so don't test
-			noShortUsage := map[string]string{
-				"images":  "",
-				"login":   "",
-				"logout":  "",
-				"network": "",
-				"stats":   "",
-			}
+		// Divide the list of commands into go routines and  run the func testcommand on the commands in parallel
+		// to save runtime of test
 
-			if _, ok := noShortUsage[cmd]; !ok {
-				// For each command run it w/o any args. It will either return
-				// valid output or print a short-usage
-				var dCmd *exec.Cmd
-				var stdout, stderr string
-				var args []string
-
-				// skipNoArgs are ones that we don't want to try w/o
-				// any args. Either because it'll hang the test or
-				// lead to incorrect test result (like false negative).
-				// Whatever the reason, skip trying to run w/o args and
-				// jump to trying with a bogus arg.
-				skipNoArgs := map[string]struct{}{
-					"daemon": {},
-					"events": {},
-					"load":   {},
-				}
+		errChan := make(chan error)
 
-				ec := 0
-				if _, ok := skipNoArgs[cmd]; !ok {
-					args = strings.Split(cmd, " ")
-					dCmd = exec.Command(dockerBinary, args...)
-					stdout, stderr, ec, err = runCommandWithStdoutStderr(dCmd)
-				}
-
-				// If its ok w/o any args then try again with an arg
-				if ec == 0 {
-					args = strings.Split(cmd+" badArg", " ")
-					dCmd = exec.Command(dockerBinary, args...)
-					stdout, stderr, ec, err = runCommandWithStdoutStderr(dCmd)
-				}
+		for index := 0; index < len(cmdsToTest); index++ {
+			go func(index int) {
+				errChan <- testCommand(cmdsToTest[index], newEnvs, scanForHome, home)
+			}(index)
+		}
 
-				if len(stdout) != 0 || len(stderr) == 0 || ec == 0 || err == nil {
-					c.Fatalf("Bad output from %q\nstdout:%q\nstderr:%q\nec:%d\nerr:%q", args, stdout, stderr, ec, err)
-				}
-				// Should have just short usage
-				c.Assert(stderr, checker.Contains, "\nUsage:\t", check.Commentf("Missing short usage on %q\n", args))
-				// But shouldn't have full usage
-				c.Assert(stderr, checker.Not(checker.Contains), "--help=false", check.Commentf("Should not have full usage on %q\n", args))
-				c.Assert(stderr, checker.Not(checker.HasSuffix), "\n\n", check.Commentf("Should not have a blank line on %q\n", args))
+		for index := 0; index < len(cmdsToTest); index++ {
+			err := <-errChan
+			if err != nil {
+				c.Fatal(err)
 			}
-
 		}
 
 		// Number of commands for standard release and experimental release
@@ -296,3 +207,136 @@ func (s *DockerSuite) TestHelpExitCodesHelpOutput(c *check.C) {
 	c.Assert(stdout, checker.Equals, "")
 	c.Assert(stderr, checker.Equals, "docker: 'BadCmd' is not a docker command.\nSee 'docker --help'.\n", check.Commentf("Unexcepted output for 'docker badCmd'\n"))
 }
+
+func testCommand(cmd string, newEnvs []string, scanForHome bool, home string) error {
+
+	args := strings.Split(cmd+" --help", " ")
+
+	// Check the full usage text
+	helpCmd := exec.Command(dockerBinary, args...)
+	helpCmd.Env = newEnvs
+	out, stderr, _, err := runCommandWithStdoutStderr(helpCmd)
+	if len(stderr) != 0 {
+		return fmt.Errorf("Error on %q help. non-empty stderr:%q\n", cmd, stderr)
+	}
+	if strings.HasSuffix(out, "\n\n") {
+		return fmt.Errorf("Should not have blank line on %q\n", cmd)
+	}
+	if !strings.Contains(out, "--help") {
+		return fmt.Errorf("All commands should mention '--help'. Command '%v' did not.\n", cmd)
+	}
+
+	if err != nil {
+		return fmt.Errorf(out)
+	}
+
+	// Check each line for lots of stuff
+	lines := strings.Split(out, "\n")
+	for _, line := range lines {
+		if len(line) > 107 {
+			return fmt.Errorf("Help for %q is too long:\n%s\n", cmd, line)
+		}
+
+		if scanForHome && strings.Contains(line, `"`+home) {
+			return fmt.Errorf("Help for %q should use ~ instead of %q on:\n%s\n",
+				cmd, home, line)
+		}
+		i := strings.Index(line, "~")
+		if i >= 0 && i != len(line)-1 && line[i+1] != '/' {
+			return fmt.Errorf("Help for %q should not have used ~:\n%s", cmd, line)
+		}
+
+		// If a line starts with 4 spaces then assume someone
+		// added a multi-line description for an option and we need
+		// to flag it
+		if strings.HasPrefix(line, "    ") {
+			return fmt.Errorf("Help for %q should not have a multi-line option", cmd)
+		}
+
+		// Options should NOT end with a period
+		if strings.HasPrefix(line, "  -") && strings.HasSuffix(line, ".") {
+			return fmt.Errorf("Help for %q should not end with a period: %s", cmd, line)
+		}
+
+		// Options should NOT end with a space
+		if strings.HasSuffix(line, " ") {
+			return fmt.Errorf("Help for %q should not end with a space", cmd)
+		}
+
+	}
+
+	// For each command make sure we generate an error
+	// if we give a bad arg
+	args = strings.Split(cmd+" --badArg", " ")
+
+	out, _, err = dockerCmdWithError(args...)
+	if err == nil {
+		return fmt.Errorf(out)
+	}
+
+	// Be really picky
+	if strings.HasSuffix(stderr, "\n\n") {
+		return fmt.Errorf("Should not have a blank line at the end of 'docker rm'\n")
+	}
+
+	// Now make sure that each command will print a short-usage
+	// (not a full usage - meaning no opts section) if we
+	// are missing a required arg or pass in a bad arg
+
+	// These commands will never print a short-usage so don't test
+	noShortUsage := map[string]string{
+		"images":  "",
+		"login":   "",
+		"logout":  "",
+		"network": "",
+		"stats":   "",
+	}
+
+	if _, ok := noShortUsage[cmd]; !ok {
+		// For each command run it w/o any args. It will either return
+		// valid output or print a short-usage
+		var dCmd *exec.Cmd
+
+		// skipNoArgs are ones that we don't want to try w/o
+		// any args. Either because it'll hang the test or
+		// lead to incorrect test result (like false negative).
+		// Whatever the reason, skip trying to run w/o args and
+		// jump to trying with a bogus arg.
+		skipNoArgs := map[string]struct{}{
+			"daemon": {},
+			"events": {},
+			"load":   {},
+		}
+
+		ec := 0
+		if _, ok := skipNoArgs[cmd]; !ok {
+			args = strings.Split(cmd, " ")
+			dCmd = exec.Command(dockerBinary, args...)
+			out, stderr, ec, err = runCommandWithStdoutStderr(dCmd)
+		}
+
+		// If its ok w/o any args then try again with an arg
+		if ec == 0 {
+			args = strings.Split(cmd+" badArg", " ")
+			dCmd = exec.Command(dockerBinary, args...)
+			out, stderr, ec, err = runCommandWithStdoutStderr(dCmd)
+		}
+
+		if len(out) != 0 || len(stderr) == 0 || ec == 0 || err == nil {
+			return fmt.Errorf("Bad output from %q\nstdout:%q\nstderr:%q\nec:%d\nerr:%q\n", args, out, stderr, ec, err)
+		}
+		// Should have just short usage
+		if !strings.Contains(stderr, "\nUsage:\t") {
+			return fmt.Errorf("Missing short usage on %q\n", args)
+		}
+		// But shouldn't have full usage
+		if strings.Contains(stderr, "--help=false") {
+			return fmt.Errorf("Should not have full usage on %q\n", args)
+		}
+		if strings.HasSuffix(stderr, "\n\n") {
+			return fmt.Errorf("Should not have a blank line on %q\n", args)
+		}
+	}
+
+	return nil
+}