docs: added support for CLI yaml file generation

Signed-off-by: French Ben <frenchben@docker.com>
Signed-off-by: Tibor Vass <tibor@docker.com>

Makefile

@@ -141,6 +141,9 @@ run: build ## run the docker daemon in a container
 shell: build ## start a shell inside the build env
 	$(DOCKER_RUN_DOCKER) bash
 
+yaml-docs-gen: build ## generate documentation YAML files consumed by docs repo
+	$(DOCKER_RUN_DOCKER) sh -c 'hack/make.sh yaml-docs-generator && ( cd bundles/latest/yaml-docs-generator; mkdir docs; ./yaml-docs-generator --target $$(pwd)/docs )'
+
 test: build ## run the unit, integration and docker-py tests
 	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary cross test-unit test-integration-cli test-docker-py
 

docs/reference/commandline/load.md

@@ -26,7 +26,7 @@ Options:
                        The tarball may be compressed with gzip, bzip, or xz
   -q, --quiet          Suppress the load output but still outputs the imported images
 ```
-## Descriptino
+## Description
 
 `docker load` loads a tarred repository from a file or the standard input stream.
 It restores both images and tags.

docs/reference/commandline/stack_ls.md

@@ -27,7 +27,7 @@ Options:
       --help   Print usage
 ```
 
-## Descriptino
+## Description
 
 Lists the stacks.
 

docs/yaml/Dockerfile

@@ -0,0 +1,4 @@
+FROM scratch
+COPY docs /docs
+# CMD cannot be nil so we set it to empty string
+CMD  [""]

docs/yaml/generate.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/docker/docker/cli/command"
+	"github.com/docker/docker/cli/command/commands"
+	"github.com/docker/docker/pkg/term"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+)
+
+const descriptionSourcePath = "docs/reference/commandline/"
+
+func generateCliYaml(opts *options) error {
+	stdin, stdout, stderr := term.StdStreams()
+	dockerCli := command.NewDockerCli(stdin, stdout, stderr)
+	cmd := &cobra.Command{Use: "docker"}
+	commands.AddCommands(cmd, dockerCli)
+	source := filepath.Join(opts.source, descriptionSourcePath)
+	if err := loadLongDescription(cmd, source); err != nil {
+		return err
+	}
+
+	cmd.DisableAutoGenTag = true
+	return GenYamlTree(cmd, opts.target)
+}
+
+func loadLongDescription(cmd *cobra.Command, path ...string) error {
+	for _, cmd := range cmd.Commands() {
+		if cmd.Name() == "" {
+			continue
+		}
+		fullpath := filepath.Join(path[0], strings.Join(append(path[1:], cmd.Name()), "_")+".md")
+
+		if cmd.HasSubCommands() {
+			loadLongDescription(cmd, path[0], cmd.Name())
+		}
+
+		if _, err := os.Stat(fullpath); err != nil {
+			log.Printf("WARN: %s does not exist, skipping\n", fullpath)
+			continue
+		}
+
+		content, err := ioutil.ReadFile(fullpath)
+		if err != nil {
+			return err
+		}
+		description, examples := parseMDContent(string(content))
+		cmd.Long = description
+		cmd.Example = examples
+	}
+	return nil
+}
+
+type options struct {
+	source string
+	target string
+}
+
+func parseArgs() (*options, error) {
+	opts := &options{}
+	cwd, _ := os.Getwd()
+	flags := pflag.NewFlagSet(os.Args[0], pflag.ContinueOnError)
+	flags.StringVar(&opts.source, "root", cwd, "Path to project root")
+	flags.StringVar(&opts.target, "target", "/tmp", "Target path for generated yaml files")
+	err := flags.Parse(os.Args[1:])
+	return opts, err
+}
+
+func main() {
+	opts, err := parseArgs()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+	}
+	fmt.Printf("Project root: %s\n", opts.source)
+	fmt.Printf("Generating yaml files into %s\n", opts.target)
+	if err := generateCliYaml(opts); err != nil {
+		fmt.Fprintf(os.Stderr, "Failed to generate yaml files: %s\n", err.Error())
+	}
+}

docs/yaml/yaml.go

@@ -0,0 +1,212 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"gopkg.in/yaml.v2"
+)
+
+type cmdOption struct {
+	Option       string
+	Shorthand    string `yaml:",omitempty"`
+	DefaultValue string `yaml:"default_value,omitempty"`
+	Description  string `yaml:",omitempty"`
+}
+
+type cmdDoc struct {
+	Name             string      `yaml:"command"`
+	SeeAlso          []string    `yaml:"parent,omitempty"`
+	Version          string      `yaml:"engine_version,omitempty"`
+	Aliases          string      `yaml:",omitempty"`
+	Short            string      `yaml:",omitempty"`
+	Long             string      `yaml:",omitempty"`
+	Usage            string      `yaml:",omitempty"`
+	Pname            string      `yaml:",omitempty"`
+	Plink            string      `yaml:",omitempty"`
+	Cname            []string    `yaml:",omitempty"`
+	Clink            []string    `yaml:",omitempty"`
+	Options          []cmdOption `yaml:",omitempty"`
+	InheritedOptions []cmdOption `yaml:"inherited_options,omitempty"`
+	Example          string      `yaml:"examples,omitempty"`
+}
+
+// GenYamlTree creates yaml structured ref files
+func GenYamlTree(cmd *cobra.Command, dir string) error {
+	identity := func(s string) string { return s }
+	emptyStr := func(s string) string { return "" }
+	return GenYamlTreeCustom(cmd, dir, emptyStr, identity)
+}
+
+// GenYamlTreeCustom creates yaml structured ref files
+func GenYamlTreeCustom(cmd *cobra.Command, dir string, filePrepender, linkHandler func(string) string) error {
+	for _, c := range cmd.Commands() {
+		if !c.IsAvailableCommand() || c.IsHelpCommand() {
+			continue
+		}
+		if err := GenYamlTreeCustom(c, dir, filePrepender, linkHandler); err != nil {
+			return err
+		}
+	}
+
+	basename := strings.Replace(cmd.CommandPath(), " ", "_", -1) + ".yaml"
+	filename := filepath.Join(dir, basename)
+	f, err := os.Create(filename)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	if _, err := io.WriteString(f, filePrepender(filename)); err != nil {
+		return err
+	}
+	if err := GenYamlCustom(cmd, f, linkHandler); err != nil {
+		return err
+	}
+	return nil
+}
+
+// GenYamlCustom creates custom yaml output
+func GenYamlCustom(cmd *cobra.Command, w io.Writer, linkHandler func(string) string) error {
+	cliDoc := cmdDoc{}
+	cliDoc.Name = cmd.CommandPath()
+
+	// Check experimental: ok := cmd.Tags["experimental"]
+
+	cliDoc.Aliases = strings.Join(cmd.Aliases, ", ")
+	cliDoc.Short = cmd.Short
+	cliDoc.Long = cmd.Long
+	if len(cliDoc.Long) == 0 {
+		cliDoc.Long = cliDoc.Short
+	}
+
+	if cmd.Runnable() {
+		cliDoc.Usage = cmd.UseLine()
+	}
+
+	if len(cmd.Example) > 0 {
+		cliDoc.Example = cmd.Example
+	}
+
+	flags := cmd.NonInheritedFlags()
+	if flags.HasFlags() {
+		cliDoc.Options = genFlagResult(flags)
+	}
+	flags = cmd.InheritedFlags()
+	if flags.HasFlags() {
+		cliDoc.InheritedOptions = genFlagResult(flags)
+	}
+
+	if hasSeeAlso(cmd) {
+		if cmd.HasParent() {
+			parent := cmd.Parent()
+			cliDoc.Pname = parent.CommandPath()
+			link := cliDoc.Pname + ".yaml"
+			cliDoc.Plink = strings.Replace(link, " ", "_", -1)
+			cmd.VisitParents(func(c *cobra.Command) {
+				if c.DisableAutoGenTag {
+					cmd.DisableAutoGenTag = c.DisableAutoGenTag
+				}
+			})
+		}
+
+		children := cmd.Commands()
+		sort.Sort(byName(children))
+
+		for _, child := range children {
+			if !child.IsAvailableCommand() || child.IsHelpCommand() {
+				continue
+			}
+			currentChild := cliDoc.Name + " " + child.Name()
+			cliDoc.Cname = append(cliDoc.Cname, cliDoc.Name+" "+child.Name())
+			link := currentChild + ".yaml"
+			cliDoc.Clink = append(cliDoc.Clink, strings.Replace(link, " ", "_", -1))
+		}
+	}
+
+	final, err := yaml.Marshal(&cliDoc)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+	if _, err := fmt.Fprintln(w, string(final)); err != nil {
+		return err
+	}
+	return nil
+}
+
+func genFlagResult(flags *pflag.FlagSet) []cmdOption {
+	var result []cmdOption
+
+	flags.VisitAll(func(flag *pflag.Flag) {
+		// Todo, when we mark a shorthand is deprecated, but specify an empty message.
+		// The flag.ShorthandDeprecated is empty as the shorthand is deprecated.
+		// Using len(flag.ShorthandDeprecated) > 0 can't handle this, others are ok.
+		if !(len(flag.ShorthandDeprecated) > 0) && len(flag.Shorthand) > 0 {
+			opt := cmdOption{
+				Option:       flag.Name,
+				Shorthand:    flag.Shorthand,
+				DefaultValue: flag.DefValue,
+				Description:  forceMultiLine(flag.Usage),
+			}
+			result = append(result, opt)
+		} else {
+			opt := cmdOption{
+				Option:       flag.Name,
+				DefaultValue: forceMultiLine(flag.DefValue),
+				Description:  forceMultiLine(flag.Usage),
+			}
+			result = append(result, opt)
+		}
+	})
+
+	return result
+}
+
+// Temporary workaround for yaml lib generating incorrect yaml with long strings
+// that do not contain \n.
+func forceMultiLine(s string) string {
+	if len(s) > 60 && !strings.Contains(s, "\n") {
+		s = s + "\n"
+	}
+	return s
+}
+
+// Small duplication for cobra utils
+func hasSeeAlso(cmd *cobra.Command) bool {
+	if cmd.HasParent() {
+		return true
+	}
+	for _, c := range cmd.Commands() {
+		if !c.IsAvailableCommand() || c.IsHelpCommand() {
+			continue
+		}
+		return true
+	}
+	return false
+}
+
+func parseMDContent(mdString string) (description string, examples string) {
+	parsedContent := strings.Split(mdString, "\n## ")
+	for _, s := range parsedContent {
+		if strings.Index(s, "Description") == 0 {
+			description = strings.Trim(s, "Description\n")
+		}
+		if strings.Index(s, "Examples") == 0 {
+			examples = strings.Trim(s, "Examples\n")
+		}
+	}
+	return
+}
+
+type byName []*cobra.Command
+
+func (s byName) Len() int           { return len(s) }
+func (s byName) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+func (s byName) Less(i, j int) bool { return s[i].Name() < s[j].Name() }

hack/make/yaml-docs-generator

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -e
+
+[ -z "$KEEPDEST" ] && \
+	rm -rf "$DEST"
+
+(
+	source "${MAKEDIR}/.binary-setup"
+	export BINARY_SHORT_NAME="yaml-docs-generator"
+	export GO_PACKAGE='github.com/docker/docker/docs/yaml'
+	source "${MAKEDIR}/.binary"
+)

hooks/post_build

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+if [ -n "${BUILD_DOCS}" ]; then
+	set -e
+	DOCS_IMAGE=${DOCS_IMAGE:-${IMAGE_NAME}-docs}
+	docker run \
+		--entrypoint '' \
+		--privileged \
+		-e DOCKER_GITCOMMIT=$(git rev-parse --short HEAD) \
+		-v $(pwd)/docs/yaml/docs:/docs \
+		"${IMAGE_NAME}" \
+		sh -c 'hack/make.sh yaml-docs-generator && bundles/latest/yaml-docs-generator/yaml-docs-generator --target /docs'
+
+	(
+		cd docs/yaml
+		docker build -t ${DOCS_IMAGE} .
+		docker push ${DOCS_IMAGE}
+	)
+fi