Merge pull request #5022 from tianon/make-validate

Add "make validate" for both local and CI gofmt and DCO verification

.travis.yml

@@ -10,21 +10,11 @@ install: true
 
 before_script:
   - env | sort
-  - sudo apt-get update -qq
-  - sudo apt-get install -qq python-yaml
-  - git remote add upstream git://github.com/dotcloud/docker.git
-  - upstream=master;
-    if [ "$TRAVIS_PULL_REQUEST" != false ]; then
-      upstream=$TRAVIS_BRANCH;
-    fi;
-    git fetch --append --no-tags upstream refs/heads/$upstream:refs/remotes/upstream/$upstream
-# sometimes we have upstream master already as origin/master (PRs), but other times we don't, so let's just make sure we have a completely unambiguous way to specify "upstream master" from here out
-# but if it's a PR against non-master, we need that upstream branch instead :)
   - sudo pip install -r docs/requirements.txt
 
 script:
-  - hack/travis/dco.py
-  - hack/travis/gofmt.py
+  - hack/make.sh validate-dco
+  - hack/make.sh validate-gofmt
   - make -sC docs SPHINXOPTS=-qW docs man
 
 # vim:set sw=2 ts=2:

hack/make.sh

@@ -40,13 +40,19 @@ echo
 
 # List of bundles to create when no argument is passed
 DEFAULT_BUNDLES=(
+	validate-dco
+	validate-gofmt
+	
 	binary
+	
 	test
 	test-integration
 	test-integration-cli
+	
 	dynbinary
 	dyntest
 	dyntest-integration
+	
 	cover
 	cross
 	tgz

hack/make/.validate

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+if [ -z "$VALIDATE_UPSTREAM" ]; then
+	# this is kind of an expensive check, so let's not do this twice if we
+	# are running more than one validate bundlescript
+	
+	VALIDATE_REPO='https://github.com/dotcloud/docker.git'
+	VALIDATE_BRANCH='master'
+	
+	if [ "$TRAVIS" = 'true' -a "$TRAVIS_PULL_REQUEST" != 'false' ]; then
+		VALIDATE_REPO="https://github.com/${TRAVIS_REPO_SLUG}.git"
+		VALIDATE_BRANCH="${TRAVIS_BRANCH}"
+	fi
+	
+	VALIDATE_HEAD="$(git rev-parse --verify HEAD)"
+	
+	git fetch -q "$VALIDATE_REPO" "refs/heads/$VALIDATE_BRANCH"
+	VALIDATE_UPSTREAM="$(git rev-parse --verify FETCH_HEAD)"
+	
+	VALIDATE_COMMIT_LOG="$VALIDATE_UPSTREAM..$VALIDATE_HEAD"
+	VALIDATE_COMMIT_DIFF="$VALIDATE_UPSTREAM...$VALIDATE_HEAD"
+	
+	validate_diff() {
+		if [ "$VALIDATE_UPSTREAM" != "$VALIDATE_HEAD" ]; then
+			git diff "$VALIDATE_COMMIT_DIFF" "$@"
+		fi
+	}
+	validate_log() {
+		if [ "$VALIDATE_UPSTREAM" != "$VALIDATE_HEAD" ]; then
+			git log "$VALIDATE_COMMIT_LOG" "$@"
+		fi
+	}
+fi

hack/make/validate-dco

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+source "$(dirname "$BASH_SOURCE")/.validate"
+
+adds=$(validate_diff --numstat | awk '{ s += $1 } END { print s }')
+dels=$(validate_diff --numstat | awk '{ s += $2 } END { print s }')
+notDocs="$(validate_diff --numstat | awk '$3 !~ /^docs\// { print $3 }')"
+
+: ${adds:=0}
+: ${dels:=0}
+
+if [ $adds -eq 0 -a $dels -eq 0 ]; then
+	echo '0 adds, 0 deletions; nothing to validate! :)'
+elif [ -z "$notDocs" -a $adds -le 1 -a $dels -le 1 ]; then
+	echo 'Congratulations!  DCO small-patch-exception material!'
+else
+	dcoPrefix='Docker-DCO-1.1-Signed-off-by:'
+	dcoRegex="^$dcoPrefix ([^<]+) <([^<>@]+@[^<>]+)> \\(github: (\S+)\\)$"
+	commits=( $(validate_log --format='format:%H%n') )
+	badCommits=()
+	for commit in "${commits[@]}"; do
+		if [ -z "$(git log -1 --format='format:' --name-status "$commit")" ]; then
+			# no content (ie, Merge commit, etc)
+			continue
+		fi
+		if ! git log -1 --format='format:%B' "$commit" | grep -qE "$dcoRegex"; then
+			badCommits+=( "$commit" )
+		fi
+	done
+	if [ ${#badCommits[@]} -eq 0 ]; then
+		echo "Congratulations!  All commits are properly signed with the DCO!"
+	else
+		{
+			echo "These commits do not have a proper '$dcoPrefix' marker:"
+			for commit in "${badCommits[@]}"; do
+				echo " - $commit"
+			done
+			echo
+			echo 'Please amend each commit to include a properly formatted DCO marker.'
+			echo
+			echo 'Visit the following URL for information about the Docker DCO:'
+			echo ' https://github.com/dotcloud/docker/blob/master/CONTRIBUTING.md#sign-your-work'
+			echo
+		} >&2
+		false
+	fi
+fi

hack/make/validate-gofmt

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+source "$(dirname "$BASH_SOURCE")/.validate"
+
+IFS=$'\n'
+files=( $(validate_diff --diff-filter=ACMR --name-only -- '*.go' | grep -v '^vendor/' || true) )
+unset IFS
+
+badFiles=()
+for f in "${files[@]}"; do
+	# we use "git show" here to validate that what's committed is formatted
+	if [ "$(git show "$VALIDATE_HEAD:$f" | gofmt -s -l)" ]; then
+		badFiles+=( "$f" )
+	fi
+done
+
+if [ ${#badFiles[@]} -eq 0 ]; then
+	echo 'Congratulations!  All Go source files are properly formatted.'
+else
+	{
+		echo "These files are not properly gofmt'd:"
+		for f in "${badFiles[@]}"; do
+			echo " - $f"
+		done
+		echo
+		echo 'Please reformat the above files using "gofmt -s -w" and commit the result.'
+		echo
+	} >&2
+	false
+fi

hack/travis/dco.py

@@ -1,54 +0,0 @@
-#!/usr/bin/env python
-import re
-import subprocess
-import yaml
-
-from env import commit_range
-
-commit_format = '-%n hash: "%h"%n author: %aN <%aE>%n message: |%n%w(0,2,2).%B'
-
-gitlog = subprocess.check_output([
-	'git', 'log', '--reverse',
-	'--format=format:'+commit_format,
-	'..'.join(commit_range), '--',
-])
-
-commits = yaml.load(gitlog)
-if not commits:
-	exit(0) # what?  how can we have no commits?
-
-DCO = 'Docker-DCO-1.1-Signed-off-by:'
-
-p = re.compile(r'^{0} ([^<]+) <([^<>@]+@[^<>]+)> \(github: (\S+)\)$'.format(re.escape(DCO)), re.MULTILINE|re.UNICODE)
-
-failed_commits = 0
-
-for commit in commits:
-	commit['message'] = commit['message'][1:]
-	# trim off our '.' that exists just to prevent fun YAML parsing issues
-	# see https://github.com/dotcloud/docker/pull/3836#issuecomment-33723094
-	# and https://travis-ci.org/dotcloud/docker/builds/17926783
-	
-	commit['stat'] = subprocess.check_output([
-		'git', 'log', '--format=format:', '--max-count=1',
-		'--name-status', commit['hash'], '--',
-	])
-	if commit['stat'] == '':
-		print 'Commit {0} has no actual changed content, skipping.'.format(commit['hash'])
-		continue
-	
-	m = p.search(commit['message'])
-	if not m:
-		print 'Commit {1} does not have a properly formatted "{0}" marker.'.format(DCO, commit['hash'])
-		failed_commits += 1
-		continue # print ALL the commits that don't have a proper DCO
-	
-	(name, email, github) = m.groups()
-	
-	# TODO verify that "github" is the person who actually made this commit via the GitHub API
-
-if failed_commits > 0:
-	exit(failed_commits)
-
-print 'All commits have a valid "{0}" marker.'.format(DCO)
-exit(0)

hack/travis/env.py

@@ -1,21 +0,0 @@
-import os
-import subprocess
-
-if 'TRAVIS' not in os.environ:
-	print 'TRAVIS is not defined; this should run in TRAVIS. Sorry.'
-	exit(127)
-
-if os.environ['TRAVIS_PULL_REQUEST'] != 'false':
-	commit_range = ['upstream/' + os.environ['TRAVIS_BRANCH'], 'FETCH_HEAD']
-else:
-	try:
-		subprocess.check_call([
-			'git', 'log', '-1', '--format=format:',
-			os.environ['TRAVIS_COMMIT_RANGE'], '--',
-		])
-		commit_range = os.environ['TRAVIS_COMMIT_RANGE'].split('...')
-		if len(commit_range) == 1: # if it didn't split, it must have been separated by '..' instead
-			commit_range = commit_range[0].split('..')
-	except subprocess.CalledProcessError:
-		print 'TRAVIS_COMMIT_RANGE is invalid. This seems to be a force push. We will just assume it must be against upstream master and compare all commits in between.'
-		commit_range = ['upstream/master', 'HEAD']

hack/travis/gofmt.py

@@ -1,31 +0,0 @@
-#!/usr/bin/env python
-import subprocess
-
-from env import commit_range
-
-files = subprocess.check_output([
-	'git', 'diff', '--diff-filter=ACMR',
-	'--name-only', '...'.join(commit_range), '--',
-])
-
-exit_status = 0
-
-for filename in files.split('\n'):
-	if filename.startswith('vendor/'):
-		continue # we can't be changing our upstream vendors for gofmt, so don't even check them
-	
-	if filename.endswith('.go'):
-		try:
-			out = subprocess.check_output(['gofmt', '-s', '-l', filename])
-			if out != '':
-				print out,
-				exit_status = 1
-		except subprocess.CalledProcessError:
-			exit_status = 1
-
-if exit_status != 0:
-	print 'Reformat the files listed above with "gofmt -s -w" and try again.'
-	exit(exit_status)
-
-print 'All files pass gofmt.'
-exit(0)