Merge pull request #44049 from thaJeztah/22.06_backport_validate_yaml

[22.06 backport] validate: add additional validation on YAML files

Dockerfile

@@ -349,7 +349,8 @@ RUN update-alternatives --set iptables  /usr/sbin/iptables-legacy  || true \
  && update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy || true \
  && update-alternatives --set arptables /usr/sbin/arptables-legacy || true
 
-RUN pip3 install yamllint==1.26.1
+ARG YAMLLINT_VERSION=1.27.1
+RUN pip3 install yamllint==${YAMLLINT_VERSION}
 
 COPY --from=dockercli     /build/ /usr/local/cli
 COPY --from=frozen-images /build/ /docker-frozen-images

hack/validate/.swagger-yamllint

@@ -1,4 +0,0 @@
-extends: default
-rules:
-  document-start: disable
-  line-length: disable

hack/validate/all

@@ -2,7 +2,7 @@
 #
 # Run all validation
 
-export SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 . ${SCRIPTDIR}/default
 . ${SCRIPTDIR}/vendor

hack/validate/dco

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-export SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 source "${SCRIPTDIR}/.validate"
 
 adds=$(validate_diff --numstat | awk '{ s += $1 } END { print s }')

hack/validate/default

@@ -2,12 +2,13 @@
 #
 # Run default validation, exclude vendor because it's slow
 
-export SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 # Skip DCO check here, as it's already checked in a separate stage in the Jenkinsfile
 #. "${SCRIPTDIR}"/dco
 . "${SCRIPTDIR}"/default-seccomp
 . "${SCRIPTDIR}"/pkg-imports
+. "${SCRIPTDIR}"/yamllint
 . "${SCRIPTDIR}"/swagger
 . "${SCRIPTDIR}"/swagger-gen
 . "${SCRIPTDIR}"/toml

hack/validate/default-seccomp

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-export SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 source "${SCRIPTDIR}/.validate"
 
 IFS=$'\n'

hack/validate/deprecate-integration-cli

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 # Check that no new tests are being added to integration-cli
 
-export SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 source "${SCRIPTDIR}/.validate"
 
 new_tests=$(

hack/validate/pkg-imports

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -e
 
-export SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 source "${SCRIPTDIR}/.validate"
 
 IFS=$'\n'

hack/validate/swagger

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 set -e
-export SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 source "${SCRIPTDIR}/.validate"
 
 IFS=$'\n'
@@ -8,7 +8,7 @@ files=($(validate_diff --diff-filter=ACMR --name-only -- 'api/swagger.yaml' || t
 unset IFS
 
 if [ -n "${TEST_FORCE_VALIDATE:-}" ] || [ ${#files[@]} -gt 0 ]; then
-	LANG=C.UTF-8 yamllint -c "${SCRIPTDIR}"/.swagger-yamllint api/swagger.yaml
+	yamllint -f parsable -c "${SCRIPTDIR}"/yamllint.yaml api/swagger.yaml
 	if out=$(swagger validate api/swagger.yaml); then
 		echo "Congratulations!  ${out}"
 	else

hack/validate/swagger-gen

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-export SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 source "${SCRIPTDIR}/.validate"
 
 IFS=$'\n'

hack/validate/toml

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-export SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 source "${SCRIPTDIR}/.validate"
 
 IFS=$'\n'

hack/validate/vendor

@@ -1,20 +1,20 @@
 #!/usr/bin/env bash
 
-SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 source "${SCRIPTDIR}/.validate"
 
-validate_vendor_diff(){
+validate_vendor_diff() {
 	IFS=$'\n'
-	check_files=( 'vendor.sum' 'vendor.mod' 'vendor/' )
+	check_files=('vendor.sum' 'vendor.mod' 'vendor/')
 	# shellcheck disable=SC2207
-	changed_files=( $(validate_diff --diff-filter=ACMR --name-only -- "${check_files[@]}" || true) )
+	changed_files=($(validate_diff --diff-filter=ACMR --name-only -- "${check_files[@]}" || true))
 	unset IFS
 
 	if [ -n "${TEST_FORCE_VALIDATE:-}" ] || [ "${#changed_files[@]}" -gt 0 ]; then
 		# recreate vendor/
 		./hack/vendor.sh
 		# check if any files have changed
-		diffs="$(git status --porcelain -- "${check_files[@]}" 2>/dev/null)"
+		diffs="$(git status --porcelain -- "${check_files[@]}" 2> /dev/null)"
 		mfiles="$(echo "$diffs" | awk '/^ M / {print $2}')"
 		if [ "$diffs" ]; then
 			{
@@ -24,7 +24,7 @@ validate_vendor_diff(){
 				echo
 				echo 'Please vendor your package with hack/vendor.sh.'
 				echo
-				if [ -n "$mfiles" ] ; then
+				if [ -n "$mfiles" ]; then
 					git diff -- "$mfiles"
 				fi
 			} >&2
@@ -41,11 +41,11 @@ validate_vendor_diff(){
 # 2. make sure all the packages contain license information (just warning, because it can cause false-positive)
 validate_vendor_used() {
 	for f in $(mawk '$1 = "#" { print $2 }' 'vendor/modules.txt'); do
-	if [ -d "vendor/$f" ]; then
-		if ! echo "vendor/$f"/* | grep -qiEc '/(LICENSE|COPYING)'; then
-		echo "WARNING: could not find copyright information for $f"
+		if [ -d "vendor/$f" ]; then
+			if ! echo "vendor/$f"/* | grep -qiEc '/(LICENSE|COPYING)'; then
+				echo "WARNING: could not find copyright information for $f"
+			fi
 		fi
-	fi
 	done
 }
 

hack/validate/yamllint

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -e
+SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${SCRIPTDIR}/.validate"
+
+if [ -n "${TEST_FORCE_VALIDATE:-}" ]; then
+	files=(docs/api/*.yaml)
+else
+	IFS=$'\n'
+	files=($(validate_diff --diff-filter=ACMR --name-only -- docs/*.yaml || true))
+	unset IFS
+fi
+
+# validate the yamllint configuration file before anything else
+if out=$(yamllint -f parsable -d "{extends: default, rules: {document-start: disable}}" "${SCRIPTDIR}"/yamllint.yaml); then
+	echo "Congratulations! yamllint config file formatted correctly"
+else
+	echo "${out}" >&2
+	false
+fi
+
+# Then validate GitHub actions workflows, and conditionally lint the swagger
+# files in the docs directory, as these are large files and take some time.
+if out=$(yamllint -f parsable -c "${SCRIPTDIR}"/yamllint.yaml .github/workflows/*.yml "${files[@]}"); then
+	echo "Congratulations! YAML files are formatted correctly"
+else
+	echo "${out}" >&2
+	false
+fi

hack/validate/yamllint.yaml

@@ -0,0 +1,10 @@
+extends: default
+locale: C.UTF-8
+rules:
+  document-start: disable
+  line-length: disable
+  # ignore "warning  truthy value should be one of [false, true]" on GitHub
+  # actions workflows, which use "on:" to specify when to run.
+  truthy:
+    ignore: |
+      .github/workflows/