Merge pull request #19735 from jfrazelle/add-repo-scripts

add scripts to clean experimental rpms and debs

hack/make/clean-apt-repo

@@ -0,0 +1,43 @@
+#!/bin/bash
+set -e
+
+# This script cleans the experimental pool for the apt repo.
+# This is useful when there are a lot of old experimental debs and you only want to keep the most recent.
+#
+
+: ${DOCKER_RELEASE_DIR:=$DEST}
+APTDIR=$DOCKER_RELEASE_DIR/apt/repo/pool/experimental
+: ${DOCKER_ARCHIVE_DIR:=$DEST/archive}
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+latest_versions=$(dpkg-scanpackages "$APTDIR" /dev/null 2>/dev/null | awk -F ': ' '$1 == "Filename" { print $2 }')
+
+# get the latest version
+latest_docker_engine_file=$(echo "$latest_versions" | grep docker-engine)
+latest_docker_engine_version=$(basename ${latest_docker_engine_file%~*})
+
+echo "latest docker-engine version: $latest_docker_engine_version"
+
+# remove all the files that are not that version in experimental
+pool_dir=$(dirname "$latest_docker_engine_file")
+old_pkgs=( $(ls "$pool_dir" | grep -v "^${latest_docker_engine_version}" | grep "${latest_docker_engine_version%%~git*}") )
+
+echo "${old_pkgs[@]}"
+
+mkdir -p "$DOCKER_ARCHIVE_DIR"
+for old_pkg in "${old_pkgs[@]}"; do
+	echo "moving ${pool_dir}/${old_pkg} to $DOCKER_ARCHIVE_DIR"
+	mv "${pool_dir}/${old_pkg}" "$DOCKER_ARCHIVE_DIR"
+done
+
+echo
+echo "$pool_dir now has contents:"
+ls "$pool_dir"
+
+# now regenerate release files for experimental
+export COMPONENT=experimental
+source "${DIR}/update-apt-repo"
+
+echo "You will now want to: "
+echo "   - re-sign the repo with hack/make/sign-repo"
+echo "   - re-generate index files with hack/make/generate-index-listing"

hack/make/clean-yum-repo

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -e
+
+# This script cleans the experimental pool for the yum repo.
+# This is useful when there are a lot of old experimental rpms and you only want to keep the most recent.
+#
+
+: ${DOCKER_RELEASE_DIR:=$DEST}
+YUMDIR=$DOCKER_RELEASE_DIR/yum/repo/experimental
+
+suites=( $(find "$YUMDIR" -mindepth 1 -maxdepth 1 -type d) )
+
+for suite in "${suites[@]}"; do
+	echo "cleanup in: $suite"
+	( set -x; repomanage -k2 --old "$suite" | xargs rm -f )
+done
+
+echo "You will now want to: "
+echo "   - re-sign the repo with hack/make/sign-repo"
+echo "   - re-generate index files with hack/make/generate-index-listing"

hack/make/update-apt-repo

@@ -0,0 +1,70 @@
+#!/bin/bash
+set -e
+
+# This script updates the apt repo in $DOCKER_RELEASE_DIR/apt/repo.
+# This script is a "fix all" for any sort of problems that might have occured with
+# the Release or Package files in the repo.
+# It should only be used in the rare case of extreme emergencies to regenerate
+# Release and Package files for the apt repo.
+#
+# NOTE: Always be sure to re-sign the repo with hack/make/sign-repos after running
+# this script.
+
+: ${DOCKER_RELEASE_DIR:=$DEST}
+APTDIR=$DOCKER_RELEASE_DIR/apt/repo
+
+# supported arches/sections
+arches=( amd64 i386 )
+
+# Preserve existing components but don't add any non-existing ones
+for component in main testing experimental ; do
+	if ls "$APTDIR/dists/*/$component" >/dev/null 2>&1 ; then
+		components+=( $component )
+	fi
+done
+
+dists=( $(find "${APTDIR}/dists" -maxdepth 1 -mindepth 1 -type d) )
+
+# override component if it is set
+if [ "$COMPONENT" ]; then
+	components=( $COMPONENT )
+fi
+
+# release the debs
+for version in "${dists[@]}"; do
+	for component in "${components[@]}"; do
+		codename="${version//debootstrap-}"
+
+		# update the filelist for this codename/component
+		find "$APTDIR/pool/$component" \
+			-name *~${codename#*-}*.deb > "$APTDIR/dists/$codename/$component/filelist"
+	done
+done
+
+# run the apt-ftparchive commands so we can have pinning
+apt-ftparchive generate "$APTDIR/conf/apt-ftparchive.conf"
+
+for dist in "${dists[@]}"; do
+	version=$(basename "$dist")
+	for component in "${components[@]}"; do
+		codename="${version//debootstrap-}"
+
+		apt-ftparchive \
+			-o "APT::FTPArchive::Release::Codename=$codename" \
+			-o "APT::FTPArchive::Release::Suite=$codename" \
+			-c "$APTDIR/conf/docker-engine-release.conf" \
+			release \
+			"$APTDIR/dists/$codename" > "$APTDIR/dists/$codename/Release"
+
+		for arch in "${arches[@]}"; do
+			apt-ftparchive \
+				-o "APT::FTPArchive::Release::Codename=$codename" \
+				-o "APT::FTPArchive::Release::Suite=$codename" \
+				-o "APT::FTPArchive::Release::Component=$component" \
+				-o "APT::FTPArchive::Release::Architecture=$arch" \
+				-c "$APTDIR/conf/docker-engine-release.conf" \
+				release \
+				"$APTDIR/dists/$codename/$component/binary-$arch" > "$APTDIR/dists/$codename/$component/binary-$arch/Release"
+		done
+	done
+done

project/PACKAGE-REPO-MAINTENANCE.md

@@ -16,19 +16,14 @@ docker build --rm --force-rm -t docker-dev:master .
 docker run --rm -it --privileged \
     -v /path/to/your/repos/dir:/volumes/repos \
     -v $HOME/.gnupg:/root/.gnupg \
+    -e GPG_PASSPHRASE \
     -e DOCKER_RELEASE_DIR=/volumes/repos \
-    docker-dev:master hack/make.sh clean-apt-repo clean-yum-repo
+    docker-dev:master hack/make.sh clean-apt-repo clean-yum-repo generate-index-listing sign-repos
 ```
 
-3. Re-sign the repo with your gpg key
+3. Upload the changed repos to `s3` (if you host on s3)
 
-```bash
-./hack/make/sign-repos
-```
-
-4. Upload the changed repos to `s3` (if you host on s3)
-
-5. Purge the cache, PURGE the cache, PURGE THE CACHE!
+4. Purge the cache, PURGE the cache, PURGE THE CACHE!
 
 ### How to get out of a sticky situation
 
@@ -48,7 +43,13 @@ Otherwise CELEBRATE!
 Re-sign the repo with your gpg key:
 
 ```bash
-./hack/make/sign-repos
+docker build --rm --force-rm -t docker-dev:master .
+docker run --rm -it --privileged \
+    -v /path/to/your/repos/dir:/volumes/repos \
+    -v $HOME/.gnupg:/root/.gnupg \
+    -e GPG_PASSPHRASE \
+    -e DOCKER_RELEASE_DIR=/volumes/repos \
+    docker-dev:master hack/make.sh sign-repos
 ```
 
 Upload the changed repo to `s3` (if that is where you host)
@@ -63,16 +64,11 @@ docker build --rm --force-rm -t docker-dev:master .
 docker run --rm -it --privileged \
     -v /path/to/your/repos/dir:/volumes/repos \
     -v $HOME/.gnupg:/root/.gnupg \
+    -e GPG_PASSPHRASE \
     -e DOCKER_RELEASE_DIR=/volumes/repos \
-    docker-dev:master hack/make.sh update-apt-repo
+    docker-dev:master hack/make.sh update-apt-repo generate-index-listing sign-repos
 ```
 
-Re-sign the repo with your gpg key:
-
-```bash
-./hack/make/sign-repos
-```
-
-Upload the changed repo to `s3` (if that is where you host)
+4. Upload the changed repo to `s3` (if that is where you host)
 
 PURGE THE CACHE.