Decouple docker image from package release (#2791)

- entry point fixes for 1.6.0
 - correctly override BUILD_VERSION argument
 - manual release workflow

.github/workflows/publish-docker-master.yml

@@ -0,0 +1,47 @@
+name: Publish Docker image on Push to Master
+
+on:
+  push:
+    branches: [ master ]
+    paths:
+      - 'pkg/**'
+      - 'cmd/**'
+      - 'plugins/**'
+      - 'docker/docker_start.sh'
+      - 'docker/config.yaml'
+      - '.github/workflows/publish_docker-master.yml'
+      - '.github/workflows/publish-docker.yml'
+      - 'Dockerfile'
+      - 'Dockerfile.debian'
+      - 'go.mod'
+      - 'go.sum'
+      - 'Makefile'
+
+jobs:
+  dev-alpine:
+    uses: ./.github/workflows/publish-docker.yml
+    with:
+      platform: linux/amd64
+      crowdsec_version: ""
+      image_version: dev
+      latest: false
+      push: true
+      slim: false
+      debian: false
+    secrets:
+      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+
+  dev-debian:
+    uses: ./.github/workflows/publish-docker.yml
+    with:
+      platform: linux/amd64
+      crowdsec_version: ""
+      image_version: dev
+      latest: false
+      push: true
+      slim: false
+      debian: true
+    secrets:
+      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/publish-docker-release.yml

@@ -0,0 +1,57 @@
+name: Publish Docker images
+
+on:
+  workflow_dispatch:
+    inputs:
+      image_version:
+        description: Docker Image Version (base tag)
+        required: true
+      crowdsec_version:
+        description: Crowdsec Version (BUILD_VERSION)
+        required: true
+      latest:
+        description: Overwrite latest (and slim) tags?
+        default: false
+        required: true
+      push:
+        description: Really push?
+        default: false
+        required: true
+
+jobs:
+  alpine:
+    strategy:
+      matrix:
+        platform: ["linux/amd64", "linux/386", "linux/arm64", "linux/arm/v7", "linux/arm/v6"]
+        slim: [false, true]
+
+    uses: ./.github/workflows/publish-docker.yml
+    secrets:
+      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+    with:
+      platform: ${{ matrix.platform }}
+      image_version: ${{ github.event.inputs.image_version }}
+      crowdsec_version: ${{ github.event.inputs.crowdsec_version }}
+      latest: ${{ github.event.inputs.latest == 'true' }}
+      push: ${{ github.event.inputs.push == 'true' }}
+      slim: ${{ matrix.slim }}
+      debian: false
+
+  debian:
+    strategy:
+      matrix:
+        platform: ["linux/amd64", "linux/386", "linux/arm64"]
+
+    uses: ./.github/workflows/publish-docker.yml
+    secrets:
+      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+    with:
+      platform: ${{ matrix.platform }}
+      image_version: ${{ github.event.inputs.image_version }}
+      crowdsec_version: ${{ github.event.inputs.crowdsec_version }}
+      latest: ${{ github.event.inputs.latest == 'true' }}
+      push: ${{ github.event.inputs.push == 'true' }}
+      slim: false
+      debian: true

.github/workflows/publish-docker.yml

@@ -0,0 +1,98 @@
+name: Publish Docker image / platform
+
+on:
+  workflow_call:
+    secrets:
+      DOCKER_USERNAME:
+        required: true
+      DOCKER_PASSWORD:
+        required: true
+    inputs:
+      platform:
+        required: true
+        type: string
+      image_version:
+        required: true
+        type: string
+      crowdsec_version:
+        required: true
+        type: string
+      latest:
+        required: true
+        type: boolean
+      push:
+        required: true
+        type: boolean
+      slim:
+        required: true
+        type: boolean
+      debian:
+        required: true
+        type: boolean
+
+jobs:
+  push_to_registry:
+    name: Push Docker image to registries
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Check out the repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Prepare
+        id: prep
+        run: |
+          DOCKERHUB_IMAGE=${{ secrets.DOCKER_USERNAME }}/crowdsec
+          GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec
+          VERSION=${{ inputs.image_version }}
+          SLIM=${{ inputs.slim && '-slim' || '' }}
+          DEBIAN=${{ inputs.debian && '-debian' || '' }}
+          TAGS="${DOCKERHUB_IMAGE}:${VERSION}${SLIM}${DEBIAN},${GHCR_IMAGE}:${VERSION}${SLIM}${DEBIAN}"
+          if [[ ${{ inputs.latest }} == true ]]; then
+            if [[ ${{ inputs.slim }} == true ]]; then
+              TAGS=$TAGS,${DOCKERHUB_IMAGE}:slim${DEBIAN},${GHCR_IMAGE}:slim${DEBIAN}
+            else
+              TAGS=$TAGS,${DOCKERHUB_IMAGE}:latest${DEBIAN},${GHCR_IMAGE}:latest${DEBIAN}
+            fi
+          fi
+          echo "tags=${TAGS}" >> $GITHUB_OUTPUT
+          echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          config: .github/buildkit.toml
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./Dockerfile${{ inputs.debian && '.debian' || '' }}
+          push: ${{ inputs.push }}
+          tags: ${{ steps.prep.outputs.tags }}
+          target: ${{ inputs.slim && 'slim' || 'full' }}
+          platforms: ${{ inputs.platform }}
+          labels: |
+            org.opencontainers.image.source=${{ github.event.repository.html_url }}
+            org.opencontainers.image.created=${{ steps.prep.outputs.created }}
+            org.opencontainers.image.revision=${{ github.sha }}
+          build-args: |
+            BUILD_VERSION=${{ inputs.crowdsec_version }}

.github/workflows/publish_docker-image_on_master-debian.yml

@@ -1,71 +0,0 @@
-name: Publish Debian Docker image on Push to Master
-
-on:
-  push:
-    branches: [ master ]
-    paths:
-      - 'pkg/**'
-      - 'cmd/**'
-      - 'plugins/**'
-      - 'docker/docker_start.sh'
-      - 'docker/config.yaml'
-      - '.github/workflows/publish_docker-image_on_master-debian.yml'
-      - 'Dockerfile.debian'
-      - 'go.mod'
-      - 'go.sum'
-      - 'Makefile'
-
-jobs:
-  push_to_registry:
-    name: Push Debian Docker image to Docker Hub
-    runs-on: ubuntu-latest
-    if: ${{ github.repository_owner == 'crowdsecurity' }}
-    steps:
-
-      - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Prepare
-        id: prep
-        run: |
-          DOCKER_IMAGE=crowdsecurity/crowdsec
-          GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec
-          VERSION=dev-debian
-          TAGS="${DOCKER_IMAGE}:${VERSION},${GHCR_IMAGE}:${VERSION}"
-          echo "tags=${TAGS}" >> $GITHUB_OUTPUT
-          echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          config: .github/buildkit.toml
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build and push full image
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: ./Dockerfile.debian
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.prep.outputs.tags }}
-          platforms: linux/amd64
-          labels: |
-            org.opencontainers.image.source=${{ github.event.repository.html_url }}
-            org.opencontainers.image.created=${{ steps.prep.outputs.created }}
-            org.opencontainers.image.revision=${{ github.sha }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=min

.github/workflows/publish_docker-image_on_master.yml

@@ -1,71 +0,0 @@
-name: Publish Docker image on Push to Master
-
-on:
-  push:
-    branches: [ master ]
-    paths:
-      - 'pkg/**'
-      - 'cmd/**'
-      - 'plugins/**'
-      - 'docker/docker_start.sh'
-      - 'docker/config.yaml'
-      - '.github/workflows/publish_docker-image_on_master.yml'
-      - 'Dockerfile'
-      - 'go.mod'
-      - 'go.sum'
-      - 'Makefile'
-
-jobs:
-  push_to_registry:
-    name: Push Docker image to Docker Hub
-    runs-on: ubuntu-latest
-    if: ${{ github.repository_owner == 'crowdsecurity' }}
-    steps:
-
-      - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Prepare
-        id: prep
-        run: |
-          DOCKER_IMAGE=crowdsecurity/crowdsec
-          GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec
-          VERSION=dev
-          TAGS="${DOCKER_IMAGE}:${VERSION},${GHCR_IMAGE}:${VERSION}"
-          echo "tags=${TAGS}" >> $GITHUB_OUTPUT
-          echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          config: .github/buildkit.toml
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build and push full image
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: ./Dockerfile
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.prep.outputs.tags }}
-          platforms: linux/amd64
-          labels: |
-            org.opencontainers.image.source=${{ github.event.repository.html_url }}
-            org.opencontainers.image.created=${{ steps.prep.outputs.created }}
-            org.opencontainers.image.revision=${{ github.sha }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=min

.github/workflows/release_publish_docker-image-debian.yml

@@ -1,61 +0,0 @@
-name: Publish Docker Debian image
-
-on:
-  release:
-    types:
-      - released
-      - prereleased
-  workflow_dispatch:
-
-jobs:
-  push_to_registry:
-    name: Push Docker debian image to Docker Hub
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - name: Prepare
-        id: prep
-        run: |
-          DOCKER_IMAGE=crowdsecurity/crowdsec
-          VERSION=bullseye
-          if [[ $GITHUB_REF == refs/tags/* ]]; then
-            VERSION=${GITHUB_REF#refs/tags/}
-          elif [[ $GITHUB_REF == refs/heads/* ]]; then
-            VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -E 's#/+#-#g')
-          elif [[ $GITHUB_REF == refs/pull/* ]]; then
-            VERSION=pr-${{ github.event.number }}
-          fi
-          TAGS="${DOCKER_IMAGE}:${VERSION}-debian"
-          if [[ "${{ github.event.action }}" == "released" ]]; then
-            TAGS=$TAGS,${DOCKER_IMAGE}:latest-debian
-          fi
-          echo "version=${VERSION}" >> $GITHUB_OUTPUT
-          echo "tags=${TAGS}" >> $GITHUB_OUTPUT
-          echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          config: .github/buildkit.toml
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Build and push
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: ./Dockerfile.debian
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.prep.outputs.tags }}
-          platforms: linux/amd64,linux/arm64,linux/386
-          labels: |
-            org.opencontainers.image.source=${{ github.event.repository.html_url }}
-            org.opencontainers.image.created=${{ steps.prep.outputs.created }}
-            org.opencontainers.image.revision=${{ github.sha }}

.github/workflows/release_publish_docker-image.yml

@@ -1,86 +0,0 @@
-name: Publish Docker image
-
-on:
-  release:
-    types:
-      - released
-      - prereleased
-
-jobs:
-  push_to_registry:
-    name: Push Docker image to Docker Hub
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out the repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - name: Prepare
-        id: prep
-        run: |
-          DOCKER_IMAGE=crowdsecurity/crowdsec
-          GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec
-          VERSION=edge
-          if [[ $GITHUB_REF == refs/tags/* ]]; then
-            VERSION=${GITHUB_REF#refs/tags/}
-          elif [[ $GITHUB_REF == refs/heads/* ]]; then
-            VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -E 's#/+#-#g')
-          elif [[ $GITHUB_REF == refs/pull/* ]]; then
-            VERSION=pr-${{ github.event.number }}
-          fi
-          TAGS="${DOCKER_IMAGE}:${VERSION},${GHCR_IMAGE}:${VERSION}"
-          TAGS_SLIM="${DOCKER_IMAGE}:${VERSION}-slim,${GHCR_IMAGE}:${VERSION}-slim"
-          if [[ ${{ github.event.action }} == released ]]; then
-            TAGS=$TAGS,${DOCKER_IMAGE}:latest,${GHCR_IMAGE}:latest
-            TAGS_SLIM=$TAGS_SLIM,${DOCKER_IMAGE}:slim,${GHCR_IMAGE}:slim
-          fi
-          echo "version=${VERSION}" >> $GITHUB_OUTPUT
-          echo "tags=${TAGS}" >> $GITHUB_OUTPUT
-          echo "tags_slim=${TAGS_SLIM}" >> $GITHUB_OUTPUT
-          echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          config: .github/buildkit.toml
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build and push slim image
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: ./Dockerfile
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.prep.outputs.tags_slim }}
-          target: slim
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6,linux/386
-          labels: |
-            org.opencontainers.image.source=${{ github.event.repository.html_url }}
-            org.opencontainers.image.created=${{ steps.prep.outputs.created }}
-            org.opencontainers.image.revision=${{ github.sha }}
-
-      - name: Build and push full image
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: ./Dockerfile
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.prep.outputs.tags }}
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6,linux/386
-          labels: |
-            org.opencontainers.image.source=${{ github.event.repository.html_url }}
-            org.opencontainers.image.created=${{ steps.prep.outputs.created }}
-            org.opencontainers.image.revision=${{ github.sha }}

Dockerfile

@@ -1,8 +1,7 @@
 # vim: set ft=dockerfile:
-ARG GOVERSION=1.21.6
-ARG BUILD_VERSION
+FROM golang:1.21.6-alpine3.18 AS build
 
-FROM golang:${GOVERSION}-alpine3.18 AS build
+ARG BUILD_VERSION
 
 WORKDIR /go/src/crowdsec
 

Dockerfile.debian

@@ -1,8 +1,7 @@
 # vim: set ft=dockerfile:
-ARG GOVERSION=1.21.6
-ARG BUILD_VERSION
+FROM golang:1.21.6-bookworm AS build
 
-FROM golang:${GOVERSION}-bookworm AS build
+ARG BUILD_VERSION
 
 WORKDIR /go/src/crowdsec
 

docker/docker_start.sh

@@ -3,7 +3,7 @@
 # shellcheck disable=SC2292      # allow [ test ] syntax
 # shellcheck disable=SC2310      # allow "if function..." syntax with -e
 
-# set -e
+set -e
 shopt -s inherit_errexit
 
 # match true, TRUE, True, tRuE, etc.
@@ -109,6 +109,8 @@ cscli_if_clean() {
     for obj in $objs; do
         if cscli "$itemtype" inspect "$obj" -o json | yq -e '.tainted // false' >/dev/null 2>&1; then
             echo "Object $itemtype/$obj is tainted, skipping"
+        elif cscli "$itemtype" inspect "$obj" -o json | yq -e '.local // false' >/dev/null 2>&1; then
+            echo "Object $itemtype/$obj is local, skipping"
         else
 #            # Too verbose? Only show errors if not in debug mode
 #            if [ "$DEBUG" != "true" ]; then
@@ -301,8 +303,8 @@ fi
 conf_set_if "$PLUGIN_DIR" '.config_paths.plugin_dir = strenv(PLUGIN_DIR)'
 
 ## Install hub items
-cscli hub update
-cscli hub upgrade
+cscli hub update || true
+cscli hub upgrade || true
 
 cscli_if_clean parsers install crowdsecurity/docker-logs
 cscli_if_clean parsers install crowdsecurity/cri-logs