Revert "docker: pre-download all hub items and data, opt-in hub updat… (#2947)

.github/workflows/docker-tests.yml

@@ -59,15 +59,15 @@ jobs:
           cd docker/test
           python -m pip install --upgrade pipenv wheel
 
-      - name: "Cache virtualenvs"
+      #- name: "Cache virtualenvs"
-        id: cache-pipenv
+      #  id: cache-pipenv
-        uses: actions/cache@v4
+      #  uses: actions/cache@v4
-        with:
+      #  with:
-          path: ~/.local/share/virtualenvs
+      #    path: ~/.local/share/virtualenvs
-          key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }}
+      #    key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }}
 
       - name: "Install dependencies"
-        if: steps.cache-pipenv.outputs.cache-hit != 'true'
+        #if: steps.cache-pipenv.outputs.cache-hit != 'true'
         run: |
           cd docker/test
           pipenv install --deploy

Dockerfile

@@ -25,7 +25,6 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
     ./wizard.sh --docker-mode && \
     cd - >/dev/null && \
     cscli hub update && \
-    ./docker/preload-hub-items && \
     cscli collections install crowdsecurity/linux && \
     cscli parsers install crowdsecurity/whitelists
 

Dockerfile.debian

@@ -30,7 +30,6 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
     ./wizard.sh --docker-mode && \
     cd - >/dev/null && \
     cscli hub update && \
-    ./docker/preload-hub-items && \
     cscli collections install crowdsecurity/linux && \
     cscli parsers install crowdsecurity/whitelists
 

docker/README.md

@@ -134,6 +134,7 @@ labels:
   type: apache2
 ```
 
+
 ## Recommended configuration
 
 ### Volumes
@@ -145,14 +146,6 @@ to avoid losing credentials and decision data in case of container destruction a
 * Acquisition: `/etc/crowdsec/acquis.d` and/or `/etc/crowdsec.acquis.yaml` (yes, they can be nested in `/etc/crowdsec`)
 * Database when using SQLite (default): `/var/lib/crowdsec/data`
 
-### Hub updates
-
-To ensure you have the latest version of the collections, scenarios, parsers, etc., you can set the variable `DO_HUB_UPGRADE` to true.
-This will perform an update/upgrade of the hub every time the container is started.
-
-Be aware that if your container is misbehaving and caught in a restart loop, the CrowdSec hub may ban your IP for some time and your containers
-will run with the version of the hub that is cached in the container's image. If you enable `DO_HUB_UPGRADE`, do it when your infrastructure is running
-correctly and make sure you have some monitoring in place.
 
 ## Start a Crowdsec instance
 
@@ -323,7 +316,7 @@ config.yaml) each time the container is run.
 | `BOUNCERS_ALLOWED_OU`   | bouncer-ou | OU values allowed for bouncers, separated by comma |
 |                         | | |
 | __Hub management__      | | |
-| `DO_HUB_UPGRADE`        | false | Force hub update / upgrade when the container starts. If for some reason the container restarts too often, it may lead to a temporary ban from hub updates. |
+| `NO_HUB_UPGRADE`        | false | Skip hub update / upgrade when the container starts |
 | `COLLECTIONS`           | | Collections to install, separated by space: `-e COLLECTIONS="crowdsecurity/linux crowdsecurity/apache2"` |
 | `PARSERS`               | | Parsers to install, separated by space |
 | `SCENARIOS`             | | Scenarios to install, separated by space |

docker/docker_start.sh

@@ -304,8 +304,9 @@ conf_set_if "$PLUGIN_DIR" '.config_paths.plugin_dir = strenv(PLUGIN_DIR)'
 
 ## Install hub items
 
-if istrue "$DO_HUB_UPGRADE"; then
+cscli hub update || true
-    cscli hub update || true
+
+if isfalse "$NO_HUB_UPGRADE"; then
     cscli hub upgrade || true
 fi
 

docker/preload-hub-items

@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-
-set -eu
-
-# pre-download everything but don't install anything
-
-echo "Pre-downloading Hub content..."
-
-types=$(cscli hub types -o raw)
-
-for itemtype in $types; do
-    ALL_ITEMS=$(cscli "$itemtype" list -a -o json | itemtype="$itemtype" yq '.[env(itemtype)][] | .name')
-    if [[ -n "${ALL_ITEMS}" ]]; then
-        #shellcheck disable=SC2086
-        cscli "$itemtype" install \
-            $ALL_ITEMS \
-            --download-only \
-            --error
-    fi
-done
-
-echo " done."

test/bin/preload-hub-items

@@ -9,12 +9,20 @@ THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
 
 # pre-download everything but don't install anything
 
-echo "Pre-downloading Hub content..."
+echo -n "Purging existing hub..."
 
 types=$("$CSCLI" hub types -o raw)
 
 for itemtype in $types; do
-    ALL_ITEMS=$("$CSCLI" "$itemtype" list -a -o json | itemtype="$itemtype" yq '.[env(itemtype)][] | .name')
+    "$CSCLI" "${itemtype}" delete --all --error --purge --force
+done
+
+echo " done."
+
+echo -n "Pre-downloading Hub content..."
+
+for itemtype in $types; do
+    ALL_ITEMS=$("$CSCLI" "$itemtype" list -a -o json | jq --arg itemtype "$itemtype" -r '.[$itemtype][].name')
     if [[ -n "${ALL_ITEMS}" ]]; then
         #shellcheck disable=SC2086
         "$CSCLI" "$itemtype" install \
@@ -24,4 +32,11 @@ for itemtype in $types; do
     fi
 done
 
+# XXX: download-only works only for collections, not for parsers, scenarios, postoverflows.
+# so we have to delete the links manually, and leave the downloaded files in place
+
+for itemtype in $types; do
+    "$CSCLI" "$itemtype" delete --all --error
+done
+
 echo " done."