only run hub update on error or if it's in a volume

This reverts commit 2291a232cb.

.github/workflows/docker-tests.yml

@@ -59,15 +59,15 @@ jobs:
           cd docker/test
           python -m pip install --upgrade pipenv wheel
 
-      - name: "Cache virtualenvs"
+      #- name: "Cache virtualenvs"
-        id: cache-pipenv
+      #  id: cache-pipenv
-        uses: actions/cache@v4
+      #  uses: actions/cache@v4
-        with:
+      #  with:
-          path: ~/.local/share/virtualenvs
+      #    path: ~/.local/share/virtualenvs
-          key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }}
+      #    key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }}
 
       - name: "Install dependencies"
-        if: steps.cache-pipenv.outputs.cache-hit != 'true'
+        #if: steps.cache-pipenv.outputs.cache-hit != 'true'
         run: |
           cd docker/test
           pipenv install --deploy

docker/README.md

@@ -134,6 +134,7 @@ labels:
   type: apache2
 ```
 
+
 ## Recommended configuration
 
 ### Volumes
@@ -145,14 +146,6 @@ to avoid losing credentials and decision data in case of container destruction a
 * Acquisition: `/etc/crowdsec/acquis.d` and/or `/etc/crowdsec.acquis.yaml` (yes, they can be nested in `/etc/crowdsec`)
 * Database when using SQLite (default): `/var/lib/crowdsec/data`
 
-### Hub updates
-
-To ensure you have the latest version of the collections, scenarios, parsers, etc., you can set the variable `DO_HUB_UPGRADE` to true.
-This will perform an update/upgrade of the hub every time the container is started.
-
-Be aware that if your container is misbehaving and caught in a restart loop, the CrowdSec hub may ban your IP for some time and your containers
-will run with the version of the hub that is cached in the container's image. If you enable `DO_HUB_UPGRADE`, do it when your infrastructure is running
-correctly and make sure you have some monitoring in place.
 
 ## Start a Crowdsec instance
 
@@ -323,7 +316,7 @@ config.yaml) each time the container is run.
 | `BOUNCERS_ALLOWED_OU`   | bouncer-ou | OU values allowed for bouncers, separated by comma |
 |                         | | |
 | __Hub management__      | | |
-| `DO_HUB_UPGRADE`        | false | Force hub update / upgrade when the container starts. If for some reason the container restarts too often, it may lead to a temporary ban from hub updates. |
+| `NO_HUB_UPGRADE`        | false | Skip hub update / upgrade when the container starts |
 | `COLLECTIONS`           | | Collections to install, separated by space: `-e COLLECTIONS="crowdsecurity/linux crowdsecurity/apache2"` |
 | `PARSERS`               | | Parsers to install, separated by space |
 | `SCENARIOS`             | | Scenarios to install, separated by space |

docker/docker_start.sh

@@ -50,6 +50,34 @@ cscli() {
     command cscli -c "$CONFIG_FILE" "$@"
 }
 
+run_hub_update() {
+    index_modification_time=$(stat -c %Y /etc/crowdsec/hub/.index.json 2>/dev/null)
+    #Run cscli hub update if no date or if the index file is older than 24h
+    if [ -z "$index_modification_time" ] || [ $(( $(date +%s) - $index_modification_time )) -gt 86400 ]; then
+        cscli hub update
+    else
+        echo "Skipping hub update, index file is recent"
+    fi
+}
+
+run_hub_update_if_from_volume() {
+    path=$(readlink -f "/etc/crowdsec/hub/.index.json") # even though it's unlikely, resolve symlink
+    mounts=$(awk '{print $2}' /proc/mounts)
+    while true; do
+        if grep -qE ^"$path"$ <<< "$mounts"; then
+            echo "$path was found in a volume, running hub update"
+            run_hub_update
+            return 0
+        fi
+        path=$(dirname "$path")
+        if [ "$path" = "/" ]; then
+            echo "It looks like the hub index is not in a volume, skipping update"
+            return 1
+        fi
+    done
+    return 1 #unreachable
+}
+
 # conf_get <key> [file_path]
 # retrieve a value from a file (by default $CONFIG_FILE)
 conf_get() {
@@ -120,6 +148,12 @@ cscli_if_clean() {
             echo "Running: cscli $error_only $itemtype $action \"$obj\" $*"
             # shellcheck disable=SC2086
             cscli $error_only "$itemtype" "$action" "$obj" "$@"
+            if [ "$?" -ne 0 ]; then
+                echo "Failed to $action $itemtype/$obj, running hub update before retrying"
+                run_hub_update
+                # shellcheck disable=SC2086
+                cscli $error_only "$itemtype" "$action" "$obj" "$@"
+            fi
         fi
     done
 }
@@ -304,8 +338,9 @@ conf_set_if "$PLUGIN_DIR" '.config_paths.plugin_dir = strenv(PLUGIN_DIR)'
 
 ## Install hub items
 
-if istrue "$DO_HUB_UPGRADE"; then
+run_hub_update_if_from_volume || true
-    cscli hub update || true
+
+if isfalse "$NO_HUB_UPGRADE"; then
     cscli hub upgrade || true
 fi
 

docker/preload-hub-items

@@ -19,4 +19,4 @@ for itemtype in $types; do
     fi
 done
 
-echo " done."
+echo " done."

test/bin/preload-hub-items

@@ -24,4 +24,4 @@ for itemtype in $types; do
     fi
 done
 
-echo " done."
+echo " done."