reenable pre-loading

Dockerfile

@@ -25,6 +25,7 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
     ./wizard.sh --docker-mode && \
     cd - >/dev/null && \
     cscli hub update && \
+    ./docker/preload-hub-items && \
     cscli collections install crowdsecurity/linux && \
     cscli parsers install crowdsecurity/whitelists
 

Dockerfile.debian

@@ -30,6 +30,7 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
     ./wizard.sh --docker-mode && \
     cd - >/dev/null && \
     cscli hub update && \
+    ./docker/preload-hub-items && \
     cscli collections install crowdsecurity/linux && \
     cscli parsers install crowdsecurity/whitelists
 

docker/preload-hub-items

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -eu
+
+# pre-download everything but don't install anything
+
+echo "Pre-downloading Hub content..."
+
+types=$(cscli hub types -o raw)
+
+for itemtype in $types; do
+    ALL_ITEMS=$(cscli "$itemtype" list -a -o json | itemtype="$itemtype" yq '.[env(itemtype)][] | .name')
+    if [[ -n "${ALL_ITEMS}" ]]; then
+        #shellcheck disable=SC2086
+        cscli "$itemtype" install \
+            $ALL_ITEMS \
+            --download-only \
+            --error
+    fi
+done
+
+echo " done."

test/bin/preload-hub-items

@@ -9,20 +9,12 @@ THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
 
 # pre-download everything but don't install anything
 
-echo -n "Purging existing hub..."
+echo "Pre-downloading Hub content..."
 
 types=$("$CSCLI" hub types -o raw)
 
 for itemtype in $types; do
-    "$CSCLI" "${itemtype}" delete --all --error --purge --force
+    ALL_ITEMS=$("$CSCLI" "$itemtype" list -a -o json | itemtype="$itemtype" yq '.[env(itemtype)][] | .name')
-done
-
-echo " done."
-
-echo -n "Pre-downloading Hub content..."
-
-for itemtype in $types; do
-    ALL_ITEMS=$("$CSCLI" "$itemtype" list -a -o json | jq --arg itemtype "$itemtype" -r '.[$itemtype][].name')
     if [[ -n "${ALL_ITEMS}" ]]; then
         #shellcheck disable=SC2086
         "$CSCLI" "$itemtype" install \
@@ -32,11 +24,4 @@ for itemtype in $types; do
     fi
 done
 
-# XXX: download-only works only for collections, not for parsers, scenarios, postoverflows.
+echo " done."
-# so we have to delete the links manually, and leave the downloaded files in place
-
-for itemtype in $types; do
-    "$CSCLI" "$itemtype" delete --all --error
-done
-
-echo " done."