From c6e40191ddf7a87b8c0d946c36083c084677e5f1 Mon Sep 17 00:00:00 2001 From: blotus Date: Thu, 18 Apr 2024 15:33:51 +0200 Subject: [PATCH] =?UTF-8?q?Revert=20"docker:=20pre-download=20all=20hub=20?= =?UTF-8?q?items=20and=20data,=20opt-in=20hub=20updat=E2=80=A6=20(#2947)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/docker-tests.yml | 14 +++++++------- Dockerfile | 1 - Dockerfile.debian | 1 - docker/README.md | 11 ++--------- docker/docker_start.sh | 5 +++-- docker/preload-hub-items | 22 ---------------------- test/bin/preload-hub-items | 19 +++++++++++++++++-- 7 files changed, 29 insertions(+), 44 deletions(-) delete mode 100755 docker/preload-hub-items diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml index d3ae4f90d..3e87d3ba4 100644 --- a/.github/workflows/docker-tests.yml +++ b/.github/workflows/docker-tests.yml @@ -59,15 +59,15 @@ jobs: cd docker/test python -m pip install --upgrade pipenv wheel - - name: "Cache virtualenvs" - id: cache-pipenv - uses: actions/cache@v4 - with: - path: ~/.local/share/virtualenvs - key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }} + #- name: "Cache virtualenvs" + # id: cache-pipenv + # uses: actions/cache@v4 + # with: + # path: ~/.local/share/virtualenvs + # key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }} - name: "Install dependencies" - if: steps.cache-pipenv.outputs.cache-hit != 'true' + #if: steps.cache-pipenv.outputs.cache-hit != 'true' run: | cd docker/test pipenv install --deploy diff --git a/Dockerfile b/Dockerfile index d2b01ed77..1e311bfa8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,6 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \ ./wizard.sh --docker-mode && \ cd - >/dev/null && \ cscli hub update && \ - ./docker/preload-hub-items && \ cscli collections install crowdsecurity/linux && \ cscli parsers install crowdsecurity/whitelists diff --git a/Dockerfile.debian b/Dockerfile.debian index 0d094f8df..ee0fa32bb 100644 --- a/Dockerfile.debian +++ b/Dockerfile.debian @@ -30,7 +30,6 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \ ./wizard.sh --docker-mode && \ cd - >/dev/null && \ cscli hub update && \ - ./docker/preload-hub-items && \ cscli collections install crowdsecurity/linux && \ cscli parsers install crowdsecurity/whitelists diff --git a/docker/README.md b/docker/README.md index 2fea57a61..5e39838a1 100644 --- a/docker/README.md +++ b/docker/README.md @@ -134,6 +134,7 @@ labels: type: apache2 ``` + ## Recommended configuration ### Volumes @@ -145,14 +146,6 @@ to avoid losing credentials and decision data in case of container destruction a * Acquisition: `/etc/crowdsec/acquis.d` and/or `/etc/crowdsec.acquis.yaml` (yes, they can be nested in `/etc/crowdsec`) * Database when using SQLite (default): `/var/lib/crowdsec/data` -### Hub updates - -To ensure you have the latest version of the collections, scenarios, parsers, etc., you can set the variable `DO_HUB_UPGRADE` to true. -This will perform an update/upgrade of the hub every time the container is started. - -Be aware that if your container is misbehaving and caught in a restart loop, the CrowdSec hub may ban your IP for some time and your containers -will run with the version of the hub that is cached in the container's image. If you enable `DO_HUB_UPGRADE`, do it when your infrastructure is running -correctly and make sure you have some monitoring in place. ## Start a Crowdsec instance @@ -323,7 +316,7 @@ config.yaml) each time the container is run. | `BOUNCERS_ALLOWED_OU` | bouncer-ou | OU values allowed for bouncers, separated by comma | | | | | | __Hub management__ | | | -| `DO_HUB_UPGRADE` | false | Force hub update / upgrade when the container starts. If for some reason the container restarts too often, it may lead to a temporary ban from hub updates. | +| `NO_HUB_UPGRADE` | false | Skip hub update / upgrade when the container starts | | `COLLECTIONS` | | Collections to install, separated by space: `-e COLLECTIONS="crowdsecurity/linux crowdsecurity/apache2"` | | `PARSERS` | | Parsers to install, separated by space | | `SCENARIOS` | | Scenarios to install, separated by space | diff --git a/docker/docker_start.sh b/docker/docker_start.sh index 26c5b0eee..dd96184cc 100755 --- a/docker/docker_start.sh +++ b/docker/docker_start.sh @@ -304,8 +304,9 @@ conf_set_if "$PLUGIN_DIR" '.config_paths.plugin_dir = strenv(PLUGIN_DIR)' ## Install hub items -if istrue "$DO_HUB_UPGRADE"; then - cscli hub update || true +cscli hub update || true + +if isfalse "$NO_HUB_UPGRADE"; then cscli hub upgrade || true fi diff --git a/docker/preload-hub-items b/docker/preload-hub-items deleted file mode 100755 index d02b09485..000000000 --- a/docker/preload-hub-items +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -set -eu - -# pre-download everything but don't install anything - -echo "Pre-downloading Hub content..." - -types=$(cscli hub types -o raw) - -for itemtype in $types; do - ALL_ITEMS=$(cscli "$itemtype" list -a -o json | itemtype="$itemtype" yq '.[env(itemtype)][] | .name') - if [[ -n "${ALL_ITEMS}" ]]; then - #shellcheck disable=SC2086 - cscli "$itemtype" install \ - $ALL_ITEMS \ - --download-only \ - --error - fi -done - -echo " done." diff --git a/test/bin/preload-hub-items b/test/bin/preload-hub-items index ddf7fecba..14e9cff99 100755 --- a/test/bin/preload-hub-items +++ b/test/bin/preload-hub-items @@ -9,12 +9,20 @@ THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) # pre-download everything but don't install anything -echo "Pre-downloading Hub content..." +echo -n "Purging existing hub..." types=$("$CSCLI" hub types -o raw) for itemtype in $types; do - ALL_ITEMS=$("$CSCLI" "$itemtype" list -a -o json | itemtype="$itemtype" yq '.[env(itemtype)][] | .name') + "$CSCLI" "${itemtype}" delete --all --error --purge --force +done + +echo " done." + +echo -n "Pre-downloading Hub content..." + +for itemtype in $types; do + ALL_ITEMS=$("$CSCLI" "$itemtype" list -a -o json | jq --arg itemtype "$itemtype" -r '.[$itemtype][].name') if [[ -n "${ALL_ITEMS}" ]]; then #shellcheck disable=SC2086 "$CSCLI" "$itemtype" install \ @@ -24,4 +32,11 @@ for itemtype in $types; do fi done +# XXX: download-only works only for collections, not for parsers, scenarios, postoverflows. +# so we have to delete the links manually, and leave the downloaded files in place + +for itemtype in $types; do + "$CSCLI" "$itemtype" delete --all --error +done + echo " done."