CI: refactor makefile for plugins and vendor target (#2256)

Makefile

@@ -8,17 +8,8 @@ BUILD_CODENAME ?= alphaga
 CROWDSEC_FOLDER = ./cmd/crowdsec
 CSCLI_FOLDER = ./cmd/crowdsec-cli/
 
-HTTP_PLUGIN_FOLDER = plugins/notifications/http
-SLACK_PLUGIN_FOLDER = plugins/notifications/slack
-SPLUNK_PLUGIN_FOLDER = plugins/notifications/splunk
-EMAIL_PLUGIN_FOLDER = plugins/notifications/email
-DUMMY_PLUGIN_FOLDER = plugins/notifications/dummy
-
-HTTP_PLUGIN_BIN = notification-http$(EXT)
-SLACK_PLUGIN_BIN = notification-slack$(EXT)
-SPLUNK_PLUGIN_BIN = notification-splunk$(EXT)
-EMAIL_PLUGIN_BIN = notification-email$(EXT)
-DUMMY_PLUGIN_BIN = notification-dummy$(EXT)
+PLUGINS ?= $(patsubst ./plugins/notifications/%,%,$(wildcard ./plugins/notifications/*))
+PLUGINS_DIR = ./plugins/notifications
 
 CROWDSEC_BIN = crowdsec$(EXT)
 CSCLI_BIN = cscli$(EXT)
@@ -57,13 +48,20 @@ RELDIR = crowdsec-$(BUILD_VERSION)
 MAKE_FLAGS = --no-print-directory GOARCH=$(GOARCH) GOOS=$(GOOS) RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)"
 
 .PHONY: build
-build: goversion crowdsec cscli plugins
+build: pre-build goversion crowdsec cscli plugins
+
+pre-build:
+	$(info Building $(BUILD_VERSION) ($(BUILD_TAG)) for $(GOOS)/$(GOARCH))
+	$(info )
 
 .PHONY: all
 all: clean test build
 
 .PHONY: plugins
-plugins: http-plugin slack-plugin splunk-plugin email-plugin dummy-plugin
+plugins:
+	@$(foreach plugin,$(PLUGINS), \
+		$(MAKE) -C $(PLUGINS_DIR)/$(plugin) build $(MAKE_FLAGS); \
+	)
 
 .PHONY: clean
 clean: testclean
@@ -73,11 +71,9 @@ clean: testclean
 	@$(RM) $(CSCLI_BIN) $(WIN_IGNORE_ERR)
 	@$(RM) *.log $(WIN_IGNORE_ERR)
 	@$(RM) crowdsec-release.tgz $(WIN_IGNORE_ERR)
-	@$(RM) ./$(HTTP_PLUGIN_FOLDER)/$(HTTP_PLUGIN_BIN) $(WIN_IGNORE_ERR)
-	@$(RM) ./$(SLACK_PLUGIN_FOLDER)/$(SLACK_PLUGIN_BIN) $(WIN_IGNORE_ERR)
-	@$(RM) ./$(SPLUNK_PLUGIN_FOLDER)/$(SPLUNK_PLUGIN_BIN) $(WIN_IGNORE_ERR)
-	@$(RM) ./$(EMAIL_PLUGIN_FOLDER)/$(EMAIL_PLUGIN_BIN) $(WIN_IGNORE_ERR)
-	@$(RM) ./$(DUMMY_PLUGIN_FOLDER)/$(DUMMY_PLUGIN_BIN) $(WIN_IGNORE_ERR)
+	@$(foreach plugin,$(PLUGINS), \
+		$(MAKE) -C $(PLUGINS_DIR)/$(plugin) clean $(MAKE_FLAGS); \
+	)
 
 
 cscli: goversion
@@ -86,21 +82,6 @@ cscli: goversion
 crowdsec: goversion
 	@$(MAKE) -C $(CROWDSEC_FOLDER) build $(MAKE_FLAGS)
 
-http-plugin: goversion
-	@$(MAKE) -C ./$(HTTP_PLUGIN_FOLDER) build $(MAKE_FLAGS)
-
-slack-plugin: goversion
-	@$(MAKE) -C ./$(SLACK_PLUGIN_FOLDER) build $(MAKE_FLAGS)
-
-splunk-plugin: goversion
-	@$(MAKE) -C ./$(SPLUNK_PLUGIN_FOLDER) build $(MAKE_FLAGS)
-
-email-plugin: goversion
-	@$(MAKE) -C ./$(EMAIL_PLUGIN_FOLDER) build $(MAKE_FLAGS)
-
-dummy-plugin: goversion
-	$(MAKE) -C ./$(DUMMY_PLUGIN_FOLDER) build $(MAKE_FLAGS)
-
 .PHONY: testclean
 testclean: bats-clean
 	@$(RM) pkg/apiserver/ent $(WIN_IGNORE_ERR)
@@ -132,35 +113,33 @@ localstack:
 localstack-stop:
 	docker-compose -f test/localstack/docker-compose.yml down
 
-package-common:
+.PHONY: vendor
+vendor:
+	@echo "Vendoring dependencies"
+	@$(GOCMD) mod vendor
+	@$(foreach plugin,$(PLUGINS), \
+		$(MAKE) -C $(PLUGINS_DIR)/$(plugin) vendor $(MAKE_FLAGS); \
+	)
+
+.PHONY: package
+package:
 	@echo "Building Release to dir $(RELDIR)"
 	@$(MKDIR) $(RELDIR)/cmd/crowdsec
 	@$(MKDIR) $(RELDIR)/cmd/crowdsec-cli
-	@$(MKDIR) $(RELDIR)/$(HTTP_PLUGIN_FOLDER)
-	@$(MKDIR) $(RELDIR)/$(SLACK_PLUGIN_FOLDER)
-	@$(MKDIR) $(RELDIR)/$(SPLUNK_PLUGIN_FOLDER)
-	@$(MKDIR) $(RELDIR)/$(EMAIL_PLUGIN_FOLDER)
-
 	@$(CP) $(CROWDSEC_FOLDER)/$(CROWDSEC_BIN) $(RELDIR)/cmd/crowdsec
 	@$(CP) $(CSCLI_FOLDER)/$(CSCLI_BIN) $(RELDIR)/cmd/crowdsec-cli
 
-	@$(CP) ./$(HTTP_PLUGIN_FOLDER)/$(HTTP_PLUGIN_BIN) $(RELDIR)/$(HTTP_PLUGIN_FOLDER)
-	@$(CP) ./$(SLACK_PLUGIN_FOLDER)/$(SLACK_PLUGIN_BIN) $(RELDIR)/$(SLACK_PLUGIN_FOLDER)
-	@$(CP) ./$(SPLUNK_PLUGIN_FOLDER)/$(SPLUNK_PLUGIN_BIN) $(RELDIR)/$(SPLUNK_PLUGIN_FOLDER)
-	@$(CP) ./$(EMAIL_PLUGIN_FOLDER)/$(EMAIL_PLUGIN_BIN) $(RELDIR)/$(EMAIL_PLUGIN_FOLDER)
-
-	@$(CP) ./$(HTTP_PLUGIN_FOLDER)/http.yaml $(RELDIR)/$(HTTP_PLUGIN_FOLDER)
-	@$(CP) ./$(SLACK_PLUGIN_FOLDER)/slack.yaml $(RELDIR)/$(SLACK_PLUGIN_FOLDER)
-	@$(CP) ./$(SPLUNK_PLUGIN_FOLDER)/splunk.yaml $(RELDIR)/$(SPLUNK_PLUGIN_FOLDER)
-	@$(CP) ./$(EMAIL_PLUGIN_FOLDER)/email.yaml $(RELDIR)/$(EMAIL_PLUGIN_FOLDER)
+	@$(foreach plugin,$(PLUGINS), \
+		$(MKDIR) $(RELDIR)/$(PLUGINS_DIR)/$(plugin); \
+		$(CP) $(PLUGINS_DIR)/$(plugin)/notification-$(plugin)$(EXT) $(RELDIR)/$(PLUGINS_DIR)/$(plugin); \
+		$(CP) $(PLUGINS_DIR)/$(plugin)/$(plugin).yaml $(RELDIR)/$(PLUGINS_DIR)/$(plugin)/; \
+	)
 
 	@$(CPR) ./config $(RELDIR)
 	@$(CP) wizard.sh $(RELDIR)
 	@$(CP) scripts/test_env.sh $(RELDIR)
 	@$(CP) scripts/test_env.ps1 $(RELDIR)
 
-.PHONY: package
-package: package-common
 	@tar cvzf crowdsec-release.tgz $(RELDIR)
 
 .PHONY: check_release

cmd/crowdsec/Makefile

@@ -57,7 +57,7 @@ install-conf:
 install-bin:
 	install -v -m 755 -D "$(CROWDSEC_BIN)" "$(BIN_PREFIX)/$(CROWDSEC_BIN)" || exit
 
-.PHONY: systemd"$(BIN_PREFI"$(BIN_PREFIX)/$(CROWDSEC_BIN)""$(BIN_PREFIX)/$(CROWDSEC_BIN)"X)/$(CROWDSEC_BIN)"
+.PHONY: systemd
 systemd: install
 	CFG=$(CFG_PREFIX) PID=$(PID_DIR) BIN=$(BIN_PREFIX)"/"$(CROWDSEC_BIN) envsubst < ../../config/crowdsec.service > "$(SYSTEMD_PATH_FILE)"
 	systemctl daemon-reload

mk/platform/freebsd.mk

@@ -1,5 +1,3 @@
 # FreeBSD specific
 
 MAKE=gmake
-
-$(info building for FreeBSD)

mk/platform/linux.mk

@@ -1,5 +1,3 @@
 # Linux specific
 
 MAKE=make
-
-$(info Building for linux)

mk/platform/openbsd.mk

@@ -1,5 +1,3 @@
 # OpenBSD specific
 
 MAKE=gmake
-
-$(info building for OpenBSD)

mk/platform/windows.mk

@@ -18,5 +18,3 @@ CP=Copy-Item
 CPR=Copy-Item -Recurse
 MKDIR=New-Item -ItemType directory
 WIN_IGNORE_ERR=; exit 0
-
-$(info Building for windows)

plugins/notifications/dummy/Makefile

@@ -4,14 +4,20 @@ ifeq ($(OS), Windows_NT)
 	EXT = .exe
 endif
 
-# Go parameters
+PLUGIN = dummy
+BINARY_NAME = notification-$(PLUGIN)$(EXT)
+
 GOCMD = go
 GOBUILD = $(GOCMD) build
 
-BINARY_NAME = notification-dummy$(EXT)
-
 build: clean
 	$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
 
+.PHONY: clean
 clean:
 	@$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR)
+
+.PHONY: vendor
+vendor:
+	@echo "vendoring $(PLUGIN) plugin..."
+	@$(GOCMD) mod vendor

plugins/notifications/email/Makefile

@@ -4,14 +4,20 @@ ifeq ($(OS), Windows_NT)
 	EXT = .exe
 endif
 
-# Go parameters
+PLUGIN = email
+BINARY_NAME = notification-$(PLUGIN)$(EXT)
+
 GOCMD = go
 GOBUILD = $(GOCMD) build
 
-BINARY_NAME = notification-email$(EXT)
-
 build: clean
 	$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
 
+.PHONY: clean
 clean:
 	@$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR)
+
+.PHONY: vendor
+vendor:
+	@echo "vendoring $(PLUGIN) plugin..."
+	@$(GOCMD) mod vendor

plugins/notifications/http/Makefile

@@ -4,14 +4,20 @@ ifeq ($(OS), Windows_NT)
 	EXT = .exe
 endif
 
-# Go parameters
+PLUGIN=http
+BINARY_NAME = notification-$(PLUGIN)$(EXT)
+
 GOCMD = go
 GOBUILD = $(GOCMD) build
 
-BINARY_NAME = notification-http$(EXT)
-
 build: clean
 	$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
 
+.PHONY: clean
 clean:
 	@$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR)
+
+.PHONY: vendor
+vendor:
+	@echo "vendoring $(PLUGIN) plugin..."
+	@$(GOCMD) mod vendor

plugins/notifications/slack/Makefile

@@ -4,14 +4,20 @@ ifeq ($(OS), Windows_NT)
 	EXT = .exe
 endif
 
-# Go parameters
+PLUGIN=slack
+BINARY_NAME = notification-$(PLUGIN)$(EXT)
+
 GOCMD = go
 GOBUILD = $(GOCMD) build
 
-BINARY_NAME = notification-slack$(EXT)
-
 build: clean
 	$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
 
+.PHONY: clean
 clean:
 	@$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR)
+
+.PHONY: vendor
+vendor:
+	@echo "vendoring $(PLUGIN) plugin..."
+	@$(GOCMD) mod vendor

plugins/notifications/splunk/Makefile

@@ -4,14 +4,20 @@ ifeq ($(OS), Windows_NT)
 	EXT = .exe
 endif
 
-# Go parameters
+PLUGIN=splunk
+BINARY_NAME = notification-$(PLUGIN)$(EXT)
+
 GOCMD = go
 GOBUILD = $(GOCMD) build
 
-BINARY_NAME = notification-splunk$(EXT)
-
 build: clean
 	$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
 
+.PHONY: clean
 clean:
 	@$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR)
+
+.PHONY: vendor
+vendor:
+	@echo "vendoring $(PLUGIN) plugin..."
+	@$(GOCMD) mod vendor

test/bats.mk

@@ -24,7 +24,8 @@ DATA_DIR = $(LOCAL_DIR)/var/lib/crowdsec/data
 LOCAL_INIT_DIR = $(TEST_DIR)/local-init
 LOG_DIR = $(LOCAL_DIR)/var/log
 PID_DIR = $(LOCAL_DIR)/var/run
-PLUGIN_DIR = $(LOCAL_DIR)/lib/crowdsec/plugins
+# do not shadow $(PLUGINS_DIR) from the main Makefile
+BATS_PLUGIN_DIR = $(LOCAL_DIR)/lib/crowdsec/plugins
 DB_BACKEND ?= sqlite
 
 CROWDSEC ?= $(BIN_DIR)/crowdsec
@@ -43,7 +44,7 @@ export CONFIG_YAML="$(CONFIG_DIR)/config.yaml"
 export LOCAL_INIT_DIR="$(LOCAL_INIT_DIR)"
 export LOG_DIR="$(LOG_DIR)"
 export PID_DIR="$(PID_DIR)"
-export PLUGIN_DIR="$(PLUGIN_DIR)"
+export PLUGIN_DIR="$(BATS_PLUGIN_DIR)"
 export DB_BACKEND="$(DB_BACKEND)"
 export INIT_BACKEND="$(INIT_BACKEND)"
 export CONFIG_BACKEND="$(CONFIG_BACKEND)"
@@ -66,10 +67,10 @@ bats-check-requirements:
 
 # Build and installs crowdsec in a local directory. Rebuilds if already exists.
 bats-build: bats-environment bats-check-requirements
-	@mkdir -p $(BIN_DIR) $(LOG_DIR) $(PID_DIR) $(PLUGIN_DIR)
-	@TEST_COVERAGE=$(TEST_COVERAGE) DEFAULT_CONFIGDIR=$(CONFIG_DIR) DEFAULT_DATADIR=$(DATA_DIR) $(MAKE) goversion crowdsec cscli plugins
+	@$(MKDIR) $(BIN_DIR) $(LOG_DIR) $(PID_DIR) $(BATS_PLUGIN_DIR)
+	@TEST_COVERAGE=$(TEST_COVERAGE) DEFAULT_CONFIGDIR=$(CONFIG_DIR) DEFAULT_DATADIR=$(DATA_DIR) $(MAKE) build
 	@install -m 0755 cmd/crowdsec/crowdsec cmd/crowdsec-cli/cscli $(BIN_DIR)/
-	@install -m 0755 plugins/notifications/*/notification-* $(PLUGIN_DIR)/
+	@install -m 0755 plugins/notifications/*/notification-* $(BATS_PLUGIN_DIR)/
 
 # Create a reusable package with initial configuration + data
 bats-fixture:
@@ -99,10 +100,7 @@ bats-lint:
 	@shellcheck --version >/dev/null 2>&1 || (echo "ERROR: shellcheck is required."; exit 1)
 	@shellcheck -x $(TEST_DIR)/bats/*.bats
 
-
 bats-test-package: bats-environment
 	$(TEST_DIR)/instance-data make
 	$(TEST_DIR)/run-tests $(TEST_DIR)/bats
 	$(TEST_DIR)/run-tests $(TEST_DIR)/dyn-bats
-
-.PHONY: bats-environment