Refactor configuration management (#698)

cmd/crowdsec-cli/alerts.go

@@ -199,6 +199,9 @@ func NewAlertsCmd() *cobra.Command {
 		Args:  cobra.MinimumNArgs(1),
 		PersistentPreRun: func(cmd *cobra.Command, args []string) {
 			var err error
+			if err := csConfig.LoadAPIClient(); err != nil {
+				log.Fatalf("loading api client: %s", err.Error())
+			}
 			if csConfig.API.Client == nil {
 				log.Fatalln("There is no configuration on 'api_client:'")
 			}

cmd/crowdsec-cli/bouncers.go

@@ -31,6 +31,9 @@ To list/add/delete bouncers
 		Args: cobra.MinimumNArgs(1),
 		PersistentPreRun: func(cmd *cobra.Command, args []string) {
 			var err error
+			if err := csConfig.LoadDBConfig(); err != nil {
+				log.Fatalf(err.Error())
+			}
 			dbClient, err = database.NewClient(csConfig.DbConfig)
 			if err != nil {
 				log.Fatalf("unable to create new database client: %s", err)

cmd/crowdsec-cli/capi.go

@@ -28,11 +28,14 @@ func NewCapiCmd() *cobra.Command {
 		Short: "Manage interaction with Central API (CAPI)",
 		Args:  cobra.MinimumNArgs(1),
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if err := csConfig.LoadAPIServer(); err != nil {
+				log.Fatalf(err.Error())
+			}
 			if csConfig.API.Server == nil {
 				log.Fatalln("There is no API->server configuration")
 			}
 			if csConfig.API.Server.OnlineClient == nil {
-				log.Fatalf("no configuration for crowdsec API in '%s'", *csConfig.Self)
+				log.Fatalf("no configuration for crowdsec API in '%s'", *csConfig.FilePath)
 			}
 
 			return nil
@@ -124,7 +127,12 @@ func NewCapiCmd() *cobra.Command {
 			if err != nil {
 				log.Fatalf("parsing api url ('%s'): %s", csConfig.API.Server.OnlineClient.Credentials.URL, err)
 			}
-			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+
+			if err := csConfig.LoadHub(); err != nil {
+				log.Fatalf(err.Error())
+			}
+
+			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
 				log.Fatalf("Failed to load hub index : %s", err)
 				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
 			}

cmd/crowdsec-cli/collections.go

@@ -18,13 +18,22 @@ func NewCollectionsCmd() *cobra.Command {
 		/*TBD fix help*/
 		Args: cobra.MinimumNArgs(1),
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if csConfig.Cscli == nil {
+			if err := csConfig.LoadHub(); err != nil {
+				log.Fatalf(err.Error())
+			}
+			if csConfig.Hub == nil {
 				return fmt.Errorf("you must configure cli before interacting with hub")
 			}
 
 			if err := setHubBranch(); err != nil {
 				return fmt.Errorf("error while setting hub branch: %s", err)
 			}
+
+			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
+			}
+
 			return nil
 		},
 		PersistentPostRun: func(cmd *cobra.Command, args []string) {
@@ -42,10 +51,6 @@ func NewCollectionsCmd() *cobra.Command {
 		Example: `cscli collections install crowdsec/xxx crowdsec/xyz`,
 		Args:    cobra.MinimumNArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
-			}
 			for _, name := range args {
 				InstallItem(name, cwhub.COLLECTIONS, forceAction)
 			}
@@ -62,11 +67,6 @@ func NewCollectionsCmd() *cobra.Command {
 		Example: `cscli collections remove crowdsec/xxx crowdsec/xyz`,
 		Args:    cobra.MinimumNArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
-			}
-
 			if all {
 				RemoveMany(cwhub.COLLECTIONS, "")
 			} else {
@@ -95,10 +95,6 @@ func NewCollectionsCmd() *cobra.Command {
 		Long:    `Fetch and upgrade given collection(s) from hub`,
 		Example: `cscli collections upgrade crowdsec/xxx crowdsec/xyz`,
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
-			}
 			if all {
 				UpgradeConfig(cwhub.COLLECTIONS, "", forceAction)
 			} else {
@@ -119,10 +115,6 @@ func NewCollectionsCmd() *cobra.Command {
 		Example: `cscli collections inspect crowdsec/xxx crowdsec/xyz`,
 		Args:    cobra.MinimumNArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
-			}
 			for _, name := range args {
 				InspectItem(name, cwhub.COLLECTIONS)
 			}
@@ -138,10 +130,6 @@ func NewCollectionsCmd() *cobra.Command {
 		Example: `cscli collections list`,
 		Args:    cobra.ExactArgs(0),
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
-			}
 			ListItem(cwhub.COLLECTIONS, args)
 		},
 	}

cmd/crowdsec-cli/config.go

@@ -44,7 +44,7 @@ func backupConfigToDirectory(dirPath string) error {
 		return errors.Wrapf(err, "while checking parent directory %s existence", parentDir)
 	}
 
-	if err = os.Mkdir(dirPath, 0600); err != nil {
+	if err = os.Mkdir(dirPath, 0700); err != nil {
 		return fmt.Errorf("error while creating %s : %s", dirPath, err)
 	}
 
@@ -68,7 +68,7 @@ func backupConfigToDirectory(dirPath string) error {
 	}
 
 	acquisBackupDir := dirPath + "/acquis/"
-	if err = os.Mkdir(acquisBackupDir, 0600); err != nil {
+	if err = os.Mkdir(acquisBackupDir, 0700); err != nil {
 		return fmt.Errorf("error while creating %s : %s", acquisBackupDir, err)
 	}
 
@@ -215,7 +215,7 @@ func restoreConfigFromDirectory(dirPath string) error {
 
 	/*if there is a acquisition dir, restore its content*/
 	if csConfig.Crowdsec.AcquisitionDirPath != "" {
-		if err = os.Mkdir(csConfig.Crowdsec.AcquisitionDirPath, 0600); err != nil {
+		if err = os.Mkdir(csConfig.Crowdsec.AcquisitionDirPath, 0700); err != nil {
 			return fmt.Errorf("error while creating %s : %s", csConfig.Crowdsec.AcquisitionDirPath, err)
 		}
 
@@ -389,7 +389,10 @@ func NewConfigCmd() *cobra.Command {
 		Args:    cobra.ExactArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
 			var err error
-			if err = cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+			if err := csConfig.LoadHub(); err != nil {
+				log.Fatalf(err.Error())
+			}
+			if err = cwhub.GetHubIdx(csConfig.Hub); err != nil {
 				log.Fatalf("Failed to get Hub index : %v", err)
 				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
 			}
@@ -414,7 +417,7 @@ func NewConfigCmd() *cobra.Command {
 		Args: cobra.ExactArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
 			var err error
-			if err = cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+			if err = cwhub.GetHubIdx(csConfig.Hub); err != nil {
 				log.Fatalf("Failed to get Hub index : %v", err)
 				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
 			}

cmd/crowdsec-cli/decisions.go

@@ -105,6 +105,9 @@ func NewDecisionsCmd() *cobra.Command {
 		/*TBD example*/
 		Args: cobra.MinimumNArgs(1),
 		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			if err := csConfig.LoadAPIClient(); err != nil {
+				log.Fatalf(err.Error())
+			}
 			if csConfig.API.Client == nil {
 				log.Fatalln("There is no configuration on 'api_client:'")
 			}

cmd/crowdsec-cli/hub.go

@@ -40,7 +40,10 @@ cscli hub update # Download list of available configurations from the hub
 		Short: "List installed configs",
 		Args:  cobra.ExactArgs(0),
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+			if err := csConfig.LoadHub(); err != nil {
+				log.Fatalf(err.Error())
+			}
+			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
 				log.Fatalf("Failed to get Hub index : %v", err)
 				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
 			}
@@ -77,7 +80,10 @@ Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.inde
 			return nil
 		},
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.UpdateHubIdx(csConfig.Cscli); err != nil {
+			if err := csConfig.LoadHub(); err != nil {
+				log.Fatalf(err.Error())
+			}
+			if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
 				log.Fatalf("Failed to get Hub index : %v", err)
 			}
 		},
@@ -102,7 +108,10 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if
 			return nil
 		},
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+			if err := csConfig.LoadHub(); err != nil {
+				log.Fatalf(err.Error())
+			}
+			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
 				log.Fatalf("Failed to get Hub index : %v", err)
 				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
 			}

cmd/crowdsec-cli/lapi.go

@@ -28,11 +28,14 @@ func NewLapiCmd() *cobra.Command {
 		Short: "Manage interaction with Local API (LAPI)",
 		Args:  cobra.MinimumNArgs(1),
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if err := csConfig.LoadAPIClient(); err != nil {
+				return fmt.Errorf("loading api client: %s", err.Error())
+			}
 			if csConfig.API.Client == nil {
 				log.Fatalln("There is no API->client configuration")
 			}
 			if csConfig.API.Client.Credentials == nil {
-				log.Fatalf("no configuration for crowdsec API in '%s'", *csConfig.Self)
+				log.Fatalf("no configuration for crowdsec API in '%s'", *csConfig.FilePath)
 			}
 			return nil
 		},
@@ -133,7 +136,11 @@ Keep in mind the machine needs to be validated by an administrator on LAPI side
 			if err != nil {
 				log.Fatalf("parsing api url ('%s'): %s", apiurl, err)
 			}
-			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+			if err := csConfig.LoadHub(); err != nil {
+				log.Fatalf(err.Error())
+			}
+
+			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
 				log.Fatalf("Failed to load hub index : %s", err)
 				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
 			}

cmd/crowdsec-cli/machines.go

@@ -84,9 +84,15 @@ func NewMachinesCmd() *cobra.Command {
 		Long: `
 Machines Management.
 
-To list/add/delete/register/validate machines
+To list/add/delete/validate machines
 `,
 		Example: `cscli machines [action]`,
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if err := csConfig.LoadDBConfig(); err != nil {
+				log.Fatalf(err.Error())
+			}
+			return nil
+		},
 	}
 
 	var cmdMachinesList = &cobra.Command{
@@ -97,6 +103,7 @@ To list/add/delete/register/validate machines
 		Args:    cobra.MaximumNArgs(1),
 		PersistentPreRun: func(cmd *cobra.Command, args []string) {
 			var err error
+
 			dbClient, err = database.NewClient(csConfig.DbConfig)
 			if err != nil {
 				log.Fatalf("unable to create new database client: %s", err)

cmd/crowdsec-cli/main.go

@@ -14,7 +14,7 @@ import (
 var trace_lvl, dbg_lvl, nfo_lvl, wrn_lvl, err_lvl bool
 
 var ConfigFilePath string
-var csConfig *csconfig.GlobalConfig
+var csConfig *csconfig.Config
 var dbClient *database.Client
 
 var OutputFormat string
@@ -28,7 +28,7 @@ var restoreOldBackup bool
 var prometheusURL string
 
 func initConfig() {
-
+	var err error
 	if trace_lvl {
 		log.SetLevel(log.TraceLevel)
 	} else if dbg_lvl {
@@ -42,12 +42,15 @@ func initConfig() {
 	}
 	logFormatter := &log.TextFormatter{TimestampFormat: "02-01-2006 03:04:05 PM", FullTimestamp: true}
 	log.SetFormatter(logFormatter)
-	csConfig = csconfig.NewConfig()
-
+	csConfig, err = csconfig.NewConfig(ConfigFilePath, false, false)
+	if err != nil {
+		log.Fatalf(err.Error())
+	}
 	log.Debugf("Using %s as configuration file", ConfigFilePath)
-	if err := csConfig.LoadConfigurationFile(ConfigFilePath, csConfig.DisableAPI, csConfig.DisableAgent); err != nil {
+	if err := csConfig.LoadCSCLI(); err != nil {
 		log.Fatalf(err.Error())
 	}
+
 	if csConfig.Cscli == nil {
 		log.Fatalf("missing 'cscli' configuration in '%s', exiting", ConfigFilePath)
 	}

cmd/crowdsec-cli/metrics.go

@@ -377,6 +377,9 @@ func NewMetricsCmd() *cobra.Command {
 		Long:  `Fetch metrics from the prometheus server and display them in a human-friendly way`,
 		Args:  cobra.ExactArgs(0),
 		Run: func(cmd *cobra.Command, args []string) {
+			if err := csConfig.LoadPrometheus(); err != nil {
+				log.Fatalf(err.Error())
+			}
 			if !csConfig.Prometheus.Enabled {
 				log.Warningf("Prometheus is not enabled, can't show metrics")
 				os.Exit(1)
@@ -387,7 +390,7 @@ func NewMetricsCmd() *cobra.Command {
 			}
 
 			if prometheusURL == "" {
-				log.Errorf("No prometheus url, please specify in %s or via -u", *csConfig.Self)
+				log.Errorf("No prometheus url, please specify in %s or via -u", *csConfig.FilePath)
 				os.Exit(1)
 			}
 

cmd/crowdsec-cli/parsers.go

@@ -22,13 +22,21 @@ cscli parsers remove crowdsecurity/sshd-logs
 `,
 		Args: cobra.MinimumNArgs(1),
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if csConfig.Cscli == nil {
+			if err := csConfig.LoadHub(); err != nil {
+				log.Fatalf(err.Error())
+			}
+			if csConfig.Hub == nil {
 				return fmt.Errorf("you must configure cli before interacting with hub")
 			}
 
 			if err := setHubBranch(); err != nil {
 				return fmt.Errorf("error while setting hub branch: %s", err)
 			}
+
+			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
+			}
 			return nil
 		},
 		PersistentPostRun: func(cmd *cobra.Command, args []string) {
@@ -46,10 +54,6 @@ cscli parsers remove crowdsecurity/sshd-logs
 		Example: `cscli parsers install crowdsec/xxx crowdsec/xyz`,
 		Args:    cobra.MinimumNArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
-			}
 			for _, name := range args {
 				InstallItem(name, cwhub.PARSERS, forceAction)
 			}
@@ -66,11 +70,6 @@ cscli parsers remove crowdsecurity/sshd-logs
 		Example: `cscli parsers remove crowdsec/xxx crowdsec/xyz`,
 		Args:    cobra.MinimumNArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
-			}
-
 			if all {
 				RemoveMany(cwhub.PARSERS, "")
 			} else {
@@ -91,10 +90,6 @@ cscli parsers remove crowdsecurity/sshd-logs
 		Long:    `Fetch and upgrade given parser(s) from hub`,
 		Example: `cscli parsers upgrade crowdsec/xxx crowdsec/xyz`,
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
-			}
 			if all {
 				UpgradeConfig(cwhub.PARSERS, "", forceAction)
 			} else {
@@ -115,11 +110,6 @@ cscli parsers remove crowdsecurity/sshd-logs
 		Example: `cscli parsers inspect crowdsec/xxx`,
 		Args:    cobra.MinimumNArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
-			}
-
 			InspectItem(args[0], cwhub.PARSERS)
 		},
 	}
@@ -133,10 +123,6 @@ cscli parsers remove crowdsecurity/sshd-logs
 		Example: `cscli parsers list
 cscli parser list crowdsecurity/xxx`,
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
-			}
 			ListItem(cwhub.PARSERS, args)
 		},
 	}

cmd/crowdsec-cli/postoverflows.go

@@ -21,13 +21,21 @@ func NewPostOverflowsCmd() *cobra.Command {
 		cscli postoverflows remove crowdsecurity/cdn-whitelist`,
 		Args: cobra.MinimumNArgs(1),
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if csConfig.Cscli == nil {
+			if err := csConfig.LoadHub(); err != nil {
+				log.Fatalf(err.Error())
+			}
+			if csConfig.Hub == nil {
 				return fmt.Errorf("you must configure cli before interacting with hub")
 			}
 
 			if err := setHubBranch(); err != nil {
 				return fmt.Errorf("error while setting hub branch: %s", err)
 			}
+
+			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
+			}
 			return nil
 		},
 		PersistentPostRun: func(cmd *cobra.Command, args []string) {
@@ -45,10 +53,6 @@ func NewPostOverflowsCmd() *cobra.Command {
 		Example: `cscli postoverflows install crowdsec/xxx crowdsec/xyz`,
 		Args:    cobra.MinimumNArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
-			}
 			for _, name := range args {
 				InstallItem(name, cwhub.PARSERS_OVFLW, forceAction)
 			}
@@ -65,11 +69,6 @@ func NewPostOverflowsCmd() *cobra.Command {
 		Example: `cscli postoverflows remove crowdsec/xxx crowdsec/xyz`,
 		Args:    cobra.MinimumNArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
-			}
-
 			if all {
 				RemoveMany(cwhub.PARSERS_OVFLW, "")
 			} else {
@@ -90,10 +89,6 @@ func NewPostOverflowsCmd() *cobra.Command {
 		Long:    `Fetch and Upgrade given postoverflow(s) from hub`,
 		Example: `cscli postoverflows upgrade crowdsec/xxx crowdsec/xyz`,
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
-			}
 			if all {
 				UpgradeConfig(cwhub.PARSERS_OVFLW, "", forceAction)
 			} else {
@@ -114,10 +109,6 @@ func NewPostOverflowsCmd() *cobra.Command {
 		Example: `cscli postoverflows inspect crowdsec/xxx crowdsec/xyz`,
 		Args:    cobra.MinimumNArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
-			}
 			InspectItem(args[0], cwhub.PARSERS_OVFLW)
 		},
 	}
@@ -130,10 +121,6 @@ func NewPostOverflowsCmd() *cobra.Command {
 		Example: `cscli postoverflows list
 cscli postoverflows list crowdsecurity/xxx`,
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
-			}
 			ListItem(cwhub.PARSERS_OVFLW, args)
 		},
 	}

cmd/crowdsec-cli/scenarios.go

@@ -22,13 +22,21 @@ cscli scenarios remove crowdsecurity/ssh-bf
 `,
 		Args: cobra.MinimumNArgs(1),
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if csConfig.Cscli == nil {
+			if err := csConfig.LoadHub(); err != nil {
+				log.Fatalf(err.Error())
+			}
+			if csConfig.Hub == nil {
 				return fmt.Errorf("you must configure cli before interacting with hub")
 			}
 
 			if err := setHubBranch(); err != nil {
 				return fmt.Errorf("error while setting hub branch: %s", err)
 			}
+
+			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
+			}
 			return nil
 		},
 		PersistentPostRun: func(cmd *cobra.Command, args []string) {
@@ -46,10 +54,6 @@ cscli scenarios remove crowdsecurity/ssh-bf
 		Example: `cscli scenarios install crowdsec/xxx crowdsec/xyz`,
 		Args:    cobra.MinimumNArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
-			}
 			for _, name := range args {
 				InstallItem(name, cwhub.SCENARIOS, forceAction)
 			}
@@ -66,11 +70,6 @@ cscli scenarios remove crowdsecurity/ssh-bf
 		Example: `cscli scenarios remove crowdsec/xxx crowdsec/xyz`,
 		Args:    cobra.MinimumNArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
-			}
-
 			if all {
 				RemoveMany(cwhub.SCENARIOS, "")
 			} else {
@@ -91,10 +90,6 @@ cscli scenarios remove crowdsecurity/ssh-bf
 		Long:    `Fetch and Upgrade given scenario(s) from hub`,
 		Example: `cscli scenarios upgrade crowdsec/xxx crowdsec/xyz`,
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
-			}
 			if all {
 				UpgradeConfig(cwhub.SCENARIOS, "", forceAction)
 			} else {
@@ -115,10 +110,6 @@ cscli scenarios remove crowdsecurity/ssh-bf
 		Example: `cscli scenarios inspect crowdsec/xxx`,
 		Args:    cobra.MinimumNArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
-			}
 			InspectItem(args[0], cwhub.SCENARIOS)
 		},
 	}
@@ -132,10 +123,6 @@ cscli scenarios remove crowdsecurity/ssh-bf
 		Example: `cscli scenarios list
 cscli scenarios list crowdsecurity/xxx`,
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
-			}
 			ListItem(cwhub.SCENARIOS, args)
 		},
 	}

cmd/crowdsec-cli/simulation.go

@@ -11,25 +11,25 @@ import (
 )
 
 func addToExclusion(name string) error {
-	csConfig.Crowdsec.SimulationConfig.Exclusions = append(csConfig.Crowdsec.SimulationConfig.Exclusions, name)
+	csConfig.Cscli.SimulationConfig.Exclusions = append(csConfig.Cscli.SimulationConfig.Exclusions, name)
 	return nil
 }
 
 func removeFromExclusion(name string) error {
-	index := indexOf(name, csConfig.Crowdsec.SimulationConfig.Exclusions)
+	index := indexOf(name, csConfig.Cscli.SimulationConfig.Exclusions)
 
 	// Remove element from the slice
-	csConfig.Crowdsec.SimulationConfig.Exclusions[index] = csConfig.Crowdsec.SimulationConfig.Exclusions[len(csConfig.Crowdsec.SimulationConfig.Exclusions)-1]
-	csConfig.Crowdsec.SimulationConfig.Exclusions[len(csConfig.Crowdsec.SimulationConfig.Exclusions)-1] = ""
-	csConfig.Crowdsec.SimulationConfig.Exclusions = csConfig.Crowdsec.SimulationConfig.Exclusions[:len(csConfig.Crowdsec.SimulationConfig.Exclusions)-1]
+	csConfig.Cscli.SimulationConfig.Exclusions[index] = csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1]
+	csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1] = ""
+	csConfig.Cscli.SimulationConfig.Exclusions = csConfig.Cscli.SimulationConfig.Exclusions[:len(csConfig.Cscli.SimulationConfig.Exclusions)-1]
 
 	return nil
 }
 
 func enableGlobalSimulation() error {
-	csConfig.Crowdsec.SimulationConfig.Simulation = new(bool)
-	*csConfig.Crowdsec.SimulationConfig.Simulation = true
-	csConfig.Crowdsec.SimulationConfig.Exclusions = []string{}
+	csConfig.Cscli.SimulationConfig.Simulation = new(bool)
+	*csConfig.Cscli.SimulationConfig.Simulation = true
+	csConfig.Cscli.SimulationConfig.Exclusions = []string{}
 
 	if err := dumpSimulationFile(); err != nil {
 		log.Fatalf("unable to dump simulation file: %s", err.Error())
@@ -41,7 +41,7 @@ func enableGlobalSimulation() error {
 }
 
 func dumpSimulationFile() error {
-	newConfigSim, err := yaml.Marshal(csConfig.Crowdsec.SimulationConfig)
+	newConfigSim, err := yaml.Marshal(csConfig.Cscli.SimulationConfig)
 	if err != nil {
 		return fmt.Errorf("unable to marshal simulation configuration: %s", err)
 	}
@@ -55,11 +55,11 @@ func dumpSimulationFile() error {
 }
 
 func disableGlobalSimulation() error {
-	csConfig.Crowdsec.SimulationConfig.Simulation = new(bool)
-	*csConfig.Crowdsec.SimulationConfig.Simulation = false
+	csConfig.Cscli.SimulationConfig.Simulation = new(bool)
+	*csConfig.Cscli.SimulationConfig.Simulation = false
 
-	csConfig.Crowdsec.SimulationConfig.Exclusions = []string{}
-	newConfigSim, err := yaml.Marshal(csConfig.Crowdsec.SimulationConfig)
+	csConfig.Cscli.SimulationConfig.Exclusions = []string{}
+	newConfigSim, err := yaml.Marshal(csConfig.Cscli.SimulationConfig)
 	if err != nil {
 		return fmt.Errorf("unable to marshal new simulation configuration: %s", err)
 	}
@@ -73,23 +73,23 @@ func disableGlobalSimulation() error {
 }
 
 func simulationStatus() error {
-	if csConfig.Crowdsec.SimulationConfig == nil {
+	if csConfig.Cscli.SimulationConfig == nil {
 		log.Printf("global simulation: disabled (configuration file is missing)")
 		return nil
 	}
-	if *csConfig.Crowdsec.SimulationConfig.Simulation {
+	if *csConfig.Cscli.SimulationConfig.Simulation {
 		log.Println("global simulation: enabled")
-		if len(csConfig.Crowdsec.SimulationConfig.Exclusions) > 0 {
+		if len(csConfig.Cscli.SimulationConfig.Exclusions) > 0 {
 			log.Println("Scenarios not in simulation mode :")
-			for _, scenario := range csConfig.Crowdsec.SimulationConfig.Exclusions {
+			for _, scenario := range csConfig.Cscli.SimulationConfig.Exclusions {
 				log.Printf("  - %s", scenario)
 			}
 		}
 	} else {
 		log.Println("global simulation: disabled")
-		if len(csConfig.Crowdsec.SimulationConfig.Exclusions) > 0 {
+		if len(csConfig.Cscli.SimulationConfig.Exclusions) > 0 {
 			log.Println("Scenarios in simulation mode :")
-			for _, scenario := range csConfig.Crowdsec.SimulationConfig.Exclusions {
+			for _, scenario := range csConfig.Cscli.SimulationConfig.Exclusions {
 				log.Printf("  - %s", scenario)
 			}
 		}
@@ -105,9 +105,15 @@ func NewSimulationCmds() *cobra.Command {
 cscli simulation enable crowdsecurity/ssh-bf
 cscli simulation disable crowdsecurity/ssh-bf`,
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if err := csConfig.LoadSimulation(); err != nil {
+				log.Fatalf(err.Error())
+			}
 			if csConfig.Cscli == nil {
 				return fmt.Errorf("you must configure cli before using simulation")
 			}
+			if csConfig.Cscli.SimulationConfig == nil {
+				return fmt.Errorf("no simulation configured")
+			}
 			return nil
 		},
 		PersistentPostRun: func(cmd *cobra.Command, args []string) {
@@ -125,7 +131,10 @@ cscli simulation disable crowdsecurity/ssh-bf`,
 		Short:   "Enable the simulation, globally or on specified scenarios",
 		Example: `cscli simulation enable`,
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+			if err := csConfig.LoadHub(); err != nil {
+				log.Fatalf(err.Error())
+			}
+			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
 				log.Fatalf("Failed to get Hub index : %v", err)
 				log.Infoln("Run 'sudo cscli hub update' to get the hub index")
 			}
@@ -143,16 +152,16 @@ cscli simulation disable crowdsecurity/ssh-bf`,
 					if !item.Installed {
 						log.Warningf("'%s' isn't enabled", scenario)
 					}
-					isExcluded := inSlice(scenario, csConfig.Crowdsec.SimulationConfig.Exclusions)
-					if *csConfig.Crowdsec.SimulationConfig.Simulation && !isExcluded {
+					isExcluded := inSlice(scenario, csConfig.Cscli.SimulationConfig.Exclusions)
+					if *csConfig.Cscli.SimulationConfig.Simulation && !isExcluded {
 						log.Warningf("global simulation is already enabled")
 						continue
 					}
-					if !*csConfig.Crowdsec.SimulationConfig.Simulation && isExcluded {
+					if !*csConfig.Cscli.SimulationConfig.Simulation && isExcluded {
 						log.Warningf("simulation for '%s' already enabled", scenario)
 						continue
 					}
-					if *csConfig.Crowdsec.SimulationConfig.Simulation && isExcluded {
+					if *csConfig.Cscli.SimulationConfig.Simulation && isExcluded {
 						if err := removeFromExclusion(scenario); err != nil {
 							log.Fatalf(err.Error())
 						}
@@ -186,12 +195,12 @@ cscli simulation disable crowdsecurity/ssh-bf`,
 		Run: func(cmd *cobra.Command, args []string) {
 			if len(args) > 0 {
 				for _, scenario := range args {
-					isExcluded := inSlice(scenario, csConfig.Crowdsec.SimulationConfig.Exclusions)
-					if !*csConfig.Crowdsec.SimulationConfig.Simulation && !isExcluded {
+					isExcluded := inSlice(scenario, csConfig.Cscli.SimulationConfig.Exclusions)
+					if !*csConfig.Cscli.SimulationConfig.Simulation && !isExcluded {
 						log.Warningf("%s isn't in simulation mode", scenario)
 						continue
 					}
-					if !*csConfig.Crowdsec.SimulationConfig.Simulation && isExcluded {
+					if !*csConfig.Cscli.SimulationConfig.Simulation && isExcluded {
 						if err := removeFromExclusion(scenario); err != nil {
 							log.Fatalf(err.Error())
 						}

cmd/crowdsec-cli/utils.go

@@ -141,16 +141,16 @@ func InstallItem(name string, obtype string, force bool) {
 			return
 		}
 	}
-	item, err := cwhub.DownloadLatest(csConfig.Cscli, item, force)
+	item, err := cwhub.DownloadLatest(csConfig.Hub, item, force)
 	if err != nil {
 		log.Fatalf("error while downloading %s : %v", item.Name, err)
 	}
 	cwhub.AddItem(obtype, item)
 	if downloadOnly {
-		log.Infof("Downloaded %s to %s", item.Name, csConfig.Cscli.HubDir+"/"+item.RemotePath)
+		log.Infof("Downloaded %s to %s", item.Name, csConfig.Hub.HubDir+"/"+item.RemotePath)
 		return
 	}
-	item, err = cwhub.EnableItem(csConfig.Cscli, item)
+	item, err = cwhub.EnableItem(csConfig.Hub, item)
 	if err != nil {
 		log.Fatalf("error while enabled %s : %v.", item.Name, err)
 	}
@@ -168,7 +168,7 @@ func RemoveMany(itemType string, name string) {
 			log.Fatalf("unable to retrieve: %s", name)
 		}
 		item := *it
-		item, err = cwhub.DisableItem(csConfig.Cscli, item, purge, forceAction)
+		item, err = cwhub.DisableItem(csConfig.Hub, item, purge, forceAction)
 		if err != nil {
 			log.Fatalf("unable to disable %s : %v", item.Name, err)
 		}
@@ -176,7 +176,7 @@ func RemoveMany(itemType string, name string) {
 		return
 	} else if name == "" && all {
 		for _, v := range cwhub.GetItemMap(itemType) {
-			v, err = cwhub.DisableItem(csConfig.Cscli, v, purge, forceAction)
+			v, err = cwhub.DisableItem(csConfig.Hub, v, purge, forceAction)
 			if err != nil {
 				log.Fatalf("unable to disable %s : %v", v.Name, err)
 			}
@@ -219,7 +219,7 @@ func UpgradeConfig(itemType string, name string, force bool) {
 				continue
 			}
 		}
-		v, err = cwhub.DownloadLatest(csConfig.Cscli, v, force)
+		v, err = cwhub.DownloadLatest(csConfig.Hub, v, force)
 		if err != nil {
 			log.Fatalf("%s : download failed : %v", v.Name, err)
 		}
@@ -264,7 +264,7 @@ func InspectItem(name string, objecitemType string) {
 	fmt.Printf("%s", string(buff))
 	if csConfig.Prometheus.Enabled {
 		if csConfig.Prometheus.ListenAddr == "" || csConfig.Prometheus.ListenPort == 0 {
-			log.Warningf("No prometheus address or port specified in '%s', can't show metrics", *csConfig.Self)
+			log.Warningf("No prometheus address or port specified in '%s', can't show metrics", *csConfig.FilePath)
 			return
 		}
 		if prometheusURL == "" {
@@ -500,7 +500,7 @@ func silenceInstallItem(name string, obtype string) (string, error) {
 	if downloadOnly && it.Downloaded && it.UpToDate {
 		return fmt.Sprintf("%s is already downloaded and up-to-date", it.Name), nil
 	}
-	it, err := cwhub.DownloadLatest(csConfig.Cscli, it, forceAction)
+	it, err := cwhub.DownloadLatest(csConfig.Hub, it, forceAction)
 	if err != nil {
 		return "", fmt.Errorf("error while downloading %s : %v", it.Name, err)
 	}
@@ -511,7 +511,7 @@ func silenceInstallItem(name string, obtype string) (string, error) {
 	if downloadOnly {
 		return fmt.Sprintf("Downloaded %s to %s", it.Name, csConfig.Cscli.HubDir+"/"+it.RemotePath), nil
 	}
-	it, err = cwhub.EnableItem(csConfig.Cscli, it)
+	it, err = cwhub.EnableItem(csConfig.Hub, it)
 	if err != nil {
 		return "", fmt.Errorf("error while enabled %s : %v", it.Name, err)
 	}

cmd/crowdsec/api.go

@@ -9,7 +9,7 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-func initAPIServer(cConfig *csconfig.GlobalConfig) (*apiserver.APIServer, error) {
+func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) {
 	apiServer, err := apiserver.NewServer(cConfig.API.Server)
 	if err != nil {
 		return nil, fmt.Errorf("unable to run local API: %s", err)

cmd/crowdsec/crowdsec.go

@@ -14,14 +14,14 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-func initCrowdsec(cConfig *csconfig.GlobalConfig) (*parser.Parsers, error) {
+func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
 	err := exprhelpers.Init()
 	if err != nil {
 		return &parser.Parsers{}, fmt.Errorf("Failed to init expr helpers : %s", err)
 	}
 
 	// Populate cwhub package tools
-	if err := cwhub.GetHubIdx(cConfig.Cscli); err != nil {
+	if err := cwhub.GetHubIdx(cConfig.Hub); err != nil {
 		return &parser.Parsers{}, fmt.Errorf("Failed to load hub index : %s", err)
 	}
 
@@ -41,7 +41,7 @@ func initCrowdsec(cConfig *csconfig.GlobalConfig) (*parser.Parsers, error) {
 	return csParsers, nil
 }
 
-func runCrowdsec(cConfig *csconfig.GlobalConfig, parsers *parser.Parsers) error {
+func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
 	inputLineChan := make(chan types.Event)
 	inputEventChan := make(chan types.Event)
 
@@ -117,7 +117,7 @@ func runCrowdsec(cConfig *csconfig.GlobalConfig, parsers *parser.Parsers) error
 	return nil
 }
 
-func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.GlobalConfig) {
+func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config) {
 	crowdsecTomb.Go(func() error {
 		defer types.CatchPanic("crowdsec/serveCrowdsec")
 		go func() {

cmd/crowdsec/main.go

@@ -33,9 +33,6 @@ var (
 	apiTomb      tomb.Tomb
 	crowdsecTomb tomb.Tomb
 
-	disableAPI   bool
-	disableAgent bool
-
 	flags *Flags
 
 	/*the state of acquisition*/
@@ -112,7 +109,7 @@ func newParsers() *parser.Parsers {
 	return parsers
 }
 
-func LoadBuckets(cConfig *csconfig.GlobalConfig) error {
+func LoadBuckets(cConfig *csconfig.Config) error {
 
 	var (
 		err   error
@@ -140,7 +137,7 @@ func LoadBuckets(cConfig *csconfig.GlobalConfig) error {
 	return nil
 }
 
-func LoadAcquisition(cConfig *csconfig.GlobalConfig) error {
+func LoadAcquisition(cConfig *csconfig.Config) error {
 	var err error
 
 	if flags.SingleFilePath != "" || flags.SingleJournalctlFilter != "" {
@@ -191,31 +188,25 @@ func (f *Flags) Parse() {
 }
 
 // LoadConfig return configuration parsed from configuration file
-func LoadConfig(cConfig *csconfig.GlobalConfig) error {
-	disableAPI = flags.DisableAPI
-	disableAgent = flags.DisableAgent
-	if flags.ConfigFile != "" {
-		if err := cConfig.LoadConfigurationFile(flags.ConfigFile, disableAPI, disableAgent); err != nil {
-			return fmt.Errorf("while loading configuration : %s", err)
+func LoadConfig(cConfig *csconfig.Config) error {
+
+	if !flags.DisableAgent {
+		if err := cConfig.LoadCrowdsec(); err != nil {
+			return err
 		}
-	} else {
-		log.Warningf("no configuration file provided")
-	}
-	if !disableAPI && (cConfig.API == nil || cConfig.API.Server == nil) {
-		log.Errorf("no API server configuration found, will not start the local API")
-		disableAPI = true
 	}
 
-	if !disableAgent && cConfig.Crowdsec == nil {
-		log.Errorf("no configuration found crowdsec agent, will not start the agent")
-		disableAgent = true
+	if !flags.DisableAPI {
+		if err := cConfig.LoadAPIServer(); err != nil {
+			return err
+		}
 	}
 
-	if !disableAgent && (cConfig.API == nil || cConfig.API.Client == nil || cConfig.API.Client.Credentials == nil) {
+	if !cConfig.DisableAgent && (cConfig.API == nil || cConfig.API.Client == nil || cConfig.API.Client.Credentials == nil) {
 		log.Fatalf("missing local API credentials for crowdsec agent, abort")
 	}
 
-	if disableAPI && disableAgent {
+	if cConfig.DisableAPI && cConfig.DisableAgent {
 		log.Fatalf("You must run at least the API Server or crowdsec")
 	}
 
@@ -244,14 +235,14 @@ func LoadConfig(cConfig *csconfig.GlobalConfig) error {
 		cConfig.Common.LogLevel = &logLevel
 	}
 
-	if flags.TestMode && !disableAgent {
+	if flags.TestMode && !cConfig.DisableAgent {
 		cConfig.Crowdsec.LintOnly = true
 	}
 
 	if flags.SingleFilePath != "" || flags.SingleJournalctlFilter != "" {
 		cConfig.API.Server.OnlineClient = nil
 		/*if the api is disabled as well, just read file and exit, don't daemonize*/
-		if disableAPI {
+		if flags.DisableAPI {
 			cConfig.Common.Daemonize = false
 		}
 		cConfig.Common.LogMedia = "stdout"
@@ -263,13 +254,12 @@ func LoadConfig(cConfig *csconfig.GlobalConfig) error {
 
 func main() {
 	var (
-		cConfig *csconfig.GlobalConfig
+		cConfig *csconfig.Config
 		err     error
 	)
 
 	defer types.CatchPanic("crowdsec/main")
 
-	cConfig = csconfig.NewConfig()
 	// Handle command line arguments
 	flags = &Flags{}
 	flags.Parse()
@@ -278,6 +268,10 @@ func main() {
 		os.Exit(0)
 	}
 
+	cConfig, err = csconfig.NewConfig(flags.ConfigFile, flags.DisableAgent, flags.DisableAPI)
+	if err != nil {
+		log.Fatalf(err.Error())
+	}
 	if err := LoadConfig(cConfig); err != nil {
 		log.Fatalf(err.Error())
 	}
@@ -288,19 +282,6 @@ func main() {
 
 	log.Infof("Crowdsec %s", cwversion.VersionStr())
 
-	if !flags.DisableAPI && (cConfig.API == nil || cConfig.API.Server == nil) {
-		log.Errorf("no API server configuration found, will not start the local API")
-		flags.DisableAPI = true
-	}
-
-	if !flags.DisableAgent && cConfig.Crowdsec == nil {
-		log.Errorf("no configuration found crowdsec agent, will not start the agent")
-		flags.DisableAgent = true
-	}
-
-	if !flags.DisableAgent && (cConfig.API == nil || cConfig.API.Client == nil || cConfig.API.Client.Credentials == nil) {
-		log.Fatalf("missing local API credentials for crowdsec agent, abort")
-	}
 	// Enable profiling early
 	if cConfig.Prometheus != nil {
 		go registerPrometheus(cConfig.Prometheus)

cmd/crowdsec/pour.go

@@ -10,7 +10,7 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-func runPour(input chan types.Event, holders []leaky.BucketFactory, buckets *leaky.Buckets, cConfig *csconfig.GlobalConfig) error {
+func runPour(input chan types.Event, holders []leaky.BucketFactory, buckets *leaky.Buckets, cConfig *csconfig.Config) error {
 	var (
 		count int
 	)

cmd/crowdsec/serve.go

@@ -19,7 +19,7 @@ import (
 )
 
 //debugHandler is kept as a dev convenience : it shuts down and serialize internal state
-func debugHandler(sig os.Signal, cConfig *csconfig.GlobalConfig) error {
+func debugHandler(sig os.Signal, cConfig *csconfig.Config) error {
 	var tmpFile string
 	var err error
 	//stop go routines
@@ -37,12 +37,12 @@ func debugHandler(sig os.Signal, cConfig *csconfig.GlobalConfig) error {
 	return nil
 }
 
-func reloadHandler(sig os.Signal, cConfig *csconfig.GlobalConfig) error {
+func reloadHandler(sig os.Signal, cConfig *csconfig.Config) error {
 	var tmpFile string
 	var err error
 
 	//stop go routines
-	if !disableAgent {
+	if !cConfig.DisableAgent {
 		if err := shutdownCrowdsec(); err != nil {
 			log.Fatalf("Failed to shut down crowdsec routines: %s", err)
 		}
@@ -57,7 +57,7 @@ func reloadHandler(sig os.Signal, cConfig *csconfig.GlobalConfig) error {
 		}
 	}
 
-	if !disableAPI {
+	if !cConfig.DisableAPI {
 		if err := shutdownAPI(); err != nil {
 			log.Fatalf("Failed to shut down api routines: %s", err)
 		}
@@ -81,7 +81,7 @@ func reloadHandler(sig os.Signal, cConfig *csconfig.GlobalConfig) error {
 		log.Fatal(err.Error())
 	}
 
-	if !disableAPI {
+	if !cConfig.DisableAPI {
 		apiServer, err := initAPIServer(cConfig)
 		if err != nil {
 			return fmt.Errorf("unable to init api server: %s", err)
@@ -90,7 +90,7 @@ func reloadHandler(sig os.Signal, cConfig *csconfig.GlobalConfig) error {
 		serveAPIServer(apiServer)
 	}
 
-	if !disableAgent {
+	if !cConfig.DisableAgent {
 		csParsers, err := initCrowdsec(cConfig)
 		if err != nil {
 			return fmt.Errorf("unable to init crowdsec: %s", err)
@@ -186,7 +186,7 @@ func termHandler(sig os.Signal) error {
 	return nil
 }
 
-func HandleSignals(cConfig *csconfig.GlobalConfig) {
+func HandleSignals(cConfig *csconfig.Config) {
 	signalChan := make(chan os.Signal, 1)
 	signal.Notify(signalChan,
 		syscall.SIGHUP,
@@ -220,7 +220,7 @@ func HandleSignals(cConfig *csconfig.GlobalConfig) {
 	os.Exit(code)
 }
 
-func Serve(cConfig *csconfig.GlobalConfig) error {
+func Serve(cConfig *csconfig.Config) error {
 	acquisTomb = tomb.Tomb{}
 	parsersTomb = tomb.Tomb{}
 	bucketsTomb = tomb.Tomb{}
@@ -228,7 +228,7 @@ func Serve(cConfig *csconfig.GlobalConfig) error {
 	apiTomb = tomb.Tomb{}
 	crowdsecTomb = tomb.Tomb{}
 
-	if !disableAPI {
+	if !cConfig.DisableAPI {
 		apiServer, err := initAPIServer(cConfig)
 		if err != nil {
 			return errors.Wrap(err, "api server init")
@@ -238,7 +238,7 @@ func Serve(cConfig *csconfig.GlobalConfig) error {
 		}
 	}
 
-	if !disableAgent {
+	if !cConfig.DisableAgent {
 		csParsers, err := initCrowdsec(cConfig)
 		if err != nil {
 			return errors.Wrap(err, "crowdsec init")

pkg/apiserver/apiserver_test.go

@@ -31,8 +31,8 @@ var MachineTest = models.WatcherAuthRequest{
 
 var UserAgent = fmt.Sprintf("crowdsec-test/%s", cwversion.Version)
 
-func LoadTestConfig() csconfig.GlobalConfig {
-	config := csconfig.GlobalConfig{}
+func LoadTestConfig() csconfig.Config {
+	config := csconfig.Config{}
 	maxAge := "1h"
 	flushConfig := csconfig.FlushDBCfg{
 		MaxAge: &maxAge,
@@ -57,8 +57,8 @@ func LoadTestConfig() csconfig.GlobalConfig {
 	return config
 }
 
-func LoadTestConfigForwardedFor() csconfig.GlobalConfig {
-	config := csconfig.GlobalConfig{}
+func LoadTestConfigForwardedFor() csconfig.Config {
+	config := csconfig.Config{}
 	maxAge := "1h"
 	flushConfig := csconfig.FlushDBCfg{
 		MaxAge: &maxAge,

pkg/csconfig/api.go

@@ -1,6 +1,15 @@
 package csconfig
 
-import log "github.com/sirupsen/logrus"
+import (
+	"fmt"
+	"io/ioutil"
+	"strings"
+
+	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
+)
 
 type APICfg struct {
 	Client *LocalApiClientCfg `yaml:"client"`
@@ -26,6 +35,47 @@ type LocalApiClientCfg struct {
 	InsecureSkipVerify  *bool              `yaml:"insecure_skip_verify"` // check if api certificate is bad or not
 }
 
+func (o *OnlineApiClientCfg) Load() error {
+	o.Credentials = new(ApiCredentialsCfg)
+	fcontent, err := ioutil.ReadFile(o.CredentialsFilePath)
+	if err != nil {
+		return errors.Wrapf(err, "failed to read api server credentials configuration file '%s'", o.CredentialsFilePath)
+	}
+	err = yaml.UnmarshalStrict(fcontent, o.Credentials)
+	if err != nil {
+		return errors.Wrapf(err, "failed unmarshaling api server credentials configuration file '%s'", o.CredentialsFilePath)
+	}
+	if o.Credentials.Login == "" || o.Credentials.Password == "" || o.Credentials.URL == "" {
+		log.Warningf("can't load CAPI credentials from '%s' (missing field)", o.CredentialsFilePath)
+		o.Credentials = nil
+	}
+	return nil
+}
+
+func (l *LocalApiClientCfg) Load() error {
+	fcontent, err := ioutil.ReadFile(l.CredentialsFilePath)
+	if err != nil {
+		return errors.Wrapf(err, "failed to read api client credential configuration file '%s'", l.CredentialsFilePath)
+	}
+	err = yaml.UnmarshalStrict(fcontent, &l.Credentials)
+	if err != nil {
+		return errors.Wrapf(err, "failed unmarshaling api client credential configuration file '%s'", l.CredentialsFilePath)
+	}
+	if l.Credentials != nil && l.Credentials.URL != "" {
+		if !strings.HasSuffix(l.Credentials.URL, "/") {
+			l.Credentials.URL = l.Credentials.URL + "/"
+		}
+	} else {
+		log.Warningf("no credentials or URL found in api client configuration '%s'", l.CredentialsFilePath)
+	}
+	if l.InsecureSkipVerify == nil {
+		apiclient.InsecureSkipVerify = false
+	} else {
+		apiclient.InsecureSkipVerify = *l.InsecureSkipVerify
+	}
+	return nil
+}
+
 /*local api service configuration*/
 type LocalApiServerCfg struct {
 	ListenURI              string              `yaml:"listen_uri,omitempty"` //127.0.0.1:8080
@@ -44,3 +94,44 @@ type TLSCfg struct {
 	CertFilePath string `yaml:"cert_file"`
 	KeyFilePath  string `yaml:"key_file"`
 }
+
+func (c *Config) LoadAPIServer() error {
+	if c.API.Server != nil && !c.DisableAPI {
+		if err := c.LoadCommon(); err != nil {
+			return fmt.Errorf("loading common configuration: %s", err.Error())
+		}
+		c.API.Server.LogDir = c.Common.LogDir
+		c.API.Server.LogMedia = c.Common.LogMedia
+		if err := c.API.Server.LoadProfiles(); err != nil {
+			return errors.Wrap(err, "while loading profiles for LAPI")
+		}
+		if c.API.Server.OnlineClient != nil && c.API.Server.OnlineClient.CredentialsFilePath != "" {
+			if err := c.API.Server.OnlineClient.Load(); err != nil {
+				return errors.Wrap(err, "loading online client credentials")
+			}
+		}
+		if c.API.Server.OnlineClient == nil || c.API.Server.OnlineClient.Credentials == nil {
+			log.Printf("push and pull to crowdsec API disabled")
+		}
+		if err := c.LoadDBConfig(); err != nil {
+			return err
+		}
+	} else {
+		log.Warningf("crowdsec local API is disabled")
+		c.DisableAPI = true
+	}
+
+	return nil
+}
+
+func (c *Config) LoadAPIClient() error {
+	if c.API != nil && c.API.Client != nil && c.API.Client.CredentialsFilePath != "" && !c.DisableAgent {
+		if err := c.API.Client.Load(); err != nil {
+			return err
+		}
+	} else {
+		return fmt.Errorf("no API client section in configuration")
+	}
+
+	return nil
+}

pkg/csconfig/api_test.go

@@ -0,0 +1,268 @@
+package csconfig
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"gopkg.in/yaml.v2"
+)
+
+func TestLoadLocalApiClientCfg(t *testing.T) {
+	True := true
+	tests := []struct {
+		name           string
+		Input          *LocalApiClientCfg
+		expectedResult *ApiCredentialsCfg
+		err            string
+	}{
+		{
+			name: "basic valid configuration",
+			Input: &LocalApiClientCfg{
+				CredentialsFilePath: "./tests/lapi-secrets.yaml",
+			},
+			expectedResult: &ApiCredentialsCfg{
+				URL:      "http://localhost:8080/",
+				Login:    "test",
+				Password: "testpassword",
+			},
+		},
+		{
+			name: "invalid configuration",
+			Input: &LocalApiClientCfg{
+				CredentialsFilePath: "./tests/bad_lapi-secrets.yaml",
+			},
+			expectedResult: &ApiCredentialsCfg{},
+		},
+		{
+			name: "invalid configuration filepath",
+			Input: &LocalApiClientCfg{
+				CredentialsFilePath: "./tests/nonexist_lapi-secrets.yaml",
+			},
+			expectedResult: nil,
+		},
+		{
+			name: "valid configuration with insecure skip verify",
+			Input: &LocalApiClientCfg{
+				CredentialsFilePath: "./tests/lapi-secrets.yaml",
+				InsecureSkipVerify:  &True,
+			},
+			expectedResult: &ApiCredentialsCfg{
+				URL:      "http://localhost:8080/",
+				Login:    "test",
+				Password: "testpassword",
+			},
+		},
+	}
+
+	for idx, test := range tests {
+		fmt.Printf("TEST '%s'\n", test.name)
+		err := test.Input.Load()
+		if err == nil && test.err != "" {
+			t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
+		} else if test.err != "" {
+			if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
+				t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
+					test.err,
+					fmt.Sprintf("%s", err))
+			}
+		}
+
+		isOk := assert.Equal(t, test.expectedResult, test.Input.Credentials)
+		if !isOk {
+			t.Fatalf("test '%s' failed", test.name)
+		}
+
+	}
+}
+
+func TestLoadOnlineApiClientCfg(t *testing.T) {
+	tests := []struct {
+		name           string
+		Input          *OnlineApiClientCfg
+		expectedResult *ApiCredentialsCfg
+		err            string
+	}{
+		{
+			name: "basic valid configuration",
+			Input: &OnlineApiClientCfg{
+				CredentialsFilePath: "./tests/online-api-secrets.yaml",
+			},
+			expectedResult: &ApiCredentialsCfg{
+				URL:      "http://crowdsec.api",
+				Login:    "test",
+				Password: "testpassword",
+			},
+		},
+		{
+			name: "invalid configuration",
+			Input: &OnlineApiClientCfg{
+				CredentialsFilePath: "./tests/bad_lapi-secrets.yaml",
+			},
+			expectedResult: &ApiCredentialsCfg{},
+			err:            "failed unmarshaling api server credentials",
+		},
+		{
+			name: "missing field configuration",
+			Input: &OnlineApiClientCfg{
+				CredentialsFilePath: "./tests/bad_online-api-secrets.yaml",
+			},
+			expectedResult: nil,
+		},
+		{
+			name: "invalid configuration filepath",
+			Input: &OnlineApiClientCfg{
+				CredentialsFilePath: "./tests/nonexist_online-api-secrets.yaml",
+			},
+			expectedResult: &ApiCredentialsCfg{},
+			err:            "failed to read api server credentials",
+		},
+	}
+
+	for idx, test := range tests {
+		err := test.Input.Load()
+		if err == nil && test.err != "" {
+			fmt.Printf("TEST '%s': NOK\n", test.name)
+			t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
+		} else if test.err != "" {
+			if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
+				fmt.Printf("TEST '%s': NOK\n", test.name)
+				t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
+					test.err,
+					fmt.Sprintf("%s", err))
+			}
+		}
+
+		isOk := assert.Equal(t, test.expectedResult, test.Input.Credentials)
+		if !isOk {
+			t.Fatalf("TEST '%s': NOK", test.name)
+		} else {
+			fmt.Printf("TEST '%s': OK\n", test.name)
+		}
+
+	}
+}
+
+func TestLoadAPIServer(t *testing.T) {
+	tmpLAPI := &LocalApiServerCfg{
+		ProfilesPath: "./tests/profiles.yaml",
+	}
+	if err := tmpLAPI.LoadProfiles(); err != nil {
+		t.Fatalf("loading tmp profiles: %+v", err)
+	}
+
+	LogDirFullPath, err := filepath.Abs("./tests")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	config := &Config{}
+	fcontent, err := ioutil.ReadFile("./tests/config.yaml")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+	configData := os.ExpandEnv(string(fcontent))
+	err = yaml.UnmarshalStrict([]byte(configData), &config)
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+	tests := []struct {
+		name           string
+		Input          *Config
+		expectedResult *LocalApiServerCfg
+		err            string
+	}{
+		{
+			name: "basic valid configuration",
+			Input: &Config{
+				Self: []byte(configData),
+				API: &APICfg{
+					Server: &LocalApiServerCfg{
+						ListenURI: "http://crowdsec.api",
+						OnlineClient: &OnlineApiClientCfg{
+							CredentialsFilePath: "./tests/online-api-secrets.yaml",
+						},
+						ProfilesPath: "./tests/profiles.yaml",
+					},
+				},
+				DbConfig: &DatabaseCfg{
+					Type:   "sqlite",
+					DbPath: "./tests/test.db",
+				},
+				Common: &CommonCfg{
+					LogDir:   "./tests/",
+					LogMedia: "stdout",
+				},
+				DisableAPI: false,
+			},
+			expectedResult: &LocalApiServerCfg{
+				ListenURI: "http://crowdsec.api",
+				TLS:       nil,
+				DbConfig: &DatabaseCfg{
+					DbPath: "./tests/test.db",
+					Type:   "sqlite",
+				},
+				LogDir:   LogDirFullPath,
+				LogMedia: "stdout",
+				OnlineClient: &OnlineApiClientCfg{
+					CredentialsFilePath: "./tests/online-api-secrets.yaml",
+					Credentials: &ApiCredentialsCfg{
+						URL:      "http://crowdsec.api",
+						Login:    "test",
+						Password: "testpassword",
+					},
+				},
+				Profiles:               tmpLAPI.Profiles,
+				ProfilesPath:           "./tests/profiles.yaml",
+				UseForwardedForHeaders: false,
+			},
+			err: "",
+		},
+		{
+			name: "basic valid configuration",
+			Input: &Config{
+				Self: []byte(configData),
+				API: &APICfg{
+					Server: &LocalApiServerCfg{},
+				},
+				Common: &CommonCfg{
+					LogDir:   "./tests/",
+					LogMedia: "stdout",
+				},
+				DisableAPI: false,
+			},
+			expectedResult: &LocalApiServerCfg{
+				LogDir:   LogDirFullPath,
+				LogMedia: "stdout",
+			},
+			err: "while loading profiles for LAPI",
+		},
+	}
+
+	for idx, test := range tests {
+		err := test.Input.LoadAPIServer()
+		if err == nil && test.err != "" {
+			fmt.Printf("TEST '%s': NOK\n", test.name)
+			t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
+		} else if test.err != "" {
+			if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
+				fmt.Printf("TEST '%s': NOK\n", test.name)
+				t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
+					test.err,
+					fmt.Sprintf("%s", err))
+			}
+		}
+
+		isOk := assert.Equal(t, test.expectedResult, test.Input.API.Server)
+		if !isOk {
+			t.Fatalf("TEST '%s': NOK", test.name)
+		} else {
+			fmt.Printf("TEST '%s': OK\n", test.name)
+		}
+
+	}
+}

pkg/csconfig/common.go

@@ -1,6 +1,12 @@
 package csconfig
 
-import log "github.com/sirupsen/logrus"
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+)
 
 /*daemonization/service related stuff*/
 type CommonCfg struct {
@@ -11,3 +17,27 @@ type CommonCfg struct {
 	LogLevel   *log.Level `yaml:"log_level"`
 	WorkingDir string     `yaml:"working_dir,omitempty"` ///var/run
 }
+
+func (c *Config) LoadCommon() error {
+	var err error
+	if c.Common == nil {
+		return fmt.Errorf("no common block provided in configuration file")
+	}
+
+	var CommonCleanup = []*string{
+		&c.Common.PidDir,
+		&c.Common.LogDir,
+		&c.Common.WorkingDir,
+	}
+	for _, k := range CommonCleanup {
+		if *k == "" {
+			continue
+		}
+		*k, err = filepath.Abs(*k)
+		if err != nil {
+			return errors.Wrapf(err, "failed to get absolute path of '%s'", *k)
+		}
+	}
+
+	return nil
+}

pkg/csconfig/common_test.go

@@ -0,0 +1,98 @@
+package csconfig
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLoadCommon(t *testing.T) {
+	PidDirFullPath, err := filepath.Abs("./tests/")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	LogDirFullPath, err := filepath.Abs("./tests/log/")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	WorkingDirFullPath, err := filepath.Abs("./tests")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	tests := []struct {
+		name           string
+		Input          *Config
+		expectedResult *CommonCfg
+		err            string
+	}{
+		{
+			name: "basic valid configuration",
+			Input: &Config{
+				Common: &CommonCfg{
+					Daemonize:  true,
+					PidDir:     "./tests",
+					LogMedia:   "file",
+					LogDir:     "./tests/log/",
+					WorkingDir: "./tests/",
+				},
+			},
+			expectedResult: &CommonCfg{
+				Daemonize:  true,
+				PidDir:     PidDirFullPath,
+				LogMedia:   "file",
+				LogDir:     LogDirFullPath,
+				WorkingDir: WorkingDirFullPath,
+			},
+		},
+		{
+			name: "empty working dir",
+			Input: &Config{
+				Common: &CommonCfg{
+					Daemonize: true,
+					PidDir:    "./tests",
+					LogMedia:  "file",
+					LogDir:    "./tests/log/",
+				},
+			},
+			expectedResult: &CommonCfg{
+				Daemonize: true,
+				PidDir:    PidDirFullPath,
+				LogMedia:  "file",
+				LogDir:    LogDirFullPath,
+			},
+		},
+		{
+			name:           "no common",
+			Input:          &Config{},
+			expectedResult: nil,
+		},
+	}
+
+	for idx, test := range tests {
+		err := test.Input.LoadCommon()
+		if err == nil && test.err != "" {
+			fmt.Printf("TEST '%s': NOK\n", test.name)
+			t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
+		} else if test.err != "" {
+			if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
+				fmt.Printf("TEST '%s': NOK\n", test.name)
+				t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
+					test.err,
+					fmt.Sprintf("%s", err))
+			}
+		}
+
+		isOk := assert.Equal(t, test.expectedResult, test.Input.Common)
+		if !isOk {
+			t.Fatalf("TEST '%s': NOK", test.name)
+		} else {
+			fmt.Printf("TEST '%s': OK\n", test.name)
+		}
+	}
+}

pkg/csconfig/config.go

@@ -4,19 +4,17 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"path/filepath"
-	"strings"
 
-	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v2"
 )
 
 /*top-level config : defaults,overriden by cfg file,overriden by cli*/
-type GlobalConfig struct {
+type Config struct {
 	//just a path to ourself :p
-	Self         *string             `yaml:"-"`
+	FilePath     *string             `yaml:"-"`
+	Self         []byte              `yaml:"-"`
 	Common       *CommonCfg          `yaml:"common,omitempty"`
 	Prometheus   *PrometheusCfg      `yaml:"prometheus,omitempty"`
 	Crowdsec     *CrowdsecServiceCfg `yaml:"crowdsec_service,omitempty"`
@@ -26,9 +24,10 @@ type GlobalConfig struct {
 	ConfigPaths  *ConfigurationPaths `yaml:"config_paths,omitempty"`
 	DisableAPI   bool                `yaml:"-"`
 	DisableAgent bool                `yaml:"-"`
+	Hub          *Hub                `yaml:"-"`
 }
 
-func (c *GlobalConfig) Dump() error {
+func (c *Config) Dump() error {
 	out, err := yaml.Marshal(c)
 	if err != nil {
 		return errors.Wrap(err, "failed marshaling config")
@@ -37,192 +36,26 @@ func (c *GlobalConfig) Dump() error {
 	return nil
 }
 
-func (c *GlobalConfig) LoadConfigurationFile(path string, disableAPI bool, disableAgent bool) error {
-	c.DisableAPI = disableAPI
-	c.DisableAgent = disableAgent
-	fcontent, err := ioutil.ReadFile(path)
+func NewConfig(configFile string, disableAgent bool, disableAPI bool) (*Config, error) {
+	fcontent, err := ioutil.ReadFile(configFile)
 	if err != nil {
-		return errors.Wrap(err, "failed to read config file")
+		return nil, errors.Wrap(err, "failed to read config file")
 	}
 	configData := os.ExpandEnv(string(fcontent))
-	err = yaml.UnmarshalStrict([]byte(configData), c)
-	if err != nil {
-		return errors.Wrap(err, "failed unmarshaling config")
-	}
-	path, err = filepath.Abs(path)
-	if err != nil {
-		return errors.Wrap(err, "failed to load absolute path")
-	}
-	c.Self = &path
-	if err := c.LoadConfiguration(); err != nil {
-		return errors.Wrap(err, "failed to load sub configurations")
-	}
-
-	return nil
-}
-
-func (c *GlobalConfig) LoadConfiguration() error {
-	if c.ConfigPaths.ConfigDir == "" {
-		return fmt.Errorf("please provide a configuration directory with the 'config_dir' directive in the 'config_paths' section")
-	}
-
-	if c.ConfigPaths.DataDir == "" {
-		return fmt.Errorf("please provide a data directory with the 'data_dir' directive in the 'config_paths' section")
-	}
-
-	if c.ConfigPaths.HubDir == "" {
-		c.ConfigPaths.HubDir = filepath.Clean(c.ConfigPaths.ConfigDir + "/hub")
-	}
-
-	if c.ConfigPaths.HubIndexFile == "" {
-		c.ConfigPaths.HubIndexFile = filepath.Clean(c.ConfigPaths.HubDir + "/.index.json")
-	}
-
-	if err := c.LoadSimulation(); err != nil {
-		return err
-	}
-	if c.Crowdsec != nil {
-		if c.Crowdsec.AcquisitionFilePath != "" {
-			log.Debugf("non-empty acquisition file path %s", c.Crowdsec.AcquisitionFilePath)
-			if _, err := os.Stat(c.Crowdsec.AcquisitionFilePath); err != nil {
-				return errors.Wrapf(err, "while checking acquisition path %s", c.Crowdsec.AcquisitionFilePath)
-			}
-			c.Crowdsec.AcquisitionFiles = append(c.Crowdsec.AcquisitionFiles, c.Crowdsec.AcquisitionFilePath)
-		}
-		if c.Crowdsec.AcquisitionDirPath != "" {
-			files, err := filepath.Glob(c.Crowdsec.AcquisitionDirPath + "/*.yaml")
-			c.Crowdsec.AcquisitionFiles = append(c.Crowdsec.AcquisitionFiles, files...)
-			if err != nil {
-				return errors.Wrap(err, "while globing acquis_dir")
-			}
-		}
-		if c.Crowdsec.AcquisitionDirPath == "" && c.Crowdsec.AcquisitionFilePath == "" {
-			return fmt.Errorf("no acquisition_path nor acquisition_dir")
-		}
-
-		c.Crowdsec.ConfigDir = c.ConfigPaths.ConfigDir
-		c.Crowdsec.DataDir = c.ConfigPaths.DataDir
-		c.Crowdsec.HubDir = c.ConfigPaths.HubDir
-		c.Crowdsec.HubIndexFile = c.ConfigPaths.HubIndexFile
-		if c.Crowdsec.ParserRoutinesCount <= 0 {
-			c.Crowdsec.ParserRoutinesCount = 1
-		}
-
-		if c.Crowdsec.BucketsRoutinesCount <= 0 {
-			c.Crowdsec.BucketsRoutinesCount = 1
-		}
-
-		if c.Crowdsec.OutputRoutinesCount <= 0 {
-			c.Crowdsec.OutputRoutinesCount = 1
-		}
-	}
-
-	if err := c.CleanupPaths(); err != nil {
-		return errors.Wrap(err, "invalid config")
-	}
-
-	if c.Cscli != nil {
-		c.Cscli.DbConfig = c.DbConfig
-		c.Cscli.ConfigDir = c.ConfigPaths.ConfigDir
-		c.Cscli.DataDir = c.ConfigPaths.DataDir
-		c.Cscli.HubDir = c.ConfigPaths.HubDir
-		c.Cscli.HubIndexFile = c.ConfigPaths.HubIndexFile
-		if c.Cscli.PrometheusUrl == "" {
-			port := 6060
-			if c.Prometheus.ListenPort != 0 {
-				port = c.Prometheus.ListenPort
-			}
-			c.Cscli.PrometheusUrl = fmt.Sprintf("http://127.0.0.1:%d/", port)
-		}
-	}
-
-	if c.API.Client != nil && c.API.Client.CredentialsFilePath != "" && !c.DisableAgent {
-		fcontent, err := ioutil.ReadFile(c.API.Client.CredentialsFilePath)
-		if err != nil {
-			return errors.Wrap(err, fmt.Sprintf("failed to read api client credential configuration file '%s'", c.API.Client.CredentialsFilePath))
-		}
-		err = yaml.UnmarshalStrict(fcontent, &c.API.Client.Credentials)
-		if err != nil {
-			return errors.Wrap(err, fmt.Sprintf("failed unmarshaling api client credential configuration file '%s'", c.API.Client.CredentialsFilePath))
-		}
-		if c.API.Client.Credentials != nil && c.API.Client.Credentials.URL != "" {
-			if !strings.HasSuffix(c.API.Client.Credentials.URL, "/") {
-				c.API.Client.Credentials.URL = c.API.Client.Credentials.URL + "/"
-			}
-		}
-		if c.API.Client.InsecureSkipVerify == nil {
-			apiclient.InsecureSkipVerify = false
-		} else {
-			apiclient.InsecureSkipVerify = *c.API.Client.InsecureSkipVerify
-		}
-	}
-	if c.API.Server != nil && !c.DisableAPI {
-		c.API.Server.DbConfig = c.DbConfig
-		c.API.Server.LogDir = c.Common.LogDir
-		c.API.Server.LogMedia = c.Common.LogMedia
-		if err := c.API.Server.LoadProfiles(); err != nil {
-			return errors.Wrap(err, "while loading profiles for LAPI")
-		}
-		if c.API.Server.OnlineClient != nil && c.API.Server.OnlineClient.CredentialsFilePath != "" {
-			c.API.Server.OnlineClient.Credentials = new(ApiCredentialsCfg)
-			fcontent, err := ioutil.ReadFile(c.API.Server.OnlineClient.CredentialsFilePath)
-			if err != nil {
-				return errors.Wrap(err, fmt.Sprintf("failed to read api server credentials configuration file '%s'", c.API.Server.OnlineClient.CredentialsFilePath))
-			}
-			err = yaml.UnmarshalStrict(fcontent, c.API.Server.OnlineClient.Credentials)
-			if err != nil {
-				return errors.Wrap(err, fmt.Sprintf("failed unmarshaling api server credentials configuration file '%s'", c.API.Server.OnlineClient.CredentialsFilePath))
-			}
-			if c.API.Server.OnlineClient.Credentials.Login == "" || c.API.Server.OnlineClient.Credentials.Password == "" || c.API.Server.OnlineClient.Credentials.URL == "" {
-				log.Debugf("can't load CAPI credentials from '%s' (missing field)", c.API.Server.OnlineClient.CredentialsFilePath)
-				c.API.Server.OnlineClient.Credentials = nil
-			}
-		}
-		if c.API.Server.OnlineClient == nil || c.API.Server.OnlineClient.Credentials == nil {
-			log.Printf("push and pull to crowdsec API disabled")
-		}
-	}
-
-	return nil
-}
-
-func (c *GlobalConfig) LoadSimulation() error {
-	if c.ConfigPaths == nil {
-		return fmt.Errorf("ConfigPaths is empty")
+	cfg := Config{
+		FilePath:     &configFile,
+		DisableAgent: disableAgent,
+		DisableAPI:   disableAPI,
 	}
 
-	simCfg := SimulationConfig{}
-
-	if c.ConfigPaths.SimulationFilePath == "" {
-		c.ConfigPaths.SimulationFilePath = filepath.Clean(c.ConfigPaths.ConfigDir + "/simulation.yaml")
-	}
-
-	rcfg, err := ioutil.ReadFile(c.ConfigPaths.SimulationFilePath)
+	err = yaml.UnmarshalStrict([]byte(configData), &cfg)
 	if err != nil {
-		return errors.Wrapf(err, "while reading '%s'", c.ConfigPaths.SimulationFilePath)
-	} else {
-		if err := yaml.UnmarshalStrict(rcfg, &simCfg); err != nil {
-			return fmt.Errorf("while unmarshaling simulation file '%s' : %s", c.ConfigPaths.SimulationFilePath, err)
-		}
-	}
-	if simCfg.Simulation == nil {
-		simCfg.Simulation = new(bool)
-	}
-	if c.Crowdsec != nil {
-		c.Crowdsec.SimulationConfig = &simCfg
-	}
-	if c.Cscli != nil {
-		c.Cscli.SimulationConfig = &simCfg
+		return nil, err
 	}
-	return nil
-}
-
-func NewConfig() *GlobalConfig {
-	cfg := GlobalConfig{}
-	return &cfg
+	return &cfg, nil
 }
 
-func NewDefaultConfig() *GlobalConfig {
+func NewDefaultConfig() *Config {
 	logLevel := log.InfoLevel
 	CommonCfg := CommonCfg{
 		Daemonize: false,
@@ -270,7 +103,7 @@ func NewDefaultConfig() *GlobalConfig {
 		DbPath: "/var/lib/crowdsec/data/crowdsec.db",
 	}
 
-	globalCfg := GlobalConfig{
+	globalCfg := Config{
 		Common:      &CommonCfg,
 		Prometheus:  &prometheus,
 		Crowdsec:    &crowdsecCfg,
@@ -282,60 +115,3 @@ func NewDefaultConfig() *GlobalConfig {
 
 	return &globalCfg
 }
-
-func (c *GlobalConfig) CleanupPaths() error {
-	var err error
-
-	if c.Common != nil {
-		var CommonCleanup = []*string{
-			&c.Common.PidDir,
-			&c.Common.LogDir,
-			&c.Common.WorkingDir,
-		}
-		for _, k := range CommonCleanup {
-			if *k == "" {
-				continue
-			}
-			*k, err = filepath.Abs(*k)
-			if err != nil {
-				return errors.Wrap(err, "failed to clean path")
-			}
-		}
-	}
-
-	if c.Crowdsec != nil {
-		var crowdsecCleanup = []*string{
-			&c.Crowdsec.AcquisitionFilePath,
-		}
-		for _, k := range crowdsecCleanup {
-			if *k == "" {
-				continue
-			}
-			*k, err = filepath.Abs(*k)
-			if err != nil {
-				return errors.Wrap(err, "failed to clean path")
-			}
-		}
-	}
-
-	if c.ConfigPaths != nil {
-		var configPathsCleanup = []*string{
-			&c.ConfigPaths.HubDir,
-			&c.ConfigPaths.HubIndexFile,
-			&c.ConfigPaths.ConfigDir,
-			&c.ConfigPaths.DataDir,
-			&c.ConfigPaths.SimulationFilePath,
-		}
-		for _, k := range configPathsCleanup {
-			if *k == "" {
-				continue
-			}
-			*k, err = filepath.Abs(*k)
-			if err != nil {
-				return errors.Wrap(err, "failed to clean path")
-			}
-		}
-	}
-
-	return nil
-}

pkg/csconfig/config_paths.go

@@ -1,5 +1,12 @@
 package csconfig
 
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+)
+
 type ConfigurationPaths struct {
 	ConfigDir          string `yaml:"config_dir"`
 	DataDir            string `yaml:"data_dir,omitempty"`
@@ -7,3 +14,41 @@ type ConfigurationPaths struct {
 	HubIndexFile       string `yaml:"index_path,omitempty"` //path of the .index.json
 	HubDir             string `yaml:"hub_dir,omitempty"`
 }
+
+func (c *Config) LoadConfigurationPaths() error {
+	var err error
+	if c.ConfigPaths == nil {
+		return fmt.Errorf("no configuration paths provided")
+	}
+
+	if c.ConfigPaths.DataDir == "" {
+		return fmt.Errorf("please provide a data directory with the 'data_dir' directive in the 'config_paths' section")
+	}
+
+	if c.ConfigPaths.HubDir == "" {
+		c.ConfigPaths.HubDir = filepath.Clean(c.ConfigPaths.ConfigDir + "/hub")
+	}
+
+	if c.ConfigPaths.HubIndexFile == "" {
+		c.ConfigPaths.HubIndexFile = filepath.Clean(c.ConfigPaths.HubDir + "/.index.json")
+	}
+
+	var configPathsCleanup = []*string{
+		&c.ConfigPaths.HubDir,
+		&c.ConfigPaths.HubIndexFile,
+		&c.ConfigPaths.ConfigDir,
+		&c.ConfigPaths.DataDir,
+		&c.ConfigPaths.SimulationFilePath,
+	}
+	for _, k := range configPathsCleanup {
+		if *k == "" {
+			continue
+		}
+		*k, err = filepath.Abs(*k)
+		if err != nil {
+			return errors.Wrapf(err, "failed to get absolute path of '%s'", *k)
+		}
+	}
+
+	return nil
+}

pkg/csconfig/config_test.go

@@ -2,203 +2,59 @@ package csconfig
 
 import (
 	"fmt"
+	"log"
 	"strings"
 	"testing"
 
-	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 )
 
-func TestDefaultConfig(t *testing.T) {
-	x := NewDefaultConfig()
-	x.Dump()
-}
-
 func TestNormalLoad(t *testing.T) {
 
-	x := NewConfig()
-	err := x.LoadConfigurationFile("./tests/config.yaml", false, false)
+	_, err := NewConfig("./tests/config.yaml", false, false)
 	if err != nil {
 		t.Fatalf("unexpected error %s", err)
 	}
 
-	x = NewConfig()
-	err = x.LoadConfigurationFile("./tests/xxx.yaml", false, false)
+	_, err = NewConfig("./tests/xxx.yaml", false, false)
 	if fmt.Sprintf("%s", err) != "failed to read config file: open ./tests/xxx.yaml: no such file or directory" {
 		t.Fatalf("unexpected error %s", err)
 	}
 
-	x = NewConfig()
-	err = x.LoadConfigurationFile("./tests/simulation.yaml", false, false)
-	if !strings.HasPrefix(fmt.Sprintf("%s", err), "failed unmarshaling config: yaml: unmarshal error") {
+	_, err = NewConfig("./tests/simulation.yaml", false, false)
+	if !strings.HasPrefix(fmt.Sprintf("%s", err), "yaml: unmarshal errors:") {
 		t.Fatalf("unexpected error %s", err)
 	}
 
 }
 
-func TestCleanupPaths(t *testing.T) {
-	tests := []struct {
-		name           string
-		Input          *GlobalConfig
-		expectedResult *GlobalConfig
-		err            string
-	}{
-		{
-			name: "daemon cleanup",
-			Input: &GlobalConfig{
-				Common: &CommonCfg{
-					PidDir:     "////tmp//",
-					LogDir:     "/////tmp///",
-					WorkingDir: "/////tmp///",
-				},
-			},
-			expectedResult: &GlobalConfig{
-				Common: &CommonCfg{
-					PidDir:     "/tmp",
-					LogDir:     "/tmp",
-					WorkingDir: "/tmp",
-				},
-			},
-		},
-		//
-		{
-			name: "crowdsec cleanup",
-			Input: &GlobalConfig{
-				Crowdsec: &CrowdsecServiceCfg{
-					AcquisitionFilePath: "////tmp//x.yaml",
-				},
-			},
-			expectedResult: &GlobalConfig{
-				Crowdsec: &CrowdsecServiceCfg{
-					AcquisitionFilePath: "/tmp/x.yaml",
-				},
-			},
-		},
-		//
-		{
-			name: "config paths cleanup",
-			Input: &GlobalConfig{
-				ConfigPaths: &ConfigurationPaths{
-					HubDir:             "////tmp//",
-					HubIndexFile:       "////tmp//x.yaml",
-					ConfigDir:          "////tmp//",
-					DataDir:            "////tmp//",
-					SimulationFilePath: "//tmp///toto.yaml",
-				},
-			},
-			expectedResult: &GlobalConfig{
-				ConfigPaths: &ConfigurationPaths{
-					HubDir:             "/tmp",
-					HubIndexFile:       "/tmp/x.yaml",
-					ConfigDir:          "/tmp",
-					DataDir:            "/tmp",
-					SimulationFilePath: "/tmp/toto.yaml",
-				},
-			},
-		},
-	}
-	for idx, test := range tests {
-		err := test.Input.CleanupPaths()
-		if test.err != "" {
-			if strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
-				t.Fatalf("%d/%d expected err %s got %s", idx, len(tests), test.err, fmt.Sprintf("%s", err))
-			}
-		}
-		isOk := assert.Equal(t, test.expectedResult, test.Input)
-		if !isOk {
-			t.Fatalf("%d/%d failed test", idx, len(tests))
-		}
-	}
-}
-
-func TestSimulationLoading(t *testing.T) {
-	tests := []struct {
-		name           string
-		Input          *GlobalConfig
-		expectedResult *SimulationConfig
-		err            string
-	}{
-		{
-			name: "basic valid simulation",
-			Input: &GlobalConfig{
-				ConfigPaths: &ConfigurationPaths{
-					SimulationFilePath: "./tests/simulation.yaml",
-				},
-				Crowdsec: &CrowdsecServiceCfg{},
-			},
-			expectedResult: &SimulationConfig{Simulation: new(bool)},
-		},
-		{
-			name: "basic bad file name",
-			Input: &GlobalConfig{
-				ConfigPaths: &ConfigurationPaths{
-					SimulationFilePath: "./tests/xxx.yaml",
-				},
-				Crowdsec: &CrowdsecServiceCfg{},
-			},
-			err: "while reading './tests/xxx.yaml': open ./tests/xxx.yaml: no such file or directory",
-		},
-		{
-			name: "basic nil config",
-			Input: &GlobalConfig{
-				ConfigPaths: &ConfigurationPaths{
-					SimulationFilePath: "",
-				},
-				Crowdsec: &CrowdsecServiceCfg{},
-			},
-		},
-		{
-			name: "basic bad file content",
-			Input: &GlobalConfig{
-				ConfigPaths: &ConfigurationPaths{
-					SimulationFilePath: "./tests/config.yaml",
-				},
-				Crowdsec: &CrowdsecServiceCfg{},
-			},
-			err: "while unmarshaling simulation file './tests/config.yaml' : yaml: unmarshal errors",
-		},
-	}
-
-	for idx, test := range tests {
-		err := test.Input.LoadSimulation()
-		if err == nil && test.err != "" {
-			t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
-		} else if test.err != "" {
-			if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
-				t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
-					test.err,
-					fmt.Sprintf("%s", err))
-			}
-		}
-
-		isOk := assert.Equal(t, test.expectedResult, test.Input.Crowdsec.SimulationConfig)
-		if !isOk {
-			t.Fatalf("test '%s' failed", test.name)
-		}
-
-	}
-
-}
-
 func TestNewCrowdSecConfig(t *testing.T) {
 	tests := []struct {
 		name           string
-		expectedResult *GlobalConfig
+		expectedResult *Config
 		err            string
 	}{
 		{
 			name:           "new configuration: basic",
-			expectedResult: &GlobalConfig{},
+			expectedResult: &Config{},
 			err:            "",
 		},
 	}
 	for _, test := range tests {
-		result := NewConfig()
+		result := &Config{}
 		isOk := assert.Equal(t, test.expectedResult, result)
 		if !isOk {
-			t.Fatalf("test '%s' failed", test.name)
+			t.Fatalf("TEST '%s': NOK", test.name)
+		} else {
+			fmt.Printf("TEST '%s': OK\n", test.name)
 		}
-		log.Infof("test '%s' : OK", test.name)
 	}
 
 }
+
+func TestDefaultConfig(t *testing.T) {
+	x := NewDefaultConfig()
+	if err := x.Dump(); err != nil {
+		log.Fatal(err)
+	}
+}

pkg/csconfig/crowdsec_service.go

@@ -1,5 +1,14 @@
 package csconfig
 
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+)
+
 /*Configurations needed for crowdsec to load parser/scenarios/... + acquisition*/
 type CrowdsecServiceCfg struct {
 	AcquisitionFilePath string `yaml:"acquisition_path,omitempty"`
@@ -21,3 +30,82 @@ type CrowdsecServiceCfg struct {
 	HubIndexFile       string `yaml:"-"`
 	SimulationFilePath string `yaml:"-"`
 }
+
+func (c *Config) LoadCrowdsec() error {
+	var err error
+	// Configuration paths are dependency to load crowdsec configuration
+	if err := c.LoadConfigurationPaths(); err != nil {
+		return err
+	}
+
+	if c.Crowdsec == nil {
+		log.Warningf("crowdsec agent is disabled")
+		c.DisableAgent = true
+		return nil
+	}
+	if c.Crowdsec.AcquisitionFilePath != "" {
+		log.Debugf("non-empty acquisition file path %s", c.Crowdsec.AcquisitionFilePath)
+		if _, err := os.Stat(c.Crowdsec.AcquisitionFilePath); err != nil {
+			return errors.Wrapf(err, "while checking acquisition path %s", c.Crowdsec.AcquisitionFilePath)
+		}
+		c.Crowdsec.AcquisitionFiles = append(c.Crowdsec.AcquisitionFiles, c.Crowdsec.AcquisitionFilePath)
+	}
+	if c.Crowdsec.AcquisitionDirPath != "" {
+		c.Crowdsec.AcquisitionDirPath, err = filepath.Abs(c.Crowdsec.AcquisitionDirPath)
+		if err != nil {
+			return errors.Wrapf(err, "can't get absolute path of '%s'", c.Crowdsec.AcquisitionDirPath)
+		}
+		files, err := filepath.Glob(c.Crowdsec.AcquisitionDirPath + "/*.yaml")
+		if err != nil {
+			return errors.Wrap(err, "while globing acquis_dir")
+		}
+		c.Crowdsec.AcquisitionFiles = append(c.Crowdsec.AcquisitionFiles, files...)
+	}
+	if c.Crowdsec.AcquisitionDirPath == "" && c.Crowdsec.AcquisitionFilePath == "" {
+		return fmt.Errorf("no acquisition_path nor acquisition_dir")
+	}
+
+	c.Crowdsec.ConfigDir = c.ConfigPaths.ConfigDir
+	c.Crowdsec.DataDir = c.ConfigPaths.DataDir
+	c.Crowdsec.HubDir = c.ConfigPaths.HubDir
+	c.Crowdsec.HubIndexFile = c.ConfigPaths.HubIndexFile
+	if c.Crowdsec.ParserRoutinesCount <= 0 {
+		c.Crowdsec.ParserRoutinesCount = 1
+	}
+
+	if c.Crowdsec.BucketsRoutinesCount <= 0 {
+		c.Crowdsec.BucketsRoutinesCount = 1
+	}
+
+	if c.Crowdsec.OutputRoutinesCount <= 0 {
+		c.Crowdsec.OutputRoutinesCount = 1
+	}
+
+	var crowdsecCleanup = []*string{
+		&c.Crowdsec.AcquisitionFilePath,
+	}
+	for _, k := range crowdsecCleanup {
+		if *k == "" {
+			continue
+		}
+		*k, err = filepath.Abs(*k)
+		if err != nil {
+			return errors.Wrapf(err, "failed to get absolute path of '%s'", *k)
+		}
+	}
+	for i, file := range c.Crowdsec.AcquisitionFiles {
+		f, err := filepath.Abs(file)
+		if err != nil {
+			return errors.Wrapf(err, "failed to get absolute path of '%s'", file)
+		}
+		c.Crowdsec.AcquisitionFiles[i] = f
+	}
+
+	if err := c.LoadAPIClient(); err != nil {
+		return fmt.Errorf("loading api client: %s", err.Error())
+	}
+	if err := c.LoadHub(); err != nil {
+		return fmt.Errorf("loading hub: %s", err)
+	}
+	return nil
+}

pkg/csconfig/crowdsec_service_test.go

@@ -0,0 +1,192 @@
+package csconfig
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLoadCrowdsec(t *testing.T) {
+	acquisFullPath, err := filepath.Abs("./tests/acquis.yaml")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	acquisInDirFullPath, err := filepath.Abs("./tests/acquis/acquis.yaml")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	acquisDirFullPath, err := filepath.Abs("./tests/acquis")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	hubFullPath, err := filepath.Abs("./hub")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	dataFullPath, err := filepath.Abs("./data")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	configDirFullPath, err := filepath.Abs("./tests")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	tests := []struct {
+		name           string
+		Input          *Config
+		expectedResult *CrowdsecServiceCfg
+		err            string
+	}{
+		{
+			name: "basic valid configuration",
+			Input: &Config{
+				ConfigPaths: &ConfigurationPaths{
+					ConfigDir: "./tests",
+					DataDir:   "./data",
+					HubDir:    "./hub",
+				},
+				API: &APICfg{
+					Client: &LocalApiClientCfg{
+						CredentialsFilePath: "./tests/lapi-secrets.yaml",
+					},
+				},
+				Crowdsec: &CrowdsecServiceCfg{
+					AcquisitionFilePath: "./tests/acquis.yaml",
+				},
+			},
+			expectedResult: &CrowdsecServiceCfg{
+				AcquisitionDirPath:   "",
+				AcquisitionFilePath:  acquisFullPath,
+				ConfigDir:            configDirFullPath,
+				DataDir:              dataFullPath,
+				HubDir:               hubFullPath,
+				HubIndexFile:         hubIndexFileFullPath,
+				BucketsRoutinesCount: 1,
+				ParserRoutinesCount:  1,
+				OutputRoutinesCount:  1,
+				AcquisitionFiles:     []string{acquisFullPath},
+			},
+		},
+		{
+			name: "basic valid configuration with acquisition dir",
+			Input: &Config{
+				ConfigPaths: &ConfigurationPaths{
+					ConfigDir: "./tests",
+					DataDir:   "./data",
+					HubDir:    "./hub",
+				},
+				API: &APICfg{
+					Client: &LocalApiClientCfg{
+						CredentialsFilePath: "./tests/lapi-secrets.yaml",
+					},
+				},
+				Crowdsec: &CrowdsecServiceCfg{
+					AcquisitionFilePath: "./tests/acquis.yaml",
+					AcquisitionDirPath:  "./tests/acquis/",
+				},
+			},
+			expectedResult: &CrowdsecServiceCfg{
+				AcquisitionDirPath:   acquisDirFullPath,
+				AcquisitionFilePath:  acquisFullPath,
+				ConfigDir:            configDirFullPath,
+				HubIndexFile:         hubIndexFileFullPath,
+				DataDir:              dataFullPath,
+				HubDir:               hubFullPath,
+				BucketsRoutinesCount: 1,
+				ParserRoutinesCount:  1,
+				OutputRoutinesCount:  1,
+				AcquisitionFiles:     []string{acquisFullPath, acquisInDirFullPath},
+			},
+		},
+		{
+			name: "no acquisition file and dir",
+			Input: &Config{
+				ConfigPaths: &ConfigurationPaths{
+					ConfigDir: "./tests",
+					DataDir:   "./data",
+					HubDir:    "./hub",
+				},
+				API: &APICfg{
+					Client: &LocalApiClientCfg{
+						CredentialsFilePath: "./tests/lapi-secrets.yaml",
+					},
+				},
+				Crowdsec: &CrowdsecServiceCfg{},
+			},
+			expectedResult: &CrowdsecServiceCfg{
+				BucketsRoutinesCount: 0,
+				ParserRoutinesCount:  0,
+				OutputRoutinesCount:  0,
+			},
+		},
+		{
+			name: "non existing acquisition file",
+			Input: &Config{
+				ConfigPaths: &ConfigurationPaths{
+					ConfigDir: "./tests",
+					DataDir:   "./data",
+					HubDir:    "./hub",
+				},
+				API: &APICfg{
+					Client: &LocalApiClientCfg{
+						CredentialsFilePath: "./tests/lapi-secrets.yaml",
+					},
+				},
+				Crowdsec: &CrowdsecServiceCfg{
+					AcquisitionFilePath: "./tests/acquis_not_exist.yaml",
+				},
+			},
+			expectedResult: &CrowdsecServiceCfg{
+				AcquisitionFilePath:  "./tests/acquis_not_exist.yaml",
+				BucketsRoutinesCount: 0,
+				ParserRoutinesCount:  0,
+				OutputRoutinesCount:  0,
+			},
+		},
+		{
+			name: "agent disabled",
+			Input: &Config{
+				ConfigPaths: &ConfigurationPaths{
+					ConfigDir: "./tests",
+					DataDir:   "./data",
+					HubDir:    "./hub",
+				},
+			},
+			expectedResult: nil,
+		},
+	}
+
+	for idx, test := range tests {
+		fmt.Printf("TEST '%s'\n", test.name)
+		err := test.Input.LoadCrowdsec()
+		if err == nil && test.err != "" {
+			t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
+		} else if test.err != "" {
+			if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
+				t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
+					test.err,
+					fmt.Sprintf("%s", err))
+			}
+		}
+
+		isOk := assert.Equal(t, test.expectedResult, test.Input.Crowdsec)
+		if !isOk {
+			t.Fatalf("test '%s' failed", test.name)
+		}
+
+	}
+}

pkg/csconfig/cscli.go

@@ -13,3 +13,18 @@ type CscliCfg struct {
 	SimulationFilePath string            `yaml:"-"`
 	PrometheusUrl      string            `yaml:"prometheus_uri"`
 }
+
+func (c *Config) LoadCSCLI() error {
+	if c.Cscli == nil {
+		c.Cscli = &CscliCfg{}
+	}
+	if err := c.LoadConfigurationPaths(); err != nil {
+		return err
+	}
+	c.Cscli.ConfigDir = c.ConfigPaths.ConfigDir
+	c.Cscli.DataDir = c.ConfigPaths.DataDir
+	c.Cscli.HubDir = c.ConfigPaths.HubDir
+	c.Cscli.HubIndexFile = c.ConfigPaths.HubIndexFile
+
+	return nil
+}

pkg/csconfig/cscli_test.go

@@ -0,0 +1,84 @@
+package csconfig
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLoadCSCLI(t *testing.T) {
+	hubFullPath, err := filepath.Abs("./hub")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	dataFullPath, err := filepath.Abs("./data")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	configDirFullPath, err := filepath.Abs("./tests")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	tests := []struct {
+		name           string
+		Input          *Config
+		expectedResult *CscliCfg
+		err            string
+	}{
+		{
+			name: "basic valid configuration",
+			Input: &Config{
+				ConfigPaths: &ConfigurationPaths{
+					ConfigDir:    "./tests",
+					DataDir:      "./data",
+					HubDir:       "./hub",
+					HubIndexFile: "./hub/.index.json",
+				},
+			},
+			expectedResult: &CscliCfg{
+				ConfigDir:    configDirFullPath,
+				DataDir:      dataFullPath,
+				HubDir:       hubFullPath,
+				HubIndexFile: hubIndexFileFullPath,
+			},
+		},
+		{
+			name:           "no configuration path",
+			Input:          &Config{},
+			expectedResult: &CscliCfg{},
+		},
+	}
+
+	for idx, test := range tests {
+		err := test.Input.LoadCSCLI()
+		if err == nil && test.err != "" {
+			fmt.Printf("TEST '%s': NOK\n", test.name)
+			t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
+		} else if test.err != "" {
+			if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
+				fmt.Printf("TEST '%s': NOK\n", test.name)
+				t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
+					test.err,
+					fmt.Sprintf("%s", err))
+			}
+		}
+
+		isOk := assert.Equal(t, test.expectedResult, test.Input.Cscli)
+		if !isOk {
+			t.Fatalf("TEST '%s': NOK", test.name)
+		} else {
+			fmt.Printf("TEST '%s': OK\n", test.name)
+		}
+	}
+}

pkg/csconfig/database.go

@@ -1,6 +1,10 @@
 package csconfig
 
-import log "github.com/sirupsen/logrus"
+import (
+	"fmt"
+
+	log "github.com/sirupsen/logrus"
+)
 
 type DatabaseCfg struct {
 	User     string      `yaml:"user"`
@@ -18,3 +22,19 @@ type FlushDBCfg struct {
 	MaxItems *int    `yaml:"max_items"`
 	MaxAge   *string `yaml:"max_age"`
 }
+
+func (c *Config) LoadDBConfig() error {
+	if c.DbConfig == nil {
+		return fmt.Errorf("no database configuration provided")
+	}
+
+	if c.Cscli != nil {
+		c.Cscli.DbConfig = c.DbConfig
+	}
+
+	if c.API != nil && c.API.Server != nil {
+		c.API.Server.DbConfig = c.DbConfig
+	}
+
+	return nil
+}

pkg/csconfig/database_test.go

@@ -0,0 +1,62 @@
+package csconfig
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLoadDBConfig(t *testing.T) {
+	tests := []struct {
+		name           string
+		Input          *Config
+		expectedResult *DatabaseCfg
+		err            string
+	}{
+		{
+			name: "basic valid configuration",
+			Input: &Config{
+				DbConfig: &DatabaseCfg{
+					Type:   "sqlite",
+					DbPath: "./tests/test.db",
+				},
+				Cscli: &CscliCfg{},
+				API: &APICfg{
+					Server: &LocalApiServerCfg{},
+				},
+			},
+			expectedResult: &DatabaseCfg{
+				Type:   "sqlite",
+				DbPath: "./tests/test.db",
+			},
+		},
+		{
+			name:           "no configuration path",
+			Input:          &Config{},
+			expectedResult: nil,
+		},
+	}
+
+	for idx, test := range tests {
+		err := test.Input.LoadDBConfig()
+		if err == nil && test.err != "" {
+			fmt.Printf("TEST '%s': NOK\n", test.name)
+			t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
+		} else if test.err != "" {
+			if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
+				fmt.Printf("TEST '%s': NOK\n", test.name)
+				t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
+					test.err,
+					fmt.Sprintf("%s", err))
+			}
+		}
+		isOk := assert.Equal(t, test.expectedResult, test.Input.DbConfig)
+		if !isOk {
+			t.Fatalf("TEST '%s': NOK", test.name)
+		} else {
+			fmt.Printf("TEST '%s': OK\n", test.name)
+		}
+	}
+}

pkg/csconfig/hub.go

@@ -0,0 +1,24 @@
+package csconfig
+
+/*cscli specific config, such as hub directory*/
+type Hub struct {
+	HubDir       string `yaml:"-"`
+	ConfigDir    string `yaml:"-"`
+	HubIndexFile string `yaml:"-"`
+	DataDir      string `yaml:"-"`
+}
+
+func (c *Config) LoadHub() error {
+	if err := c.LoadConfigurationPaths(); err != nil {
+		return err
+	}
+
+	c.Hub = &Hub{
+		HubIndexFile: c.ConfigPaths.HubIndexFile,
+		ConfigDir:    c.ConfigPaths.ConfigDir,
+		HubDir:       c.ConfigPaths.HubDir,
+		DataDir:      c.ConfigPaths.DataDir,
+	}
+
+	return nil
+}

pkg/csconfig/hub_test.go

@@ -0,0 +1,94 @@
+package csconfig
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLoadHub(t *testing.T) {
+	hubFullPath, err := filepath.Abs("./hub")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	dataFullPath, err := filepath.Abs("./data")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	configDirFullPath, err := filepath.Abs("./tests")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	tests := []struct {
+		name           string
+		Input          *Config
+		expectedResult *Hub
+		err            string
+	}{
+		{
+			name: "basic valid configuration",
+			Input: &Config{
+				ConfigPaths: &ConfigurationPaths{
+					ConfigDir:    "./tests",
+					DataDir:      "./data",
+					HubDir:       "./hub",
+					HubIndexFile: "./hub/.index.json",
+				},
+			},
+			expectedResult: &Hub{
+				ConfigDir:    configDirFullPath,
+				DataDir:      dataFullPath,
+				HubDir:       hubFullPath,
+				HubIndexFile: hubIndexFileFullPath,
+			},
+		},
+		{
+			name: "no data dir",
+			Input: &Config{
+				ConfigPaths: &ConfigurationPaths{
+					ConfigDir:    "./tests",
+					HubDir:       "./hub",
+					HubIndexFile: "./hub/.index.json",
+				},
+			},
+			expectedResult: nil,
+		},
+		{
+			name:           "no configuration path",
+			Input:          &Config{},
+			expectedResult: nil,
+		},
+	}
+
+	for idx, test := range tests {
+		err := test.Input.LoadHub()
+		if err == nil && test.err != "" {
+			fmt.Printf("TEST '%s': NOK\n", test.name)
+			t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
+		} else if test.err != "" {
+			if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
+				fmt.Printf("TEST '%s': NOK\n", test.name)
+				t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
+					test.err,
+					fmt.Sprintf("%s", err))
+			}
+		}
+		isOk := assert.Equal(t, test.expectedResult, test.Input.Hub)
+		if !isOk {
+			t.Fatalf("TEST '%s': NOK", test.name)
+		} else {
+			fmt.Printf("TEST '%s': OK\n", test.name)
+		}
+	}
+}

pkg/csconfig/prometheus.go

@@ -1,5 +1,7 @@
 package csconfig
 
+import "fmt"
+
 /**/
 type PrometheusCfg struct {
 	Enabled    bool   `yaml:"enabled"`
@@ -7,3 +9,13 @@ type PrometheusCfg struct {
 	ListenAddr string `yaml:"listen_addr"`
 	ListenPort int    `yaml:"listen_port"`
 }
+
+func (c *Config) LoadPrometheus() error {
+	if c.Cscli != nil && c.Cscli.PrometheusUrl == "" && c.Prometheus != nil {
+		if c.Prometheus.ListenAddr != "" && c.Prometheus.ListenPort != 0 {
+			c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d", c.Prometheus.ListenAddr, c.Prometheus.ListenPort)
+		}
+	}
+
+	return nil
+}

pkg/csconfig/prometheus_test.go

@@ -0,0 +1,55 @@
+package csconfig
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLoadPrometheus(t *testing.T) {
+
+	tests := []struct {
+		name           string
+		Input          *Config
+		expectedResult string
+		err            string
+	}{
+		{
+			name: "basic valid configuration",
+			Input: &Config{
+				Prometheus: &PrometheusCfg{
+					Enabled:    true,
+					Level:      "full",
+					ListenAddr: "127.0.0.1",
+					ListenPort: 6060,
+				},
+				Cscli: &CscliCfg{},
+			},
+			expectedResult: "http://127.0.0.1:6060",
+		},
+	}
+
+	for idx, test := range tests {
+		err := test.Input.LoadPrometheus()
+		if err == nil && test.err != "" {
+			fmt.Printf("TEST '%s': NOK\n", test.name)
+			t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
+		} else if test.err != "" {
+			if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
+				fmt.Printf("TEST '%s': NOK\n", test.name)
+				t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
+					test.err,
+					fmt.Sprintf("%s", err))
+			}
+		}
+
+		isOk := assert.Equal(t, test.expectedResult, test.Input.Cscli.PrometheusUrl)
+		if !isOk {
+			t.Fatalf("test '%s' failed\n", test.name)
+		} else {
+			fmt.Printf("TEST '%s': OK\n", test.name)
+		}
+	}
+}

pkg/csconfig/simulation.go

@@ -1,5 +1,14 @@
 package csconfig
 
+import (
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
+)
+
 type SimulationConfig struct {
 	Simulation *bool    `yaml:"simulation"`
 	Exclusions []string `yaml:"exclusions,omitempty"`
@@ -19,3 +28,33 @@ func (s *SimulationConfig) IsSimulated(scenario string) bool {
 	}
 	return simulated
 }
+
+func (c *Config) LoadSimulation() error {
+
+	if err := c.LoadConfigurationPaths(); err != nil {
+		return err
+	}
+
+	simCfg := SimulationConfig{}
+	if c.ConfigPaths.SimulationFilePath == "" {
+		c.ConfigPaths.SimulationFilePath = filepath.Clean(c.ConfigPaths.ConfigDir + "/simulation.yaml")
+	}
+	rcfg, err := ioutil.ReadFile(c.ConfigPaths.SimulationFilePath)
+	if err != nil {
+		return errors.Wrapf(err, "while reading '%s'", c.ConfigPaths.SimulationFilePath)
+	} else {
+		if err := yaml.UnmarshalStrict(rcfg, &simCfg); err != nil {
+			return fmt.Errorf("while unmarshaling simulation file '%s' : %s", c.ConfigPaths.SimulationFilePath, err)
+		}
+	}
+	if simCfg.Simulation == nil {
+		simCfg.Simulation = new(bool)
+	}
+	if c.Crowdsec != nil {
+		c.Crowdsec.SimulationConfig = &simCfg
+	}
+	if c.Cscli != nil {
+		c.Cscli.SimulationConfig = &simCfg
+	}
+	return nil
+}

pkg/csconfig/simulation_test.go

@@ -0,0 +1,156 @@
+package csconfig
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSimulationLoading(t *testing.T) {
+
+	testXXFullPath, err := filepath.Abs("./tests/xxx.yaml")
+	if err != nil {
+		panic(err)
+	}
+
+	badYamlFullPath, err := filepath.Abs("./tests/config.yaml")
+	if err != nil {
+		panic(err)
+	}
+
+	tests := []struct {
+		name           string
+		Input          *Config
+		expectedResult *SimulationConfig
+		err            string
+	}{
+		{
+			name: "basic valid simulation",
+			Input: &Config{
+				ConfigPaths: &ConfigurationPaths{
+					SimulationFilePath: "./tests/simulation.yaml",
+					DataDir:            "./data",
+				},
+				Crowdsec: &CrowdsecServiceCfg{},
+				Cscli:    &CscliCfg{},
+			},
+			expectedResult: &SimulationConfig{Simulation: new(bool)},
+		},
+		{
+			name: "basic bad file name",
+			Input: &Config{
+				ConfigPaths: &ConfigurationPaths{
+					SimulationFilePath: "./tests/xxx.yaml",
+					DataDir:            "./data",
+				},
+				Crowdsec: &CrowdsecServiceCfg{},
+			},
+			err: fmt.Sprintf("while reading '%s': open %s: no such file or directory", testXXFullPath, testXXFullPath),
+		},
+		{
+			name: "basic nil config",
+			Input: &Config{
+				ConfigPaths: &ConfigurationPaths{
+					SimulationFilePath: "",
+					DataDir:            "./data",
+				},
+				Crowdsec: &CrowdsecServiceCfg{},
+			},
+		},
+		{
+			name: "basic bad file content",
+			Input: &Config{
+				ConfigPaths: &ConfigurationPaths{
+					SimulationFilePath: "./tests/config.yaml",
+					DataDir:            "./data",
+				},
+				Crowdsec: &CrowdsecServiceCfg{},
+			},
+			err: fmt.Sprintf("while unmarshaling simulation file '%s' : yaml: unmarshal errors", badYamlFullPath),
+		},
+		{
+			name: "basic bad file content",
+			Input: &Config{
+				ConfigPaths: &ConfigurationPaths{
+					SimulationFilePath: "./tests/config.yaml",
+					DataDir:            "./data",
+				},
+				Crowdsec: &CrowdsecServiceCfg{},
+			},
+			err: fmt.Sprintf("while unmarshaling simulation file '%s' : yaml: unmarshal errors", badYamlFullPath),
+		},
+	}
+
+	for idx, test := range tests {
+		err := test.Input.LoadSimulation()
+		if err == nil && test.err != "" {
+			fmt.Printf("TEST '%s': NOK\n", test.name)
+			t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
+		} else if test.err != "" {
+			if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
+				fmt.Printf("TEST '%s': NOK\n", test.name)
+				t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
+					test.err,
+					fmt.Sprintf("%s", err))
+			}
+		}
+		isOk := assert.Equal(t, test.expectedResult, test.Input.Crowdsec.SimulationConfig)
+		if !isOk {
+			t.Fatalf("TEST '%s': NOK\n", test.name)
+		} else {
+			fmt.Printf("TEST '%s': OK\n", test.name)
+		}
+	}
+}
+
+func TestIsSimulated(t *testing.T) {
+	simCfgOff := &SimulationConfig{
+		Simulation: new(bool),
+		Exclusions: []string{"test"},
+	}
+
+	simCfgOn := &SimulationConfig{
+		Simulation: new(bool),
+		Exclusions: []string{"test"},
+	}
+	*simCfgOn.Simulation = true
+
+	tests := []struct {
+		name             string
+		SimulationConfig *SimulationConfig
+		Input            string
+		expectedResult   bool
+		err              string
+	}{
+		{
+			name:             "No simulation except (in exclusion)",
+			SimulationConfig: simCfgOff,
+			Input:            "test",
+			expectedResult:   true,
+		},
+		{
+			name:             "All simulation (not in exclusion)",
+			SimulationConfig: simCfgOn,
+			Input:            "toto",
+			expectedResult:   true,
+		},
+		{
+			name:             "All simulation (in exclusion)",
+			SimulationConfig: simCfgOn,
+			Input:            "test",
+			expectedResult:   false,
+		},
+	}
+	for _, test := range tests {
+		IsSimulated := test.SimulationConfig.IsSimulated(test.Input)
+		isOk := assert.Equal(t, test.expectedResult, IsSimulated)
+		if !isOk {
+			fmt.Printf("TEST: '%v' failed", test.name)
+			t.Fatal()
+		}
+	}
+
+}

pkg/csconfig/tests/bad_lapi-secrets.yaml

@@ -0,0 +1 @@
+unknown_key: test

pkg/csconfig/tests/bad_online-api-secrets.yaml

@@ -0,0 +1,3 @@
+login: test
+password:
+url:

pkg/csconfig/tests/lapi-secrets.yaml

@@ -0,0 +1,3 @@
+url: http://localhost:8080
+login: test
+password: testpassword

pkg/csconfig/tests/online-api-secrets.yaml

@@ -0,0 +1,3 @@
+url: http://crowdsec.api
+login: test
+password: testpassword

pkg/cwhub/cwhub_test.go

@@ -27,12 +27,12 @@ var testDataFolder = "."
 func TestItemStatus(t *testing.T) {
 	cfg := test_prepenv()
 
-	err := UpdateHubIdx(cfg.Cscli)
+	err := UpdateHubIdx(cfg.Hub)
 	//DownloadHubIdx()
 	if err != nil {
 		t.Fatalf("failed to download index : %s", err)
 	}
-	if err := GetHubIdx(cfg.Cscli); err != nil {
+	if err := GetHubIdx(cfg.Hub); err != nil {
 		t.Fatalf("failed to load hub index : %s", err)
 	}
 
@@ -74,12 +74,12 @@ func TestItemStatus(t *testing.T) {
 func TestGetters(t *testing.T) {
 	cfg := test_prepenv()
 
-	err := UpdateHubIdx(cfg.Cscli)
+	err := UpdateHubIdx(cfg.Hub)
 	//DownloadHubIdx()
 	if err != nil {
 		t.Fatalf("failed to download index : %s", err)
 	}
-	if err := GetHubIdx(cfg.Cscli); err != nil {
+	if err := GetHubIdx(cfg.Hub); err != nil {
 		t.Fatalf("failed to load hub index : %s", err)
 	}
 
@@ -135,58 +135,58 @@ func TestIndexDownload(t *testing.T) {
 
 	cfg := test_prepenv()
 
-	err := UpdateHubIdx(cfg.Cscli)
+	err := UpdateHubIdx(cfg.Hub)
 	//DownloadHubIdx()
 	if err != nil {
 		t.Fatalf("failed to download index : %s", err)
 	}
-	if err := GetHubIdx(cfg.Cscli); err != nil {
+	if err := GetHubIdx(cfg.Hub); err != nil {
 		t.Fatalf("failed to load hub index : %s", err)
 	}
 }
 
-func test_prepenv() *csconfig.GlobalConfig {
+func test_prepenv() *csconfig.Config {
 	log.SetLevel(log.DebugLevel)
 
-	var cfg = csconfig.NewConfig()
-	cfg.Cscli = &csconfig.CscliCfg{}
-	cfg.Cscli.ConfigDir, _ = filepath.Abs("./install")
-	cfg.Cscli.HubDir, _ = filepath.Abs("./hubdir")
-	cfg.Cscli.HubIndexFile = filepath.Clean("./hubdir/.index.json")
+	var cfg = &csconfig.Config{}
+	cfg.Hub = &csconfig.Hub{}
+	cfg.Hub.ConfigDir, _ = filepath.Abs("./install")
+	cfg.Hub.HubDir, _ = filepath.Abs("./hubdir")
+	cfg.Hub.HubIndexFile = filepath.Clean("./hubdir/.index.json")
 
 	//Mock the http client
 	http.DefaultClient.Transport = newMockTransport()
 
-	if err := os.RemoveAll(cfg.Cscli.ConfigDir); err != nil {
-		log.Fatalf("failed to remove %s : %s", cfg.Cscli.ConfigDir, err)
+	if err := os.RemoveAll(cfg.Hub.ConfigDir); err != nil {
+		log.Fatalf("failed to remove %s : %s", cfg.Hub.ConfigDir, err)
 	}
 
-	if err := os.MkdirAll(cfg.Cscli.ConfigDir, 0700); err != nil {
+	if err := os.MkdirAll(cfg.Hub.ConfigDir, 0700); err != nil {
 		log.Fatalf("mkdir : %s", err)
 	}
 
-	if err := os.RemoveAll(cfg.Cscli.HubDir); err != nil {
-		log.Fatalf("failed to remove %s : %s", cfg.Cscli.HubDir, err)
+	if err := os.RemoveAll(cfg.Hub.HubDir); err != nil {
+		log.Fatalf("failed to remove %s : %s", cfg.Hub.HubDir, err)
 	}
-	if err := os.MkdirAll(cfg.Cscli.HubDir, 0700); err != nil {
-		log.Fatalf("failed to mkdir %s : %s", cfg.Cscli.HubDir, err)
+	if err := os.MkdirAll(cfg.Hub.HubDir, 0700); err != nil {
+		log.Fatalf("failed to mkdir %s : %s", cfg.Hub.HubDir, err)
 	}
 
-	if err := UpdateHubIdx(cfg.Cscli); err != nil {
+	if err := UpdateHubIdx(cfg.Hub); err != nil {
 		log.Fatalf("failed to download index : %s", err)
 	}
 
-	// if err := os.RemoveAll(cfg.Cscli.InstallDir); err != nil {
-	// 	log.Fatalf("failed to remove %s : %s", cfg.Cscli.InstallDir, err)
+	// if err := os.RemoveAll(cfg.Hub.InstallDir); err != nil {
+	// 	log.Fatalf("failed to remove %s : %s", cfg.Hub.InstallDir, err)
 	// }
-	// if err := os.MkdirAll(cfg.Cscli.InstallDir, 0700); err != nil {
-	// 	log.Fatalf("failed to mkdir %s : %s", cfg.Cscli.InstallDir, err)
+	// if err := os.MkdirAll(cfg.Hub.InstallDir, 0700); err != nil {
+	// 	log.Fatalf("failed to mkdir %s : %s", cfg.Hub.InstallDir, err)
 	// }
 	return cfg
 
 }
 
-func testInstallItem(cfg *csconfig.CscliCfg, t *testing.T, item Item) {
+func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
 
 	//Install the parser
 	item, err := DownloadLatest(cfg, item, false)
@@ -218,7 +218,7 @@ func testInstallItem(cfg *csconfig.CscliCfg, t *testing.T, item Item) {
 	}
 }
 
-func testTaintItem(cfg *csconfig.CscliCfg, t *testing.T, item Item) {
+func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) {
 	if hubIdx[item.Type][item.Name].Tainted {
 		t.Fatalf("pre-taint: %s should not be tainted", item.Name)
 	}
@@ -240,7 +240,7 @@ func testTaintItem(cfg *csconfig.CscliCfg, t *testing.T, item Item) {
 	}
 }
 
-func testUpdateItem(cfg *csconfig.CscliCfg, t *testing.T, item Item) {
+func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) {
 
 	if hubIdx[item.Type][item.Name].UpToDate {
 		t.Fatalf("update: %s should NOT be up-to-date", item.Name)
@@ -262,7 +262,7 @@ func testUpdateItem(cfg *csconfig.CscliCfg, t *testing.T, item Item) {
 	}
 }
 
-func testDisableItem(cfg *csconfig.CscliCfg, t *testing.T, item Item) {
+func testDisableItem(cfg *csconfig.Hub, t *testing.T, item Item) {
 	if !item.Installed {
 		t.Fatalf("disable: %s should be installed", item.Name)
 	}
@@ -314,20 +314,20 @@ func TestInstallParser(t *testing.T) {
 	*/
 	cfg := test_prepenv()
 
-	if err := GetHubIdx(cfg.Cscli); err != nil {
+	if err := GetHubIdx(cfg.Hub); err != nil {
 		t.Fatalf("failed to load hub index")
 	}
 	//map iteration is random by itself
 	for _, it := range hubIdx[PARSERS] {
-		testInstallItem(cfg.Cscli, t, it)
+		testInstallItem(cfg.Hub, t, it)
 		it = hubIdx[PARSERS][it.Name]
 		_ = HubStatus(PARSERS, it.Name, false)
-		testTaintItem(cfg.Cscli, t, it)
+		testTaintItem(cfg.Hub, t, it)
 		it = hubIdx[PARSERS][it.Name]
 		_ = HubStatus(PARSERS, it.Name, false)
-		testUpdateItem(cfg.Cscli, t, it)
+		testUpdateItem(cfg.Hub, t, it)
 		it = hubIdx[PARSERS][it.Name]
-		testDisableItem(cfg.Cscli, t, it)
+		testDisableItem(cfg.Hub, t, it)
 		it = hubIdx[PARSERS][it.Name]
 
 		break
@@ -347,18 +347,18 @@ func TestInstallCollection(t *testing.T) {
 	*/
 	cfg := test_prepenv()
 
-	if err := GetHubIdx(cfg.Cscli); err != nil {
+	if err := GetHubIdx(cfg.Hub); err != nil {
 		t.Fatalf("failed to load hub index")
 	}
 	//map iteration is random by itself
 	for _, it := range hubIdx[COLLECTIONS] {
-		testInstallItem(cfg.Cscli, t, it)
+		testInstallItem(cfg.Hub, t, it)
 		it = hubIdx[COLLECTIONS][it.Name]
-		testTaintItem(cfg.Cscli, t, it)
+		testTaintItem(cfg.Hub, t, it)
 		it = hubIdx[COLLECTIONS][it.Name]
-		testUpdateItem(cfg.Cscli, t, it)
+		testUpdateItem(cfg.Hub, t, it)
 		it = hubIdx[COLLECTIONS][it.Name]
-		testDisableItem(cfg.Cscli, t, it)
+		testDisableItem(cfg.Hub, t, it)
 
 		it = hubIdx[COLLECTIONS][it.Name]
 		x := HubStatus(COLLECTIONS, it.Name, false)

pkg/cwhub/download.go

@@ -22,9 +22,9 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
-func UpdateHubIdx(cscli *csconfig.CscliCfg) error {
+func UpdateHubIdx(hub *csconfig.Hub) error {
 
-	bidx, err := DownloadHubIdx(cscli)
+	bidx, err := DownloadHubIdx(hub)
 	if err != nil {
 		return errors.Wrap(err, "failed to download index")
 	}
@@ -35,13 +35,13 @@ func UpdateHubIdx(cscli *csconfig.CscliCfg) error {
 		}
 	}
 	hubIdx = ret
-	if err := LocalSync(cscli); err != nil {
+	if err := LocalSync(hub); err != nil {
 		return errors.Wrap(err, "failed to sync")
 	}
 	return nil
 }
 
-func DownloadHubIdx(cscli *csconfig.CscliCfg) ([]byte, error) {
+func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) {
 	log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile))
 	req, err := http.NewRequest("GET", fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil)
 	if err != nil {
@@ -59,7 +59,7 @@ func DownloadHubIdx(cscli *csconfig.CscliCfg) ([]byte, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to read request answer for hub index")
 	}
-	file, err := os.OpenFile(cscli.HubIndexFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+	file, err := os.OpenFile(hub.HubIndexFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
 
 	if err != nil {
 		return nil, errors.Wrap(err, "while opening hub index file")
@@ -70,12 +70,12 @@ func DownloadHubIdx(cscli *csconfig.CscliCfg) ([]byte, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "while writting hub index file")
 	}
-	log.Infof("Wrote new %d bytes index to %s", wsize, cscli.HubIndexFile)
+	log.Infof("Wrote new %d bytes index to %s", wsize, hub.HubIndexFile)
 	return body, nil
 }
 
 //DownloadLatest will download the latest version of Item to the tdir directory
-func DownloadLatest(cscli *csconfig.CscliCfg, target Item, overwrite bool) (Item, error) {
+func DownloadLatest(hub *csconfig.Hub, target Item, overwrite bool) (Item, error) {
 	var err error
 
 	log.Debugf("Downloading %s %s", target.Type, target.Name)
@@ -89,12 +89,12 @@ func DownloadLatest(cscli *csconfig.CscliCfg, target Item, overwrite bool) (Item
 					//recurse as it's a collection
 					if ptrtype == COLLECTIONS {
 						log.Tracef("collection, recurse")
-						hubIdx[ptrtype][p], err = DownloadLatest(cscli, val, overwrite)
+						hubIdx[ptrtype][p], err = DownloadLatest(hub, val, overwrite)
 						if err != nil {
 							return target, errors.Wrap(err, fmt.Sprintf("while downloading %s", val.Name))
 						}
 					}
-					item, err := DownloadItem(cscli, val, overwrite)
+					item, err := DownloadItem(hub, val, overwrite)
 					if err != nil {
 						return target, errors.Wrap(err, fmt.Sprintf("while downloading %s", val.Name))
 					}
@@ -102,7 +102,7 @@ func DownloadLatest(cscli *csconfig.CscliCfg, target Item, overwrite bool) (Item
 					// We need to enable an item when it has been added to a collection since latest release of the collection.
 					// We check if val.Downloaded is false because maybe the item has been disabled by the user.
 					if !item.Installed && !val.Downloaded {
-						if item, err = EnableItem(cscli, item); err != nil {
+						if item, err = EnableItem(hub, item); err != nil {
 							return target, errors.Wrapf(err, "enabling '%s'", item.Name)
 						}
 					}
@@ -112,20 +112,20 @@ func DownloadLatest(cscli *csconfig.CscliCfg, target Item, overwrite bool) (Item
 				}
 			}
 		}
-		target, err = DownloadItem(cscli, target, overwrite)
+		target, err = DownloadItem(hub, target, overwrite)
 		if err != nil {
 			return target, fmt.Errorf("failed to download item : %s", err)
 		}
 	} else {
-		return DownloadItem(cscli, target, overwrite)
+		return DownloadItem(hub, target, overwrite)
 	}
 	return target, nil
 }
 
-func DownloadItem(cscli *csconfig.CscliCfg, target Item, overwrite bool) (Item, error) {
+func DownloadItem(hub *csconfig.Hub, target Item, overwrite bool) (Item, error) {
 
-	var tdir = cscli.HubDir
-	var dataFolder = cscli.DataDir
+	var tdir = hub.HubDir
+	var dataFolder = hub.DataDir
 	/*if user didn't --force, don't overwrite local, tainted, up-to-date files*/
 	if !overwrite {
 		if target.Tainted {

pkg/cwhub/download_test.go

@@ -12,21 +12,27 @@ import (
 func TestDownloadHubIdx(t *testing.T) {
 	back := RawFileURLTemplate
 	//bad url template
+	fmt.Println("Test 'bad URL'")
 	RawFileURLTemplate = "x"
-	ret, err := DownloadHubIdx(&csconfig.CscliCfg{})
+	ret, err := DownloadHubIdx(&csconfig.Hub{})
 	if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed to build request for hub index: parse ") {
 		log.Errorf("unexpected error %s", err)
 	}
+	fmt.Printf("->%+v", ret)
+
 	//bad domain
+	fmt.Println("Test 'bad domain'")
 	RawFileURLTemplate = "https://baddomain/crowdsecurity/hub/%s/%s"
-	ret, err = DownloadHubIdx(&csconfig.CscliCfg{})
+	ret, err = DownloadHubIdx(&csconfig.Hub{})
 	if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed http request for hub index: Get") {
 		log.Errorf("unexpected error %s", err)
 	}
+	fmt.Printf("->%+v", ret)
 
 	//bad target path
+	fmt.Println("Test 'bad target path'")
 	RawFileURLTemplate = back
-	ret, err = DownloadHubIdx(&csconfig.CscliCfg{HubIndexFile: "/does/not/exist/index.json"})
+	ret, err = DownloadHubIdx(&csconfig.Hub{HubIndexFile: "/does/not/exist/index.json"})
 	if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "while opening hub index file: open /does/not/exist/index.json:") {
 		log.Errorf("unexpected error %s", err)
 	}

pkg/cwhub/install.go

@@ -11,9 +11,9 @@ import (
 )
 
 //DisableItem to disable an item managed by the hub, removes the symlink if purge is true
-func DisableItem(cscli *csconfig.CscliCfg, target Item, purge bool, force bool) (Item, error) {
-	var tdir = cscli.ConfigDir
-	var hdir = cscli.HubDir
+func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item, error) {
+	var tdir = hub.ConfigDir
+	var hdir = hub.HubDir
 
 	syml, err := filepath.Abs(tdir + "/" + target.Type + "/" + target.Stage + "/" + target.FileName)
 	if err != nil {
@@ -34,7 +34,7 @@ func DisableItem(cscli *csconfig.CscliCfg, target Item, purge bool, force bool)
 			ptrtype := ItemTypes[idx]
 			for _, p := range ptr {
 				if val, ok := hubIdx[ptrtype][p]; ok {
-					hubIdx[ptrtype][p], err = DisableItem(cscli, val, purge, force)
+					hubIdx[ptrtype][p], err = DisableItem(hub, val, purge, force)
 					if err != nil {
 						return target, errors.Wrap(err, fmt.Sprintf("while disabling %s", p))
 					}
@@ -51,7 +51,7 @@ func DisableItem(cscli *csconfig.CscliCfg, target Item, purge bool, force bool)
 			return target, fmt.Errorf("can't delete %s : %s doesn't exist", target.Name, syml)
 		}
 	} else {
-		//if it's managed by hub, it's a symlink to csconfig.GConfig.Cscli.HubDir / ...
+		//if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ...
 		if stat.Mode()&os.ModeSymlink == 0 {
 			log.Warningf("%s (%s) isn't a symlink, can't disable", target.Name, syml)
 			return target, fmt.Errorf("%s isn't managed by hub", target.Name)
@@ -90,9 +90,9 @@ func DisableItem(cscli *csconfig.CscliCfg, target Item, purge bool, force bool)
 	return target, nil
 }
 
-func EnableItem(cscli *csconfig.CscliCfg, target Item) (Item, error) {
-	var tdir = cscli.ConfigDir
-	var hdir = cscli.HubDir
+func EnableItem(hub *csconfig.Hub, target Item) (Item, error) {
+	var tdir = hub.ConfigDir
+	var hdir = hub.HubDir
 	var err error
 	parent_dir := filepath.Clean(tdir + "/" + target.Type + "/" + target.Stage + "/")
 	/*create directories if needed*/
@@ -123,7 +123,7 @@ func EnableItem(cscli *csconfig.CscliCfg, target Item) (Item, error) {
 			ptrtype := ItemTypes[idx]
 			for _, p := range ptr {
 				if val, ok := hubIdx[ptrtype][p]; ok {
-					hubIdx[ptrtype][p], err = EnableItem(cscli, val)
+					hubIdx[ptrtype][p], err = EnableItem(hub, val)
 					if err != nil {
 						return target, errors.Wrap(err, fmt.Sprintf("while installing %s", p))
 					}

pkg/cwhub/loader.go

@@ -49,7 +49,7 @@ func parser_visit(path string, f os.FileInfo, err error) error {
 	subs := strings.Split(path, "/")
 
 	log.Tracef("path:%s, hubdir:%s, installdir:%s", path, hubdir, installdir)
-	/*we're in hub (~/.cscli/hub/)*/
+	/*we're in hub (~/.hub/hub/)*/
 	if strings.HasPrefix(path, hubdir) {
 		log.Tracef("in hub dir")
 		inhub = true
@@ -95,7 +95,7 @@ func parser_visit(path string, f os.FileInfo, err error) error {
 
 	/*
 		we can encounter 'collections' in the form of a symlink :
-		/etc/crowdsec/.../collections/linux.yaml -> ~/.cscli/hub/collections/.../linux.yaml
+		/etc/crowdsec/.../collections/linux.yaml -> ~/.hub/hub/collections/.../linux.yaml
 		when the collection is installed, both files are created
 	*/
 	//non symlinks are local user files or hub files
@@ -108,7 +108,7 @@ func parser_visit(path string, f os.FileInfo, err error) error {
 		if err != nil {
 			return fmt.Errorf("unable to read symlink of %s", path)
 		}
-		//the symlink target doesn't exist, user might have remove ~/.cscli/hub/...yaml without deleting /etc/crowdsec/....yaml
+		//the symlink target doesn't exist, user might have remove ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml
 		_, err := os.Lstat(hubpath)
 		if os.IsNotExist(err) {
 			log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath)
@@ -294,10 +294,10 @@ func CollecDepsCheck(v *Item) error {
 	return nil
 }
 
-func SyncDir(cscli *csconfig.CscliCfg, dir string) error {
-	hubdir = cscli.HubDir
-	installdir = cscli.ConfigDir
-	indexpath = cscli.HubIndexFile
+func SyncDir(hub *csconfig.Hub, dir string) error {
+	hubdir = hub.HubDir
+	installdir = hub.ConfigDir
+	indexpath = hub.HubIndexFile
 
 	/*For each, scan PARSERS, PARSERS_OVFLW, SCENARIOS and COLLECTIONS last*/
 	for _, scan := range ItemTypes {
@@ -322,13 +322,13 @@ func SyncDir(cscli *csconfig.CscliCfg, dir string) error {
 }
 
 /* Updates the infos from HubInit() with the local state */
-func LocalSync(cscli *csconfig.CscliCfg) error {
+func LocalSync(hub *csconfig.Hub) error {
 	skippedLocal = 0
 	skippedTainted = 0
 
-	for _, dir := range []string{cscli.ConfigDir, cscli.HubDir} {
+	for _, dir := range []string{hub.ConfigDir, hub.HubDir} {
 		log.Debugf("scanning %s", dir)
-		if err := SyncDir(cscli, dir); err != nil {
+		if err := SyncDir(hub, dir); err != nil {
 			return fmt.Errorf("failed to scan %s : %s", dir, err)
 		}
 	}
@@ -336,12 +336,12 @@ func LocalSync(cscli *csconfig.CscliCfg) error {
 	return nil
 }
 
-func GetHubIdx(cscli *csconfig.CscliCfg) error {
-	if cscli == nil {
-		return fmt.Errorf("no configuration found for cscli")
+func GetHubIdx(hub *csconfig.Hub) error {
+	if hub == nil {
+		return fmt.Errorf("no configuration found for hub")
 	}
-	log.Debugf("loading hub idx %s", cscli.HubIndexFile)
-	bidx, err := ioutil.ReadFile(cscli.HubIndexFile)
+	log.Debugf("loading hub idx %s", hub.HubIndexFile)
+	bidx, err := ioutil.ReadFile(hub.HubIndexFile)
 	if err != nil {
 		return errors.Wrap(err, "unable to read index file")
 	}
@@ -353,7 +353,7 @@ func GetHubIdx(cscli *csconfig.CscliCfg) error {
 		return err
 	}
 	hubIdx = ret
-	if err := LocalSync(cscli); err != nil {
+	if err := LocalSync(hub); err != nil {
 		log.Fatalf("Failed to sync Hub index with local deployment : %v", err)
 	}
 	return nil

pkg/database/database.go

@@ -69,7 +69,6 @@ func NewClient(config *csconfig.DatabaseCfg) (*Client, error) {
 	if err := types.ConfigureLogger(clog); err != nil {
 		return nil, errors.Wrap(err, "while configuring db logger")
 	}
-	log.Infof("Log level: %+v", config.LogLevel)
 	if config.LogLevel != nil {
 		clog.SetLevel(*config.LogLevel)
 		if *config.LogLevel >= log.DebugLevel {

pkg/parser/unix_parser.go

@@ -1,8 +1,8 @@
 package parser
 
 import (
-	"io/ioutil"
 	"fmt"
+	"io/ioutil"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 
@@ -45,8 +45,7 @@ func Init(c map[string]interface{}) (*UnixParserCtx, error) {
 	return &r, nil
 }
 
-
-func LoadParsers(cConfig *csconfig.GlobalConfig, parsers *Parsers) (*Parsers, error) {
+func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
 	var err error
 
 	log.Infof("Loading grok library %s", cConfig.Crowdsec.ConfigDir+string("/patterns/"))

scripts/func_tests/config/config.yaml

@@ -0,0 +1,43 @@
+common:
+  daemonize: true
+  pid_dir: /var/run/
+  log_media: file
+  log_level: info
+  log_dir: /var/log/
+  working_dir: .
+config_paths:
+  config_dir: /etc/crowdsec/
+  data_dir: /var/lib/crowdsec/data/
+  simulation_path: /etc/crowdsec/simulation.yaml
+  hub_dir: /etc/crowdsec/hub/
+  index_path: /etc/crowdsec/hub/.index.json
+crowdsec_service:
+  acquisition_path: /etc/crowdsec/acquis.yaml
+  parser_routines: 1
+cscli:
+  output: human
+db_config:
+  log_level: info
+  type: sqlite
+  db_path: /var/lib/crowdsec/data/crowdsec.db
+  flush:
+    max_items: 5000
+    max_age: 7d
+api:
+  client:
+    insecure_skip_verify: false
+    credentials_path: /etc/crowdsec/local_api_credentials.yaml
+  server:
+    log_level: info
+    listen_uri: 127.0.0.1:8080
+    profiles_path: /etc/crowdsec/profiles.yaml
+    online_client: # Crowdsec API credentials (to push signals and receive bad IPs)
+      credentials_path: /etc/crowdsec/online_api_credentials.yaml
+#    tls:
+#      cert_file: /etc/crowdsec/ssl/cert.pem
+#      key_file: /etc/crowdsec/ssl/key.pem
+prometheus:
+  enabled: true
+  level: full
+  listen_addr: 127.0.0.1
+  listen_port: 6060

scripts/func_tests/config/config_no_agent.yaml

@@ -0,0 +1,41 @@
+
+common:
+  daemonize: true
+  pid_dir: /var/run/
+  log_media: file
+  log_level: info
+  log_dir: ./
+  working_dir: .
+config_paths:
+  config_dir: /etc/crowdsec/
+  data_dir: /var/lib/crowdsec/data/
+  simulation_path: /etc/crowdsec/simulation.yaml
+  hub_dir: /etc/crowdsec/hub/
+  index_path: /etc/crowdsec/hub/.index.json
+cscli:
+  output: human
+db_config:
+  log_level: info
+  type: sqlite
+  db_path: /var/lib/crowdsec/data/crowdsec.db
+  flush:
+    max_items: 5000
+    max_age: 7d
+api:
+  client:
+    insecure_skip_verify: false
+    credentials_path: /etc/crowdsec/local_api_credentials.yaml
+  server:
+    log_level: info
+    listen_uri: 127.0.0.1:8080
+    profiles_path: /etc/crowdsec/profiles.yaml
+    online_client: # Crowdsec API credentials (to push signals and receive bad IPs)
+      credentials_path: /etc/crowdsec/online_api_credentials.yaml
+#    tls:
+#      cert_file: /etc/crowdsec/ssl/cert.pem
+#      key_file: /etc/crowdsec/ssl/key.pem
+prometheus:
+  enabled: true
+  level: full
+  listen_addr: 127.0.0.1
+  listen_port: 6060

scripts/func_tests/config/config_no_capi.yaml

@@ -0,0 +1,38 @@
+common:
+  daemonize: true
+  pid_dir: /var/run/
+  log_media: file
+  log_level: info
+  log_dir: ./
+  working_dir: .
+config_paths:
+  config_dir: /etc/crowdsec/
+  data_dir: /var/lib/crowdsec/data/
+  simulation_path: /etc/crowdsec/simulation.yaml
+  hub_dir: /etc/crowdsec/hub/
+  index_path: /etc/crowdsec/hub/.index.json
+crowdsec_service:
+  acquisition_path: /etc/crowdsec/acquis.yaml
+  parser_routines: 1
+cscli:
+  output: human
+db_config:
+  log_level: info
+  type: sqlite
+  db_path: /var/lib/crowdsec/data/crowdsec.db
+  flush:
+    max_items: 5000
+    max_age: 7d
+api:
+  client:
+    insecure_skip_verify: false
+    credentials_path: /etc/crowdsec/local_api_credentials.yaml
+  server:
+    log_level: info
+    listen_uri: 127.0.0.1:8080
+    profiles_path: /etc/crowdsec/profiles.yaml
+prometheus:
+  enabled: true
+  level: full
+  listen_addr: 127.0.0.1
+  listen_port: 6060

scripts/func_tests/config/config_no_lapi.yaml

@@ -0,0 +1,34 @@
+common:
+  daemonize: true
+  pid_dir: /var/run/
+  log_media: file
+  log_level: info
+  log_dir: ./
+  working_dir: .
+config_paths:
+  config_dir: /etc/crowdsec/
+  data_dir: /var/lib/crowdsec/data/
+  simulation_path: /etc/crowdsec/simulation.yaml
+  hub_dir: /etc/crowdsec/hub/
+  index_path: /etc/crowdsec/hub/.index.json
+crowdsec_service:
+  acquisition_path: /etc/crowdsec/acquis.yaml
+  parser_routines: 1
+cscli:
+  output: human
+db_config:
+  log_level: info
+  type: sqlite
+  db_path: /var/lib/crowdsec/data/crowdsec.db
+  flush:
+    max_items: 5000
+    max_age: 7d
+api:
+  client:
+    insecure_skip_verify: false
+    credentials_path: /etc/crowdsec/local_api_credentials.yaml
+prometheus:
+  enabled: true
+  level: full
+  listen_addr: 127.0.0.1
+  listen_port: 6060

scripts/func_tests/systemd/crowdsec.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Crowdsec agent
+After=syslog.target network.target remote-fs.target nss-lookup.target
+
+[Service]
+Type=notify
+Environment=LC_ALL=C LANG=C
+PIDFile=/var/run/crowdsec.pid
+ExecStartPre=/usr/local/bin/crowdsec -c /etc/crowdsec/config.yaml -t
+ExecStart=/usr/local/bin/crowdsec -c /etc/crowdsec/config.yaml
+#ExecStartPost=/bin/sleep 0.1
+ExecReload=/bin/kill -HUP $MAINPID
+
+[Install]
+WantedBy=multi-user.target

scripts/func_tests/systemd/crowdsec_no_agent.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Crowdsec agent
+After=syslog.target network.target remote-fs.target nss-lookup.target
+
+[Service]
+Type=notify
+Environment=LC_ALL=C LANG=C
+PIDFile=/var/run/crowdsec.pid
+ExecStartPre=/usr/local/bin/crowdsec -c /etc/crowdsec/config.yaml -t
+ExecStart=/usr/local/bin/crowdsec -c /etc/crowdsec/config.yaml -no-cs
+#ExecStartPost=/bin/sleep 0.1
+ExecReload=/bin/kill -HUP $MAINPID
+
+[Install]
+WantedBy=multi-user.target

scripts/func_tests/systemd/crowdsec_no_lapi.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Crowdsec agent
+After=syslog.target network.target remote-fs.target nss-lookup.target
+
+[Service]
+Type=notify
+Environment=LC_ALL=C LANG=C
+PIDFile=/var/run/crowdsec.pid
+ExecStartPre=/usr/local/bin/crowdsec -c /etc/crowdsec/config.yaml -t
+ExecStart=/usr/local/bin/crowdsec -c /etc/crowdsec/config.yaml -no-api
+#ExecStartPost=/bin/sleep 0.1
+ExecReload=/bin/kill -HUP $MAINPID
+
+[Install]
+WantedBy=multi-user.target

scripts/func_tests/tests_base.sh

@@ -13,7 +13,7 @@ JQ="jq -e"
 SYSTEMCTL="sudo systemctl --no-pager"
 
 CROWDSEC="sudo crowdsec"
-
+CROWDSEC_PROCESS="crowdsec"
 # helpers
 function fail {
     echo "ACTION FAILED, STOP : $@"

scripts/func_tests/tests_post-install_0base.sh

@@ -5,9 +5,13 @@ source tests_base.sh
 
 
 
+##########################
+## TEST AGENT/LAPI/CAPI ##
+echo "CROWDSEC (AGENT+LAPI+CAPI)"
+
 ## status / start / stop
 # service should be up
-pidof crowdsec || fail "crowdsec process shouldn't be running"
+pidof crowdsec || fail "crowdsec process should be running"
 ${SYSTEMCTL} status crowdsec || fail "systemctl status crowdsec failed"
 
 #shut it down
@@ -18,43 +22,125 @@ pidof crowdsec && fail "crowdsec process shouldn't be running"
 #start it again
 ${SYSTEMCTL} start crowdsec || fail "failed to stop service"
 ${SYSTEMCTL} status crowdsec || fail "crowdsec should be down"
-pidof crowdsec || fail "crowdsec process shouldn't be running"
+pidof crowdsec || fail "crowdsec process should be running"
 
 #restart it
 ${SYSTEMCTL} restart crowdsec || fail "failed to stop service"
 ${SYSTEMCTL} status crowdsec || fail "crowdsec should be down"
-pidof crowdsec || fail "crowdsec process shouldn't be running"
-
-
-
+pidof crowdsec || fail "crowdsec process should be running"
 
 ## version
-
 ${CSCLI} version || fail "cannot run cscli version"
 
-
 ## alerts
-
 # alerts list at startup should just return one entry : comunity pull
 sleep 5
 ${CSCLI} alerts list -ojson  | ${JQ} '. | length >= 1' || fail "expected at least one entry from cscli alerts list"
+## capi
+${CSCLI} capi status || fail "capi status should be ok"
+## config
+${CSCLI} config show || fail "failed to show config"
+${CSCLI} config backup ./test || fail "failed to backup config"
+sudo rm -rf ./test
+## lapi
+${CSCLI} lapi status || fail "lapi status failed"
+## metrics
+${CSCLI} metrics || fail "failed to get metrics"
 
+${SYSTEMCTL} stop crowdsec || fail "crowdsec should be down"
 
-## capi
+#######################
+## TEST WITHOUT LAPI ##
 
-${CSCLI} capi status || fail "capi status should be ok"
+echo "CROWDSEC (AGENT)"
+
+# test with -no-api flag
+sudo cp ./systemd/crowdsec_no_lapi.service /etc/systemd/system/crowdsec.service
+${SYSTEMCTL} daemon-reload
+${SYSTEMCTL} start crowdsec
+sleep 1
+pidof crowdsec && fail "crowdsec shouldn't run without LAPI (in flag)"
+${SYSTEMCTL} stop crowdsec
 
+sudo cp ./systemd/crowdsec.service /etc/systemd/system/crowdsec.service
+${SYSTEMCTL} daemon-reload
 
+# test with no api server in configuration file
+sudo cp ./config/config_no_lapi.yaml /etc/crowdsec/config.yaml
+${SYSTEMCTL} start crowdsec
+sleep 1
+pidof crowdsec && fail "crowdsec agent should not run without lapi (in configuration file)"
+
+##### cscli test ####
+## capi
+${CSCLI} -c ./config/config_no_lapi.yaml capi status && fail "capi status shouldn't be ok"
 ## config
+${CSCLI_BIN} -c ./config/config_no_lapi.yaml config show || fail "failed to show config"
+${CSCLI} -c ./config/config_no_lapi.yaml config backup ./test || fail "failed to backup config"
+sudo rm -rf ./test
+## lapi
+${CSCLI} -c ./config/config_no_lapi.yaml lapi status && fail "lapi status should not be ok" ## if lapi status success, it means that the test fail
+## metrics
+${CSCLI_BIN} -c ./config/config_no_lapi.yaml metrics
 
-${CSCLI} config show || fail "failed to show config"
+${SYSTEMCTL} stop crowdsec
+sudo cp ./config/config.yaml /etc/crowdsec/config.yaml
 
-${CSCLI} config backup ./test || fail "failed to backup config"
+########################
+## TEST WITHOUT AGENT ##
+
+echo "CROWDSEC (LAPI+CAPI)"
 
+# test with -no-cs flag
+sudo cp ./systemd/crowdsec_no_agent.service /etc/systemd/system/crowdsec.service
+${SYSTEMCTL} daemon-reload
+${SYSTEMCTL} start crowdsec 
+pidof crowdsec || fail "crowdsec LAPI should run without agent (in flag)"
+${SYSTEMCTL} stop crowdsec
+
+sudo cp ./systemd/crowdsec.service /etc/systemd/system/crowdsec.service
+${SYSTEMCTL} daemon-reload
+
+# test with no crowdsec agent in configuration file
+sudo cp ./config/config_no_agent.yaml /etc/crowdsec/config.yaml
+${SYSTEMCTL} start crowdsec 
+pidof crowdsec || fail "crowdsec LAPI should run without agent (in configuration file)"
+
+
+## capi
+${CSCLI} -c ./config/config_no_agent.yaml capi status || fail "capi status should be ok"
+## config
+${CSCLI_BIN} -c ./config/config_no_agent.yaml config show || fail "failed to show config"
+${CSCLI} -c ./config/config_no_agent.yaml config backup ./test || fail "failed to backup config"
+sudo rm -rf ./test
 ## lapi
+${CSCLI} -c ./config/config_no_agent.yaml lapi status || fail "lapi status failed"
+## metrics
+${CSCLI_BIN} -c ./config/config_no_agent.yaml metrics || fail "failed to get metrics"
 
-${CSCLI} lapi status || fail "lapi status failed"
+${SYSTEMCTL} stop crowdsec
+sudo cp ./config/config.yaml /etc/crowdsec/config.yaml
+
+
+#######################
+## TEST WITHOUT CAPI ##
+echo "CROWDSEC (AGENT+LAPI)"
 
+# test with no online client in configuration file
+sudo cp ./config/config_no_capi.yaml /etc/crowdsec/config.yaml
+${SYSTEMCTL} start crowdsec 
+pidof crowdsec || fail "crowdsec LAPI should run without CAPI (in configuration file)"
+
+## capi
+${CSCLI} -c ./config/config_no_capi.yaml capi status && fail "capi status should not be ok" ## if capi status success, it means that the test fail
+## config
+${CSCLI_BIN} -c ./config/config_no_capi.yaml config show || fail "failed to show config"
+${CSCLI} -c ./config/config_no_capi.yaml config backup ./test || fail "failed to backup config"
+sudo rm -rf ./test
+## lapi
+${CSCLI} -c ./config/config_no_capi.yaml lapi status || fail "lapi status failed"
 ## metrics
-${CSCLI} metrics || fail "failed to get metrics"
+${CSCLI_BIN} -c ./config/config_no_capi.yaml metrics || fail "failed to get metrics"
 
+sudo cp ./config/config.yaml /etc/crowdsec/config.yaml
+${SYSTEMCTL} restart crowdsec

scripts/func_tests/tests_post-install_2collections.sh

@@ -5,12 +5,12 @@ source tests_base.sh
 
 ## collections
 
-${CSCLI} collections list || fail "failed to list collections"
+${CSCLI_BIN} collections list || fail "failed to list collections"
 
 BASE_COLLECTION_COUNT=2
 
 # we expect 1 collections : linux 
-${CSCLI} collections list -ojson | ${JQ} ". | length == ${BASE_COLLECTION_COUNT}" || fail "(first) expected exactly ${BASE_COLLECTION_COUNT} collection"
+${CSCLI_BIN} collections list -ojson | ${JQ} ". | length == ${BASE_COLLECTION_COUNT}" || fail "(first) expected exactly ${BASE_COLLECTION_COUNT} collection"
 
 # install an extra collection
 ${CSCLI} collections install crowdsecurity/mysql || fail "failed to install collection"
@@ -18,7 +18,7 @@ ${CSCLI} collections install crowdsecurity/mysql || fail "failed to install coll
 BASE_COLLECTION_COUNT=$(($BASE_COLLECTION_COUNT+1))
 
 # we should now have 2 collections :)
-${CSCLI} collections list -ojson | ${JQ} ". | length == ${BASE_COLLECTION_COUNT}" || fail "(post install) expected exactly ${BASE_COLLECTION_COUNT} collection"
+${CSCLI_BIN} collections list -ojson | ${JQ} ". | length == ${BASE_COLLECTION_COUNT}" || fail "(post install) expected exactly ${BASE_COLLECTION_COUNT} collection"
 
 # remove the collection
 ${CSCLI} collections remove crowdsecurity/mysql || fail "failed to remove collection"
@@ -26,5 +26,5 @@ ${CSCLI} collections remove crowdsecurity/mysql || fail "failed to remove collec
 BASE_COLLECTION_COUNT=$(($BASE_COLLECTION_COUNT-1))
 
 # we expect 1 collections : linux 
-${CSCLI} collections list -ojson | ${JQ} ". | length == ${BASE_COLLECTION_COUNT}" || fail "(post remove) expected exactly ${BASE_COLLECTION_COUNT} collection"
+${CSCLI_BIN} collections list -ojson | ${JQ} ". | length == ${BASE_COLLECTION_COUNT}" || fail "(post remove) expected exactly ${BASE_COLLECTION_COUNT} collection"