Merge pull request #26 from crowdsecurity/merge_cscli_crowdsec_config

Merge cscli crowdsec config

cmd/crowdsec-cli/ban.go

@@ -190,7 +190,6 @@ You can add/delete/list or flush current bans in your local ban DB.`,
 			return nil
 		},
 	}
-	cmdBan.PersistentFlags().StringVar(&config.dbPath, "db", "", "Set path to SQLite DB.")
 	cmdBan.PersistentFlags().StringVar(&remediationType, "remediation", "ban", "Set specific remediation type : ban|slow|captcha")
 	cmdBan.Flags().SortFlags = false
 	cmdBan.PersistentFlags().SortFlags = false

cmd/crowdsec-cli/config.go

@@ -1,12 +1,7 @@
 package main
 
 import (
-	"bufio"
 	"fmt"
-	"io/ioutil"
-	"os"
-	"path"
-	"strings"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -16,67 +11,13 @@ import (
 /*CliCfg is the cli configuration structure, might be unexported*/
 type cliConfig struct {
 	configured          bool
-	configFolder        string `yaml:"cliconfig,omitempty"` /*overload ~/.cscli/*/
-	output              string /*output is human, json*/
-	hubFolder           string
-	InstallFolder       string `yaml:"installdir"` /*/etc/crowdsec/*/
-	BackendPluginFolder string `yaml:"backend"`
-	dbPath              string
-}
-
-func interactiveCfg() error {
-	var err error
-	reader := bufio.NewReader(os.Stdin)
-	fmt.Print("crowdsec installation directory (default: /etc/crowdsec/config/): ")
-	config.InstallFolder, err = reader.ReadString('\n')
-	config.InstallFolder = strings.Replace(config.InstallFolder, "\n", "", -1) //CRLF to LF (windows)
-	if config.InstallFolder == "" {
-		config.InstallFolder = "/etc/crowdsec/config/"
-	}
-	if err != nil {
-		log.Fatalf("failed to read input : %v", err.Error())
-	}
-
-	fmt.Print("crowdsec backend plugin directory (default: /etc/crowdsec/plugin/backend): ")
-	config.BackendPluginFolder, err = reader.ReadString('\n')
-	config.BackendPluginFolder = strings.Replace(config.BackendPluginFolder, "\n", "", -1) //CRLF to LF (windows)
-	if config.BackendPluginFolder == "" {
-		config.BackendPluginFolder = "/etc/crowdsec/plugin/backend"
-	}
-	if err != nil {
-		log.Fatalf("failed to read input : %v", err.Error())
-	}
-	if err := writeCfg(); err != nil {
-		log.Fatalf("failed writting configuration file : %s", err)
-	}
-	return nil
-}
-
-func writeCfg() error {
-
-	if config.configFolder == "" {
-		return fmt.Errorf("config dir is unset")
-	}
-
-	config.hubFolder = config.configFolder + "/hub/"
-	if _, err := os.Stat(config.hubFolder); os.IsNotExist(err) {
-
-		log.Warningf("creating skeleton!")
-		if err := os.MkdirAll(config.hubFolder, os.ModePerm); err != nil {
-			return fmt.Errorf("failed to create missing directory : '%s'", config.hubFolder)
-		}
-	}
-	out := path.Join(config.configFolder, "/config")
-	configYaml, err := yaml.Marshal(&config)
-	if err != nil {
-		return fmt.Errorf("failed marshaling config: %s", err)
-	}
-	err = ioutil.WriteFile(out, configYaml, 0644)
-	if err != nil {
-		return fmt.Errorf("failed to write to %s : %s", out, err)
-	}
-	log.Infof("wrote config to %s ", out)
-	return nil
+	ConfigFilePath      string `yaml:"config_file"`
+	configFolder        string
+	output              string
+	HubFolder           string `yaml:"hub_folder"`
+	InstallFolder       string
+	BackendPluginFolder string `yaml:"backend_folder"`
+	DataFolder          string `yaml:"data_folder"`
 }
 
 func NewConfigCmd() *cobra.Command {
@@ -98,8 +39,9 @@ If no commands are specified, config is in interactive mode.`,
 		Run: func(cmd *cobra.Command, args []string) {
 			if config.output == "json" {
 				log.WithFields(log.Fields{
-					"installdir": config.InstallFolder,
-					"cliconfig":  path.Join(config.configFolder, "/config"),
+					"crowdsec_configuration_file": config.ConfigFilePath,
+					"backend_folder":              config.BackendPluginFolder,
+					"data_folder":                 config.DataFolder,
 				}).Warning("Current config")
 			} else {
 				x, err := yaml.Marshal(config)
@@ -107,52 +49,9 @@ If no commands are specified, config is in interactive mode.`,
 					log.Fatalf("failed to marshal current configuration : %v", err)
 				}
 				fmt.Printf("%s", x)
-				fmt.Printf("#cliconfig: %s", path.Join(config.configFolder, "/config"))
 			}
 		},
 	}
 	cmdConfig.AddCommand(cmdConfigShow)
-	var cmdConfigInterctive = &cobra.Command{
-		Use:   "prompt",
-		Short: "Prompt for configuration values in an interactive fashion",
-		Long:  `Start interactive configuration of cli. It will successively ask for install dir, db path.`,
-		Args:  cobra.ExactArgs(0),
-		Run: func(cmd *cobra.Command, args []string) {
-			err := interactiveCfg()
-			if err != nil {
-				log.Fatalf("Failed to run interactive config : %s", err)
-			}
-			log.Warningf("Configured, please run update.")
-		},
-	}
-	cmdConfig.AddCommand(cmdConfigInterctive)
-	var cmdConfigInstalldir = &cobra.Command{
-		Use:   "installdir [value]",
-		Short: `Configure installation directory`,
-		Long:  `Configure the installation directory of crowdsec, such as /etc/crowdsec/config/`,
-		Args:  cobra.ExactArgs(1),
-		Run: func(cmd *cobra.Command, args []string) {
-			config.InstallFolder = args[0]
-			if err := writeCfg(); err != nil {
-				log.Fatalf("failed writting configuration: %s", err)
-			}
-		},
-	}
-	cmdConfig.AddCommand(cmdConfigInstalldir)
-
-	var cmdConfigBackendFolder = &cobra.Command{
-		Use:   "backend [value]",
-		Short: `Configure installation directory`,
-		Long:  `Configure the backend plugin directory of crowdsec, such as /etc/crowdsec/plugins/backend`,
-		Args:  cobra.ExactArgs(1),
-		Run: func(cmd *cobra.Command, args []string) {
-			config.BackendPluginFolder = args[0]
-			if err := writeCfg(); err != nil {
-				log.Fatalf("failed writting configuration: %s", err)
-			}
-		},
-	}
-	cmdConfig.AddCommand(cmdConfigBackendFolder)
-
 	return cmdConfig
 }

cmd/crowdsec-cli/main.go

@@ -1,18 +1,17 @@
 package main
 
 import (
-	"io/ioutil"
 	"os/user"
 	"path/filepath"
 	"strings"
 
+	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/cobra/doc"
-	"gopkg.in/yaml.v2"
 )
 
 var dbg_lvl, nfo_lvl, wrn_lvl, err_lvl bool
@@ -37,6 +36,12 @@ func initConfig() {
 		log.SetLevel(log.ErrorLevel)
 	}
 
+	csConfig := csconfig.NewCrowdSecConfig()
+	if err := csConfig.GetCliConfig(&config.ConfigFilePath); err != nil {
+		log.Fatalf(err.Error())
+	}
+	config.configFolder = filepath.Clean(csConfig.CsCliFolder)
+
 	if strings.HasPrefix(config.configFolder, "~/") {
 		usr, err := user.Current()
 		if err != nil {
@@ -44,24 +49,17 @@ func initConfig() {
 		}
 		config.configFolder = usr.HomeDir + "/" + config.configFolder[2:]
 	}
+
 	/*read config*/
-	buf, err := ioutil.ReadFile(filepath.Clean(config.configFolder + "/config"))
-	if err != nil {
-		log.Infof("Failed to open config %s : %s", filepath.Clean(config.configFolder+"/config"), err)
-	} else {
-		err = yaml.UnmarshalStrict(buf, &config)
-		if err != nil {
-			log.Fatalf("Failed to parse config %s : %s, please configure", filepath.Clean(config.configFolder+"/config"), err)
-		}
-		config.InstallFolder = filepath.Clean(config.InstallFolder)
-		config.hubFolder = filepath.Clean(config.configFolder + "/hub/")
-		config.BackendPluginFolder = filepath.Clean(config.BackendPluginFolder)
-		//
-		cwhub.Installdir = config.InstallFolder
-		cwhub.Cfgdir = config.configFolder
-		cwhub.Hubdir = config.hubFolder
-		config.configured = true
-	}
+	config.InstallFolder = filepath.Clean(csConfig.ConfigFolder)
+	config.HubFolder = filepath.Clean(config.configFolder + "/hub/")
+	config.BackendPluginFolder = filepath.Clean(csConfig.OutputConfig.BackendFolder)
+	config.DataFolder = filepath.Clean(csConfig.DataFolder)
+	//
+	cwhub.Installdir = config.InstallFolder
+	cwhub.Cfgdir = config.configFolder
+	cwhub.Hubdir = config.HubFolder
+	config.configured = true
 }
 
 func main() {
@@ -112,7 +110,8 @@ API interaction:
 	rootCmd.AddCommand(cmdVersion)
 
 	//rootCmd.PersistentFlags().BoolVarP(&config.simulation, "simulate", "s", false, "No action; perform a simulation of events that would occur based on the current arguments.")
-	rootCmd.PersistentFlags().StringVarP(&config.configFolder, "config-dir", "c", "/etc/crowdsec/cscli/", "Configuration directory to use.")
+	rootCmd.PersistentFlags().StringVarP(&config.ConfigFilePath, "config", "c", "/etc/crowdsec/config/default.yaml", "path to crowdsec config file (default: /etc/crowdsec/config/default.yaml)")
+
 	rootCmd.PersistentFlags().StringVarP(&config.output, "output", "o", "human", "Output format : human, json, raw.")
 	rootCmd.PersistentFlags().BoolVar(&dbg_lvl, "debug", false, "Set logging to debug.")
 	rootCmd.PersistentFlags().BoolVar(&nfo_lvl, "info", false, "Set logging to info.")

config/dev.yaml

@@ -2,6 +2,7 @@ working_dir: "."
 data_dir: "./data"
 config_dir: "./config"
 pid_dir: "./"
+cscli_dir: "./config/crowdsec-cli"
 log_dir: "./logs"
 log_mode: "stdout"
 log_level: info

config/prod.yaml

@@ -3,6 +3,7 @@ data_dir: ${DATA}
 config_dir: ${CFG}
 pid_dir: ${PID}
 log_dir: /var/log/
+cscli_dir: ${CFG}/cscli
 log_mode: file
 log_level: info
 profiling: false

pkg/csconfig/config.go

@@ -31,6 +31,7 @@ type CrowdSec struct {
 	Profiling       bool      `yaml:"profiling,omitempty"`   //true -> enable runtime profiling
 	SQLiteFile      string    `yaml:"sqlite_path,omitempty"` //path to sqlite output
 	APIMode         bool      `yaml:"apimode,omitempty"`     //true -> enable api push
+	CsCliFolder     string    `yaml:"cscli_dir"`             //cscli folder
 	Linter          bool
 	Prometheus      bool
 	HTTPListen      string `yaml:"http_listen,omitempty"`
@@ -59,6 +60,24 @@ func NewCrowdSecConfig() *CrowdSec {
 	}
 }
 
+func (c *CrowdSec) GetCliConfig(configFile *string) error {
+	/*overriden by cfg file*/
+	if *configFile != "" {
+		rcfg, err := ioutil.ReadFile(*configFile)
+		if err != nil {
+			return fmt.Errorf("read '%s' : %s", *configFile, err)
+		}
+		if err := yaml.UnmarshalStrict(rcfg, c); err != nil {
+			return fmt.Errorf("parse '%s' : %s", *configFile, err)
+		}
+		if c.AcquisitionFile == "" {
+			c.AcquisitionFile = filepath.Clean(c.ConfigFolder + "/acquis.yaml")
+		}
+	}
+	return nil
+
+}
+
 // GetOPT return flags parsed from command line
 func (c *CrowdSec) GetOPT() error {
 

pkg/cwhub/hubMgmt.go

@@ -42,8 +42,8 @@ type ItemVersion struct {
 //Item can be : parsed, scenario, collection
 type Item struct {
 	/*descriptive info*/
-	Type                 string   `yaml:"type,omitempty"`                        //parser|postoverflows|scenario|collection(|enrich)
-	Stage                string   `json:"stage" yaml:"type,omitempty,omitempty"` //Stage for parser|postoverflow : s00-raw/s01-...
+	Type                 string   `yaml:"type,omitempty"`                         //parser|postoverflows|scenario|collection(|enrich)
+	Stage                string   `json:"stage" yaml:"stage,omitempty,omitempty"` //Stage for parser|postoverflow : s00-raw/s01-...
 	Name                 string   //as seen in .config.json, usually "author/name"
 	FileName             string   //the filename, ie. apache2-logs.yaml
 	Description          string   `yaml:"description,omitempty"`            //as seen in .config.json

wizard.sh

@@ -22,6 +22,7 @@ CROWDSEC_CONFIG_PATH="/etc/crowdsec"
 CROWDSEC_CONFIG_PATH="${CROWDSEC_CONFIG_PATH}/config"
 CROWDSEC_LOG_FILE="/var/log/crowdsec.log"
 CROWDSEC_BACKEND_FOLDER="/etc/crowdsec/plugins/backend"
+CSCLI_FOLDER="/etc/crowdsec/config/cscli"
 
 CROWDSEC_BIN="./cmd/crowdsec/crowdsec"
 CSCLI_BIN="./cmd/crowdsec-cli/cscli"
@@ -274,7 +275,8 @@ install_crowdsec() {
     mkdir -p "${CROWDSEC_CONFIG_PATH}/patterns" || exit
 
     mkdir -p "${CROWDSEC_BACKEND_FOLDER}" || exit
-    mkdir -p "${CROWDSEC_PLUGIN_BACKEND_DIR}" || exit    
+    mkdir -p "${CROWDSEC_PLUGIN_BACKEND_DIR}" || exit
+    mkdir -p "${CSCLI_FOLDER}" || exit
 
     (cd ./plugins && find . -type f -name "*.so" -exec install -Dm 644 {} "${CROWDSEC_PLUGIN_DIR}/{}" \; && cd ../) || exit
     cp -r ./config/plugins/backend/* "${CROWDSEC_BACKEND_FOLDER}" || exit
@@ -289,8 +291,6 @@ install_crowdsec() {
     CFG=${CROWDSEC_CONFIG_PATH} PID=${PID_DIR} BIN=${CROWDSEC_BIN_INSTALLED} envsubst < ./config/crowdsec.service > "${SYSTEMD_PATH_FILE}"
     install_bins
 	systemctl daemon-reload
-    log_info "Default cscli config generation"
-    configure_cli
 }
 
 update_bins() {
@@ -353,12 +353,6 @@ uninstall_crowdsec() {
     log_info "crowdsec successfully uninstalled"
 }
 
-# configure token and crowdsec configuration path for cscli
-configure_cli() {
-    ${CSCLI_BIN_INSTALLED} config installdir "$CROWDSEC_CONFIG_PATH" || log_err "unable to configure ${CSCLI_BIN_INSTALLED} crowdsec configuration path"
-    ${CSCLI_BIN_INSTALLED} config backend "$CROWDSEC_BACKEND_FOLDER" || log_err "unable to configure ${CSCLI_BIN_INSTALLED} backend folder"
-
-}
 
 setup_cron_pull() {
     cp ./config/crowdsec_pull /etc/cron.d/
@@ -437,7 +431,6 @@ main() {
         log_info "installing crowdsec"
         install_crowdsec
         log_info "configuring  ${CSCLI_BIN_INSTALLED}"
-        configure_cli
         ${CSCLI_BIN_INSTALLED} update > /dev/null 2>&1 || (log_err "fail to update crowdsec hub. exiting" && exit 1)
 
         # detect running services