minor refactor to pkg/types, cscli machines (#2270)

* cleanup: separate ui and logic
* trim some code from pkg/types

cmd/crowdsec-cli/config_backup.go

@@ -9,7 +9,6 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
-	"github.com/crowdsecurity/crowdsec/pkg/types"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
@@ -43,7 +42,7 @@ func backupConfigToDirectory(dirPath string) error {
 
 	if csConfig.ConfigPaths.SimulationFilePath != "" {
 		backupSimulation := filepath.Join(dirPath, "simulation.yaml")
-		if err = types.CopyFile(csConfig.ConfigPaths.SimulationFilePath, backupSimulation); err != nil {
+		if err = CopyFile(csConfig.ConfigPaths.SimulationFilePath, backupSimulation); err != nil {
 			return errors.Wrapf(err, "failed copy %s to %s", csConfig.ConfigPaths.SimulationFilePath, backupSimulation)
 		}
 
@@ -56,7 +55,7 @@ func backupConfigToDirectory(dirPath string) error {
 	*/
 	if csConfig.Crowdsec != nil && csConfig.Crowdsec.AcquisitionFilePath != "" {
 		backupAcquisition := filepath.Join(dirPath, "acquis.yaml")
-		if err = types.CopyFile(csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition); err != nil {
+		if err = CopyFile(csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition); err != nil {
 			return fmt.Errorf("failed copy %s to %s : %s", csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition, err)
 		}
 	}
@@ -78,7 +77,7 @@ func backupConfigToDirectory(dirPath string) error {
 				return errors.Wrapf(err, "while saving %s to %s", acquisFile, acquisBackupDir)
 			}
 
-			if err = types.CopyFile(acquisFile, targetFname); err != nil {
+			if err = CopyFile(acquisFile, targetFname); err != nil {
 				return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err)
 			}
 
@@ -88,7 +87,7 @@ func backupConfigToDirectory(dirPath string) error {
 
 	if ConfigFilePath != "" {
 		backupMain := fmt.Sprintf("%s/config.yaml", dirPath)
-		if err = types.CopyFile(ConfigFilePath, backupMain); err != nil {
+		if err = CopyFile(ConfigFilePath, backupMain); err != nil {
 			return fmt.Errorf("failed copy %s to %s : %s", ConfigFilePath, backupMain, err)
 		}
 
@@ -97,7 +96,7 @@ func backupConfigToDirectory(dirPath string) error {
 
 	if csConfig.API != nil && csConfig.API.Server != nil && csConfig.API.Server.OnlineClient != nil && csConfig.API.Server.OnlineClient.CredentialsFilePath != "" {
 		backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath)
-		if err = types.CopyFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds); err != nil {
+		if err = CopyFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds); err != nil {
 			return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds, err)
 		}
 
@@ -106,7 +105,7 @@ func backupConfigToDirectory(dirPath string) error {
 
 	if csConfig.API != nil && csConfig.API.Client != nil && csConfig.API.Client.CredentialsFilePath != "" {
 		backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath)
-		if err = types.CopyFile(csConfig.API.Client.CredentialsFilePath, backupLAPICreds); err != nil {
+		if err = CopyFile(csConfig.API.Client.CredentialsFilePath, backupLAPICreds); err != nil {
 			return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Client.CredentialsFilePath, backupLAPICreds, err)
 		}
 
@@ -115,7 +114,7 @@ func backupConfigToDirectory(dirPath string) error {
 
 	if csConfig.API != nil && csConfig.API.Server != nil && csConfig.API.Server.ProfilesPath != "" {
 		backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath)
-		if err = types.CopyFile(csConfig.API.Server.ProfilesPath, backupProfiles); err != nil {
+		if err = CopyFile(csConfig.API.Server.ProfilesPath, backupProfiles); err != nil {
 			return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Server.ProfilesPath, backupProfiles, err)
 		}
 

cmd/crowdsec-cli/config_restore.go

@@ -13,7 +13,6 @@ import (
 	"gopkg.in/yaml.v2"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
-	"github.com/crowdsecurity/crowdsec/pkg/types"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
@@ -38,7 +37,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
 		backupMain := fmt.Sprintf("%s/config.yaml", dirPath)
 		if _, err = os.Stat(backupMain); err == nil {
 			if csConfig.ConfigPaths != nil && csConfig.ConfigPaths.ConfigDir != "" {
-				if err = types.CopyFile(backupMain, fmt.Sprintf("%s/config.yaml", csConfig.ConfigPaths.ConfigDir)); err != nil {
+				if err = CopyFile(backupMain, fmt.Sprintf("%s/config.yaml", csConfig.ConfigPaths.ConfigDir)); err != nil {
 					return fmt.Errorf("failed copy %s to %s : %s", backupMain, csConfig.ConfigPaths.ConfigDir, err)
 				}
 			}
@@ -51,21 +50,21 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
 
 		backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath)
 		if _, err = os.Stat(backupCAPICreds); err == nil {
-			if err = types.CopyFile(backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath); err != nil {
+			if err = CopyFile(backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath); err != nil {
 				return fmt.Errorf("failed copy %s to %s : %s", backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath, err)
 			}
 		}
 
 		backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath)
 		if _, err = os.Stat(backupLAPICreds); err == nil {
-			if err = types.CopyFile(backupLAPICreds, csConfig.API.Client.CredentialsFilePath); err != nil {
+			if err = CopyFile(backupLAPICreds, csConfig.API.Client.CredentialsFilePath); err != nil {
 				return fmt.Errorf("failed copy %s to %s : %s", backupLAPICreds, csConfig.API.Client.CredentialsFilePath, err)
 			}
 		}
 
 		backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath)
 		if _, err = os.Stat(backupProfiles); err == nil {
-			if err = types.CopyFile(backupProfiles, csConfig.API.Server.ProfilesPath); err != nil {
+			if err = CopyFile(backupProfiles, csConfig.API.Server.ProfilesPath); err != nil {
 				return fmt.Errorf("failed copy %s to %s : %s", backupProfiles, csConfig.API.Server.ProfilesPath, err)
 			}
 		}
@@ -106,7 +105,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
 
 	backupSimulation := fmt.Sprintf("%s/simulation.yaml", dirPath)
 	if _, err = os.Stat(backupSimulation); err == nil {
-		if err = types.CopyFile(backupSimulation, csConfig.ConfigPaths.SimulationFilePath); err != nil {
+		if err = CopyFile(backupSimulation, csConfig.ConfigPaths.SimulationFilePath); err != nil {
 			return fmt.Errorf("failed copy %s to %s : %s", backupSimulation, csConfig.ConfigPaths.SimulationFilePath, err)
 		}
 	}
@@ -123,7 +122,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
 	if _, err = os.Stat(backupAcquisition); err == nil {
 		log.Debugf("restoring backup'ed %s", backupAcquisition)
 
-		if err = types.CopyFile(backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath); err != nil {
+		if err = CopyFile(backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath); err != nil {
 			return fmt.Errorf("failed copy %s to %s : %s", backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath, err)
 		}
 	}
@@ -139,7 +138,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
 
 			log.Debugf("restoring %s to %s", acquisFile, targetFname)
 
-			if err = types.CopyFile(acquisFile, targetFname); err != nil {
+			if err = CopyFile(acquisFile, targetFname); err != nil {
 				return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err)
 			}
 		}
@@ -160,7 +159,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
 				return errors.Wrapf(err, "while saving %s to %s", acquisFile, acquisBackupDir)
 			}
 
-			if err = types.CopyFile(acquisFile, targetFname); err != nil {
+			if err = CopyFile(acquisFile, targetFname); err != nil {
 				return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err)
 			}
 

cmd/crowdsec-cli/copyfile.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	log "github.com/sirupsen/logrus"
+)
+
+
+/*help to copy the file, ioutil doesn't offer the feature*/
+
+func copyFileContents(src, dst string) (err error) {
+	in, err := os.Open(src)
+	if err != nil {
+		return
+	}
+	defer in.Close()
+	out, err := os.Create(dst)
+	if err != nil {
+		return
+	}
+	defer func() {
+		cerr := out.Close()
+		if err == nil {
+			err = cerr
+		}
+	}()
+	if _, err = io.Copy(out, in); err != nil {
+		return
+	}
+	err = out.Sync()
+	return
+}
+
+/*copy the file, ioutile doesn't offer the feature*/
+func CopyFile(sourceSymLink, destinationFile string) (err error) {
+	sourceFile, err := filepath.EvalSymlinks(sourceSymLink)
+	if err != nil {
+		log.Infof("Not a symlink : %s", err)
+		sourceFile = sourceSymLink
+	}
+
+	sourceFileStat, err := os.Stat(sourceFile)
+	if err != nil {
+		return
+	}
+	if !sourceFileStat.Mode().IsRegular() {
+		// cannot copy non-regular files (e.g., directories,
+		// symlinks, devices, etc.)
+		return fmt.Errorf("copyFile: non-regular source file %s (%q)", sourceFileStat.Name(), sourceFileStat.Mode().String())
+	}
+	destinationFileStat, err := os.Stat(destinationFile)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return
+		}
+	} else {
+		if !(destinationFileStat.Mode().IsRegular()) {
+			return fmt.Errorf("copyFile: non-regular destination file %s (%q)", destinationFileStat.Name(), destinationFileStat.Mode().String())
+		}
+		if os.SameFile(sourceFileStat, destinationFileStat) {
+			return
+		}
+	}
+	if err = os.Link(sourceFile, destinationFile); err != nil {
+		err = copyFileContents(sourceFile, destinationFile)
+	}
+	return
+}
+

cmd/crowdsec-cli/machines.go

@@ -12,7 +12,6 @@ import (
 	"time"
 
 	"github.com/AlecAivazis/survey/v2"
-	"github.com/enescakir/emoji"
 	"github.com/fatih/color"
 	"github.com/go-openapi/strfmt"
 	"github.com/google/uuid"
@@ -85,22 +84,21 @@ func generateID(prefix string) (string, error) {
 	return prefix + suffix, nil
 }
 
-func displayLastHeartBeat(m *ent.Machine, fancy bool) string {
-	var hbDisplay string
+// getLastHeartbeat returns the last heartbeat timestamp of a machine
+// and a boolean indicating if the machine is considered active or not.
+func getLastHeartbeat(m *ent.Machine) (string, bool) {
+	if m.LastHeartbeat == nil {
+		return "-", false
+	}
 
-	if m.LastHeartbeat != nil {
-		lastHeartBeat := time.Now().UTC().Sub(*m.LastHeartbeat)
-		hbDisplay = lastHeartBeat.Truncate(time.Second).String()
-		if fancy && lastHeartBeat > 2*time.Minute {
-			hbDisplay = fmt.Sprintf("%s %s", emoji.Warning.String(), lastHeartBeat.Truncate(time.Second).String())
-		}
-	} else {
-		hbDisplay = "-"
-		if fancy {
-			hbDisplay = emoji.Warning.String() + " -"
-		}
+	elapsed := time.Now().UTC().Sub(*m.LastHeartbeat)
+
+	hb := elapsed.Truncate(time.Second).String()
+	if elapsed > 2*time.Minute {
+		return hb, false
 	}
-	return hbDisplay
+
+	return hb, true
 }
 
 func getAgents(out io.Writer, dbClient *database.Client) error {
@@ -130,9 +128,10 @@ func getAgents(out io.Writer, dbClient *database.Client) error {
 			} else {
 				validated = "false"
 			}
-			err := csvwriter.Write([]string{m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, displayLastHeartBeat(m, false)})
+			hb, _ := getLastHeartbeat(m)
+			err := csvwriter.Write([]string{m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, hb})
 			if err != nil {
-				return fmt.Errorf("failed to write raw output : %s", err)
+				return fmt.Errorf("failed to write raw output: %w", err)
 			}
 		}
 		csvwriter.Flush()

cmd/crowdsec-cli/machines_table.go

@@ -24,7 +24,11 @@ func getAgentsTable(out io.Writer, machines []*ent.Machine) {
 			validated = emoji.Prohibited.String()
 		}
 
-		t.AddRow(m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, displayLastHeartBeat(m, true))
+		hb, active := getLastHeartbeat(m)
+		if !active {
+			hb = emoji.Warning.String() + " " + hb
+		}
+		t.AddRow(m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, hb)
 	}
 
 	t.Render()

cmd/crowdsec-cli/support.go

@@ -26,7 +26,6 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/fflag"
 	"github.com/crowdsecurity/crowdsec/pkg/models"
-	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
 const (
@@ -48,6 +47,14 @@ const (
 	SUPPORT_CROWDSEC_PROFILE_PATH        = "config/profiles.yaml"
 )
 
+// from https://github.com/acarl005/stripansi
+var reStripAnsi = regexp.MustCompile("[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))")
+
+func stripAnsiString(str string) string {
+	// the byte version doesn't strip correctly
+	return reStripAnsi.ReplaceAllString(str, "")
+}
+
 func collectMetrics() ([]byte, []byte, error) {
 	log.Info("Collecting prometheus metrics")
 	err := csConfig.LoadPrometheus()
@@ -400,7 +407,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
 					log.Errorf("Could not add zip entry for %s: %s", filename, err)
 					continue
 				}
-				fw.Write([]byte(types.StripAnsiString(string(data))))
+				fw.Write([]byte(stripAnsiString(string(data))))
 			}
 
 			err = zipWriter.Close()

cmd/crowdsec-cli/utils.go

@@ -594,7 +594,7 @@ func RestoreHub(dirPath string) error {
 					log.Infof("Going to restore local/tainted [%s]", tfile.Name())
 					sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
 					destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
-					if err = types.CopyFile(sourceFile, destinationFile); err != nil {
+					if err = CopyFile(sourceFile, destinationFile); err != nil {
 						return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
 					}
 					log.Infof("restored %s to %s", sourceFile, destinationFile)
@@ -603,7 +603,7 @@ func RestoreHub(dirPath string) error {
 				log.Infof("Going to restore local/tainted [%s]", file.Name())
 				sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
 				destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name())
-				if err = types.CopyFile(sourceFile, destinationFile); err != nil {
+				if err = CopyFile(sourceFile, destinationFile); err != nil {
 					return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
 				}
 				log.Infof("restored %s to %s", sourceFile, destinationFile)
@@ -653,7 +653,7 @@ func BackupHub(dirPath string) error {
 				}
 				clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
 				tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
-				if err = types.CopyFile(v.LocalPath, tfile); err != nil {
+				if err = CopyFile(v.LocalPath, tfile); err != nil {
 					return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
 				}
 				clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)

pkg/types/utils.go

@@ -5,10 +5,7 @@ import (
 	"bytes"
 	"encoding/gob"
 	"fmt"
-	"io"
 	"os"
-	"path/filepath"
-	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -105,67 +102,6 @@ func ParseDuration(d string) (time.Duration, error) {
 	return duration, nil
 }
 
-/*help to copy the file, ioutil doesn't offer the feature*/
-
-func copyFileContents(src, dst string) (err error) {
-	in, err := os.Open(src)
-	if err != nil {
-		return
-	}
-	defer in.Close()
-	out, err := os.Create(dst)
-	if err != nil {
-		return
-	}
-	defer func() {
-		cerr := out.Close()
-		if err == nil {
-			err = cerr
-		}
-	}()
-	if _, err = io.Copy(out, in); err != nil {
-		return
-	}
-	err = out.Sync()
-	return
-}
-
-/*copy the file, ioutile doesn't offer the feature*/
-func CopyFile(sourceSymLink, destinationFile string) (err error) {
-	sourceFile, err := filepath.EvalSymlinks(sourceSymLink)
-	if err != nil {
-		log.Infof("Not a symlink : %s", err)
-		sourceFile = sourceSymLink
-	}
-
-	sourceFileStat, err := os.Stat(sourceFile)
-	if err != nil {
-		return
-	}
-	if !sourceFileStat.Mode().IsRegular() {
-		// cannot copy non-regular files (e.g., directories,
-		// symlinks, devices, etc.)
-		return fmt.Errorf("copyFile: non-regular source file %s (%q)", sourceFileStat.Name(), sourceFileStat.Mode().String())
-	}
-	destinationFileStat, err := os.Stat(destinationFile)
-	if err != nil {
-		if !os.IsNotExist(err) {
-			return
-		}
-	} else {
-		if !(destinationFileStat.Mode().IsRegular()) {
-			return fmt.Errorf("copyFile: non-regular destination file %s (%q)", destinationFileStat.Name(), destinationFileStat.Mode().String())
-		}
-		if os.SameFile(sourceFileStat, destinationFileStat) {
-			return
-		}
-	}
-	if err = os.Link(sourceFile, destinationFile); err != nil {
-		err = copyFileContents(sourceFile, destinationFile)
-	}
-	return
-}
-
 func UtcNow() time.Time {
 	return time.Now().UTC()
 }
@@ -183,11 +119,3 @@ func GetLineCountForFile(filepath string) int {
 	}
 	return lc
 }
-
-// from https://github.com/acarl005/stripansi
-var reStripAnsi = regexp.MustCompile("[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))")
-
-func StripAnsiString(str string) string {
-	// the byte version doesn't strip correctly
-	return reStripAnsi.ReplaceAllString(str, "")
-}