cscli support: include stack traces (#2935)

.golangci.yml

@@ -37,17 +37,10 @@ linters-settings:
     statements: 122
 
   govet:
-    enable:
-     - atomicalign
-     - deepequalerrors
-     # TODO: - fieldalignment
-     - findcall
-     - nilness
-     # TODO: - reflectvaluecompare
-     - shadow
-     - sortslice
-     - timeformat
-     - unusedwrite
+    enable-all: true
+    disable:
+     - reflectvaluecompare
+     - fieldalignment
 
   lll:
     # lower this after refactoring

cmd/crowdsec-cli/main.go

@@ -1,7 +1,9 @@
 package main
 
 import (
+	"fmt"
 	"os"
+	"path/filepath"
 	"slices"
 	"time"
 
@@ -10,14 +12,18 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
+	"github.com/crowdsecurity/go-cs-lib/trace"
+
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/fflag"
 )
 
-var ConfigFilePath string
-var csConfig *csconfig.Config
-var dbClient *database.Client
+var (
+	ConfigFilePath string
+	csConfig       *csconfig.Config
+	dbClient       *database.Client
+)
 
 type configGetter func() *csconfig.Config
 
@@ -82,6 +88,11 @@ func loadConfigFor(command string) (*csconfig.Config, string, error) {
 			return nil, "", err
 		}
 
+		// set up directory for trace files
+		if err := trace.Init(filepath.Join(config.ConfigPaths.DataDir, "trace")); err != nil {
+			return nil, "", fmt.Errorf("while setting up trace directory: %w", err)
+		}
+
 		return config, merged, nil
 	}
 

cmd/crowdsec-cli/support.go

@@ -4,6 +4,7 @@ import (
 	"archive/zip"
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -12,12 +13,14 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/blackfireio/osinfo"
 	"github.com/go-openapi/strfmt"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
+	"github.com/crowdsecurity/go-cs-lib/trace"
 	"github.com/crowdsecurity/go-cs-lib/version"
 
 	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
@@ -47,6 +50,7 @@ const (
 	SUPPORT_CAPI_STATUS_PATH             = "capi_status.txt"
 	SUPPORT_ACQUISITION_CONFIG_BASE_PATH = "config/acquis/"
 	SUPPORT_CROWDSEC_PROFILE_PATH        = "config/profiles.yaml"
+	SUPPORT_CRASH_PATH                   = "crash/"
 )
 
 // from https://github.com/acarl005/stripansi
@@ -62,7 +66,7 @@ func collectMetrics() ([]byte, []byte, error) {
 
 	if csConfig.Cscli.PrometheusUrl == "" {
 		log.Warn("No Prometheus URL configured, metrics will not be collected")
-		return nil, nil, fmt.Errorf("prometheus_uri is not set")
+		return nil, nil, errors.New("prometheus_uri is not set")
 	}
 
 	humanMetrics := bytes.NewBuffer(nil)
@@ -70,7 +74,7 @@ func collectMetrics() ([]byte, []byte, error) {
 	ms := NewMetricStore()
 
 	if err := ms.Fetch(csConfig.Cscli.PrometheusUrl); err != nil {
-		return nil, nil, fmt.Errorf("could not fetch prometheus metrics: %s", err)
+		return nil, nil, fmt.Errorf("could not fetch prometheus metrics: %w", err)
 	}
 
 	if err := ms.Format(humanMetrics, nil, "human", false); err != nil {
@@ -79,21 +83,21 @@ func collectMetrics() ([]byte, []byte, error) {
 
 	req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl, nil)
 	if err != nil {
-		return nil, nil, fmt.Errorf("could not create requests to prometheus endpoint: %s", err)
+		return nil, nil, fmt.Errorf("could not create requests to prometheus endpoint: %w", err)
 	}
 
 	client := &http.Client{}
 
 	resp, err := client.Do(req)
 	if err != nil {
-		return nil, nil, fmt.Errorf("could not get metrics from prometheus endpoint: %s", err)
+		return nil, nil, fmt.Errorf("could not get metrics from prometheus endpoint: %w", err)
 	}
 
 	defer resp.Body.Close()
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return nil, nil, fmt.Errorf("could not read metrics from prometheus endpoint: %s", err)
+		return nil, nil, fmt.Errorf("could not read metrics from prometheus endpoint: %w", err)
 	}
 
 	return humanMetrics.Bytes(), body, nil
@@ -121,19 +125,18 @@ func collectOSInfo() ([]byte, error) {
 	log.Info("Collecting OS info")
 
 	info, err := osinfo.GetOSInfo()
-
 	if err != nil {
 		return nil, err
 	}
 
 	w := bytes.NewBuffer(nil)
-	w.WriteString(fmt.Sprintf("Architecture: %s\n", info.Architecture))
-	w.WriteString(fmt.Sprintf("Family: %s\n", info.Family))
-	w.WriteString(fmt.Sprintf("ID: %s\n", info.ID))
-	w.WriteString(fmt.Sprintf("Name: %s\n", info.Name))
-	w.WriteString(fmt.Sprintf("Codename: %s\n", info.Codename))
-	w.WriteString(fmt.Sprintf("Version: %s\n", info.Version))
-	w.WriteString(fmt.Sprintf("Build: %s\n", info.Build))
+	fmt.Fprintf(w, "Architecture: %s\n", info.Architecture)
+	fmt.Fprintf(w, "Family: %s\n", info.Family)
+	fmt.Fprintf(w, "ID: %s\n", info.ID)
+	fmt.Fprintf(w, "Name: %s\n", info.Name)
+	fmt.Fprintf(w, "Codename: %s\n", info.Codename)
+	fmt.Fprintf(w, "Version: %s\n", info.Version)
+	fmt.Fprintf(w, "Build: %s\n", info.Build)
 
 	return w.Bytes(), nil
 }
@@ -163,7 +166,7 @@ func collectBouncers(dbClient *database.Client) ([]byte, error) {
 
 	bouncers, err := dbClient.ListBouncers()
 	if err != nil {
-		return nil, fmt.Errorf("unable to list bouncers: %s", err)
+		return nil, fmt.Errorf("unable to list bouncers: %w", err)
 	}
 
 	getBouncersTable(out, bouncers)
@@ -176,7 +179,7 @@ func collectAgents(dbClient *database.Client) ([]byte, error) {
 
 	machines, err := dbClient.ListMachines()
 	if err != nil {
-		return nil, fmt.Errorf("unable to list machines: %s", err)
+		return nil, fmt.Errorf("unable to list machines: %w", err)
 	}
 
 	getAgentsTable(out, machines)
@@ -264,6 +267,11 @@ func collectAcquisitionConfig() map[string][]byte {
 	return ret
 }
 
+func collectCrash() ([]string, error) {
+	log.Info("Collecting crash dumps")
+	return trace.List()
+}
+
 type cliSupport struct{}
 
 func NewCLISupport() *cliSupport {
@@ -431,11 +439,31 @@ cscli support dump -f /tmp/crowdsec-support.zip
 				}
 			}
 
+			crash, err := collectCrash()
+			if err != nil {
+				log.Errorf("could not collect crash dumps: %s", err)
+			}
+
+			for _, filename := range crash {
+				content, err := os.ReadFile(filename)
+				if err != nil {
+					log.Errorf("could not read crash dump %s: %s", filename, err)
+				}
+
+				infos[SUPPORT_CRASH_PATH+filepath.Base(filename)] = content
+			}
+
 			w := bytes.NewBuffer(nil)
 			zipWriter := zip.NewWriter(w)
 
 			for filename, data := range infos {
-				fw, err := zipWriter.Create(filename)
+				header := &zip.FileHeader{
+					Name:   filename,
+					Method: zip.Deflate,
+					// TODO: retain mtime where possible (esp. trace)
+					Modified: time.Now(),
+				}
+				fw, err := zipWriter.CreateHeader(header)
 				if err != nil {
 					log.Errorf("Could not add zip entry for %s: %s", filename, err)
 					continue

cmd/crowdsec/main.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	_ "net/http/pprof"
 	"os"
+	"path/filepath"
 	"runtime"
 	"runtime/pprof"
 	"strings"
@@ -14,6 +15,8 @@ import (
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/tomb.v2"
 
+	"github.com/crowdsecurity/go-cs-lib/trace"
+
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/csplugin"
@@ -96,8 +99,8 @@ func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error {
 	buckets = leakybucket.NewBuckets()
 
 	log.Infof("Loading %d scenario files", len(files))
-	holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, hub, files, &bucketsTomb, buckets, flags.OrderEvent)
 
+	holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, hub, files, &bucketsTomb, buckets, flags.OrderEvent)
 	if err != nil {
 		return fmt.Errorf("scenario loading failed: %w", err)
 	}
@@ -230,6 +233,10 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
 		return nil, fmt.Errorf("while loading configuration file: %w", err)
 	}
 
+	if err := trace.Init(filepath.Join(cConfig.ConfigPaths.DataDir, "trace")); err != nil {
+		return nil, fmt.Errorf("while setting up trace directory: %w", err)
+	}
+
 	cConfig.Common.LogLevel = newLogLevel(cConfig.Common.LogLevel, flags)
 
 	if dumpFolder != "" {

cmd/crowdsec/serve.go

@@ -391,7 +391,7 @@ func Serve(cConfig *csconfig.Config, agentReady chan bool) error {
 	}
 
 	if cConfig.Common != nil && cConfig.Common.Daemonize {
-		csdaemon.NotifySystemd(log.StandardLogger())
+		csdaemon.Notify(csdaemon.Ready, log.StandardLogger())
 		// wait for signals
 		return HandleSignals(cConfig)
 	}

go.mod

@@ -27,7 +27,7 @@ require (
 	github.com/corazawaf/libinjection-go v0.1.2
 	github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607
 	github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26
-	github.com/crowdsecurity/go-cs-lib v0.0.6
+	github.com/crowdsecurity/go-cs-lib v0.0.10
 	github.com/crowdsecurity/grokky v0.2.1
 	github.com/crowdsecurity/machineid v1.0.2
 	github.com/davecgh/go-spew v1.1.1

go.sum

@@ -102,8 +102,8 @@ github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607 h1:hyrYw3h
 github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607/go.mod h1:br36fEqurGYZQGit+iDYsIzW0FF6VufMbDzyyLxEuPA=
 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU=
 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk=
-github.com/crowdsecurity/go-cs-lib v0.0.6 h1:Ef6MylXe0GaJE9vrfvxEdbHb31+JUP1os+murPz7Pos=
-github.com/crowdsecurity/go-cs-lib v0.0.6/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k=
+github.com/crowdsecurity/go-cs-lib v0.0.10 h1:Twt/y/rYCUspGY1zxDnGurL2svRSREAz+2+puLepd9c=
+github.com/crowdsecurity/go-cs-lib v0.0.10/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k=
 github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4=
 github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM=
 github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc=

pkg/apiserver/apiserver.go

@@ -84,11 +84,16 @@ func recoverFromPanic(c *gin.Context) {
 	}
 
 	if brokenPipe {
-		log.Warningf("client %s disconnected : %s", c.ClientIP(), err)
+		log.Warningf("client %s disconnected: %s", c.ClientIP(), err)
 		c.Abort()
 	} else {
-		filename := trace.WriteStackTrace(err)
-		log.Warningf("client %s error : %s", c.ClientIP(), err)
+		log.Warningf("client %s error: %s", c.ClientIP(), err)
+
+		filename, err := trace.WriteStackTrace(err)
+		if err != nil {
+			log.Errorf("also while writing stacktrace: %s", err)
+		}
+
 		log.Warningf("stacktrace written to %s, please join to your issue", filename)
 		c.AbortWithStatus(http.StatusInternalServerError)
 	}