From 76429f033a71c1fba02f5fada9bfa5c3a52478af Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Thu, 8 Jun 2023 16:49:51 +0200 Subject: [PATCH] trim pkg/types: move DataSet/GetData to pkg/cwhub, removed unused Clone function (#2271) --- pkg/cwhub/cwhub_test.go | 18 ++++++++++++------ pkg/{types => cwhub}/dataset.go | 20 +++++--------------- pkg/{types => cwhub}/dataset_test.go | 2 +- pkg/cwhub/download.go | 5 ++--- pkg/cwhub/helpers_test.go | 6 +++--- pkg/types/datasource.go | 16 ++++++++++++++++ pkg/types/utils.go | 15 --------------- 7 files changed, 39 insertions(+), 43 deletions(-) rename pkg/{types => cwhub}/dataset.go (70%) rename pkg/{types => cwhub}/dataset_test.go (98%) create mode 100644 pkg/types/datasource.go diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index b0e300aa7..f91b0dced 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -25,7 +25,7 @@ import ( var responseByPath map[string]string func TestItemStatus(t *testing.T) { - cfg := envSetup() + cfg := envSetup(t) defer envTearDown(cfg) err := UpdateHubIdx(cfg.Hub) @@ -73,7 +73,7 @@ func TestItemStatus(t *testing.T) { } func TestGetters(t *testing.T) { - cfg := envSetup() + cfg := envSetup(t) defer envTearDown(cfg) err := UpdateHubIdx(cfg.Hub) @@ -134,7 +134,7 @@ func TestGetters(t *testing.T) { } func TestIndexDownload(t *testing.T) { - cfg := envSetup() + cfg := envSetup(t) defer envTearDown(cfg) err := UpdateHubIdx(cfg.Hub) @@ -155,10 +155,16 @@ func getTestCfg() (cfg *csconfig.Config) { return } -func envSetup() *csconfig.Config { +func envSetup(t *testing.T) *csconfig.Config { resetResponseByPath() log.SetLevel(log.DebugLevel) cfg := getTestCfg() + + defaultTransport := http.DefaultClient.Transport + t.Cleanup(func() { + http.DefaultClient.Transport = defaultTransport + }) + //Mock the http client http.DefaultClient.Transport = newMockTransport() @@ -321,7 +327,7 @@ func TestInstallParser(t *testing.T) { - check its status - remove it */ - cfg := envSetup() + cfg := envSetup(t) defer envTearDown(cfg) getHubIdxOrFail(t) @@ -353,7 +359,7 @@ func TestInstallCollection(t *testing.T) { - check its status - remove it */ - cfg := envSetup() + cfg := envSetup(t) defer envTearDown(cfg) getHubIdxOrFail(t) diff --git a/pkg/types/dataset.go b/pkg/cwhub/dataset.go similarity index 70% rename from pkg/types/dataset.go rename to pkg/cwhub/dataset.go index 2684342a9..848686be6 100644 --- a/pkg/types/dataset.go +++ b/pkg/cwhub/dataset.go @@ -1,4 +1,4 @@ -package types +package cwhub import ( "fmt" @@ -6,24 +6,14 @@ import ( "net/http" "os" "path" - "time" log "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/crowdsec/pkg/types" ) -type DataSource struct { - SourceURL string `yaml:"source_url"` - DestPath string `yaml:"dest_file"` - Type string `yaml:"type"` - //Control cache strategy on expensive regexps - Cache *bool `yaml:"cache"` - Strategy *string `yaml:"strategy"` - Size *int `yaml:"size"` - TTL *time.Duration `yaml:"ttl"` -} - type DataSet struct { - Data []*DataSource `yaml:"data,omitempty"` + Data []*types.DataSource `yaml:"data,omitempty"` } func downloadFile(url string, destPath string) error { @@ -66,7 +56,7 @@ func downloadFile(url string, destPath string) error { return nil } -func GetData(data []*DataSource, dataDir string) error { +func GetData(data []*types.DataSource, dataDir string) error { for _, dataS := range data { destPath := path.Join(dataDir, dataS.DestPath) log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath) diff --git a/pkg/types/dataset_test.go b/pkg/cwhub/dataset_test.go similarity index 98% rename from pkg/types/dataset_test.go rename to pkg/cwhub/dataset_test.go index 956e3316f..b903915ac 100644 --- a/pkg/types/dataset_test.go +++ b/pkg/cwhub/dataset_test.go @@ -1,4 +1,4 @@ -package types +package cwhub import ( "os" diff --git a/pkg/cwhub/download.go b/pkg/cwhub/download.go index 6923c6d05..17b5c0e00 100644 --- a/pkg/cwhub/download.go +++ b/pkg/cwhub/download.go @@ -12,7 +12,6 @@ import ( "strings" "github.com/crowdsecurity/crowdsec/pkg/csconfig" - "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" @@ -256,7 +255,7 @@ func downloadData(dataFolder string, force bool, reader io.Reader) error { dec := yaml.NewDecoder(reader) for { - data := &types.DataSet{} + data := &DataSet{} err = dec.Decode(data) if err != nil { if errors.Is(err, io.EOF) { @@ -272,7 +271,7 @@ func downloadData(dataFolder string, force bool, reader io.Reader) error { } } if download || force { - err = types.GetData(data.Data, dataFolder) + err = GetData(data.Data, dataFolder) if err != nil { return errors.Wrap(err, "while getting data") } diff --git a/pkg/cwhub/helpers_test.go b/pkg/cwhub/helpers_test.go index 143967002..bf6e84fb3 100644 --- a/pkg/cwhub/helpers_test.go +++ b/pkg/cwhub/helpers_test.go @@ -9,7 +9,7 @@ import ( //Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection // We expect the new scenario to be installed func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { - cfg := envSetup() + cfg := envSetup(t) defer envTearDown(cfg) // fresh install of collection @@ -55,7 +55,7 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { // Install a collection, disable a scenario. // Upgrade should install should not enable/download the disabled scenario. func TestUpgradeConfigInDisabledSceanarioShouldNotBeInstalled(t *testing.T) { - cfg := envSetup() + cfg := envSetup(t) defer envTearDown(cfg) // fresh install of collection @@ -103,7 +103,7 @@ func getHubIdxOrFail(t *testing.T) { // Upgrade should not enable/download the disabled scenario. // Upgrade should install and enable the newly added scenario. func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) { - cfg := envSetup() + cfg := envSetup(t) defer envTearDown(cfg) // fresh install of collection diff --git a/pkg/types/datasource.go b/pkg/types/datasource.go new file mode 100644 index 000000000..39b83fbaf --- /dev/null +++ b/pkg/types/datasource.go @@ -0,0 +1,16 @@ +package types + +import ( + "time" +) + +type DataSource struct { + SourceURL string `yaml:"source_url"` + DestPath string `yaml:"dest_file"` + Type string `yaml:"type"` + //Control cache strategy on expensive regexps + Cache *bool `yaml:"cache"` + Strategy *string `yaml:"strategy"` + Size *int `yaml:"size"` + TTL *time.Duration `yaml:"ttl"` +} diff --git a/pkg/types/utils.go b/pkg/types/utils.go index c232c1883..0485db59e 100644 --- a/pkg/types/utils.go +++ b/pkg/types/utils.go @@ -2,8 +2,6 @@ package types import ( "bufio" - "bytes" - "encoding/gob" "fmt" "os" "path/filepath" @@ -70,19 +68,6 @@ func ConfigureLogger(clog *log.Logger) error { return nil } -func Clone(a, b interface{}) error { - buff := new(bytes.Buffer) - enc := gob.NewEncoder(buff) - dec := gob.NewDecoder(buff) - if err := enc.Encode(a); err != nil { - return fmt.Errorf("failed cloning %T", a) - } - if err := dec.Decode(b); err != nil { - return fmt.Errorf("failed cloning %T", b) - } - return nil -} - func ParseDuration(d string) (time.Duration, error) { durationStr := d if strings.HasSuffix(d, "d") {