lint

cmd/crowdsec-cli/itemcli.go

@@ -473,11 +473,22 @@ func (cli cliItem) itemDiff(item *cwhub.Item, reverse bool) (string, error) {
 		return "", fmt.Errorf("'%s' is not installed", item.FQName())
 	}
 
-	latestContent, remoteURL, err := item.FetchLatest()
+	dest, err := os.CreateTemp("", "cscli-diff-*")
+	if err != nil {
+		return "", fmt.Errorf("while creating temporary file: %w", err)
+	}
+	defer os.Remove(dest.Name())
+
+	_, remoteURL, err := item.FetchContentTo(dest.Name())
 	if err != nil {
 		return "", err
 	}
 
+	latestContent, err := os.ReadFile(dest.Name())
+	if err != nil {
+		return "", fmt.Errorf("while reading %s: %w", dest.Name(), err)
+	}
+
 	localContent, err := os.ReadFile(item.State.LocalPath)
 	if err != nil {
 		return "", fmt.Errorf("while reading %s: %w", item.State.LocalPath, err)

go.mod

@@ -27,7 +27,7 @@ require (
 	github.com/corazawaf/libinjection-go v0.1.2
 	github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607
 	github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26
-	github.com/crowdsecurity/go-cs-lib v0.0.10
+	github.com/crowdsecurity/go-cs-lib v0.0.11-0.20240422215546-8104b9078bfd
 	github.com/crowdsecurity/grokky v0.2.1
 	github.com/crowdsecurity/machineid v1.0.2
 	github.com/davecgh/go-spew v1.1.1

go.sum

@@ -102,8 +102,8 @@ github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607 h1:hyrYw3h
 github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607/go.mod h1:br36fEqurGYZQGit+iDYsIzW0FF6VufMbDzyyLxEuPA=
 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU=
 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk=
-github.com/crowdsecurity/go-cs-lib v0.0.10 h1:Twt/y/rYCUspGY1zxDnGurL2svRSREAz+2+puLepd9c=
-github.com/crowdsecurity/go-cs-lib v0.0.10/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k=
+github.com/crowdsecurity/go-cs-lib v0.0.11-0.20240422215546-8104b9078bfd h1:/fZYw5NQWI7OjRgoLdCMksnz7GbqHB07ykG8+9RtJNE=
+github.com/crowdsecurity/go-cs-lib v0.0.11-0.20240422215546-8104b9078bfd/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k=
 github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4=
 github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM=
 github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc=

pkg/cwhub/cwhub.go

@@ -23,7 +23,7 @@ func (t *hubTransport) RoundTrip(req *http.Request) (*http.Response, error) {
 
 // hubClient is the HTTP client used to communicate with the CrowdSec Hub.
 var hubClient = &http.Client{
-	Timeout: 120 * time.Second,
+	Timeout:   120 * time.Second,
 	Transport: &hubTransport{http.DefaultTransport},
 }
 

pkg/cwhub/dataset.go

@@ -1,19 +1,17 @@
 package cwhub
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io"
-	"io/fs"
-	"net/http"
-	"os"
-	"path/filepath"
-	"runtime"
 	"time"
 
 	"github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v3"
 
+	"github.com/crowdsecurity/go-cs-lib/downloader"
+
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
@@ -22,128 +20,6 @@ type DataSet struct {
 	Data []types.DataSource `yaml:"data,omitempty"`
 }
 
-// downloadFile downloads a file and writes it to disk, with no hash verification.
-func downloadFile(url string, destPath string) error {
-	resp, err := hubClient.Get(url)
-	if err != nil {
-		return fmt.Errorf("while downloading %s: %w", url, err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("bad http code %d for %s", resp.StatusCode, url)
-	}
-
-	// Download to a temporary location to avoid corrupting files
-	// that are currently in use or memory mapped.
-
-	tmpFile, err := os.CreateTemp(filepath.Dir(destPath), filepath.Base(destPath)+".*.tmp")
-	if err != nil {
-		return err
-	}
-
-	tmpFileName := tmpFile.Name()
-	defer func() {
-		tmpFile.Close()
-		os.Remove(tmpFileName)
-	}()
-
-	// avoid reading the whole file in memory
-	_, err = io.Copy(tmpFile, resp.Body)
-	if err != nil {
-		return err
-	}
-
-	if err = tmpFile.Sync(); err != nil {
-		return err
-	}
-
-	if err = tmpFile.Close(); err != nil {
-		return err
-	}
-
-	// a check on stdout is used while scripting to know if the hub has been upgraded
-	// and a configuration reload is required
-	// TODO: use a better way to communicate this
-	fmt.Printf("updated %s\n", filepath.Base(destPath))
-
-	if runtime.GOOS == "windows" {
-		// On Windows, rename will fail if the destination file already exists
-		// so we remove it first.
-		err = os.Remove(destPath)
-		switch {
-		case errors.Is(err, fs.ErrNotExist):
-			break
-		case err != nil:
-			return err
-		}
-	}
-
-	if err = os.Rename(tmpFileName, destPath); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// needsUpdate checks if a data file has to be downloaded (or updated).
-// if the local file doesn't exist, update.
-// if the remote is newer than the local file, update.
-// if the remote has no modification date, but local file has been modified > a week ago, update.
-func needsUpdate(destPath string, url string, logger *logrus.Logger) bool {
-	fileInfo, err := os.Stat(destPath)
-
-	switch {
-	case os.IsNotExist(err):
-		return true
-	case err != nil:
-		logger.Errorf("while getting %s: %s", destPath, err)
-		return true
-	}
-
-	resp, err := hubClient.Head(url)
-	if err != nil {
-		logger.Errorf("while getting %s: %s", url, err)
-		// Head failed, Get would likely fail too -> no update
-		return false
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		logger.Errorf("bad http code %d for %s", resp.StatusCode, url)
-		return false
-	}
-
-	// update if local file is older than this
-	shelfLife := 7 * 24 * time.Hour
-
-	lastModify := fileInfo.ModTime()
-
-	localIsOld := lastModify.Add(shelfLife).Before(time.Now())
-
-	remoteLastModified := resp.Header.Get("Last-Modified")
-	if remoteLastModified == "" {
-		if localIsOld {
-			logger.Infof("no last modified date for %s, but local file is older than %s", url, shelfLife)
-		}
-
-		return localIsOld
-	}
-
-	lastAvailable, err := time.Parse(time.RFC1123, remoteLastModified)
-	if err != nil {
-		logger.Warningf("while parsing last modified date for %s: %s", url, err)
-		return localIsOld
-	}
-
-	if lastModify.Before(lastAvailable) {
-		logger.Infof("new version available, updating %s", destPath)
-		return true
-	}
-
-	return false
-}
-
 // downloadDataSet downloads all the data files for an item.
 func downloadDataSet(dataFolder string, force bool, reader io.Reader, logger *logrus.Logger) error {
 	dec := yaml.NewDecoder(reader)
@@ -165,12 +41,31 @@ func downloadDataSet(dataFolder string, force bool, reader io.Reader, logger *lo
 				return err
 			}
 
-			if force || needsUpdate(destPath, dataS.SourceURL, logger) {
-				logger.Debugf("downloading %s in %s", dataS.SourceURL, destPath)
+			d := downloader.
+				New().
+				WithHTTPClient(hubClient).
+				ToFile(destPath).
+				CompareContent().
+				WithLogger(logrus.WithFields(logrus.Fields{"url": dataS.SourceURL}))
 
-				if err := downloadFile(dataS.SourceURL, destPath); err != nil {
-					return fmt.Errorf("while getting data: %w", err)
-				}
+			if !force {
+				d = d.WithLastModified().
+					WithShelfLife(7 * 24 * time.Hour)
+			}
+
+			ctx := context.TODO()
+
+			downloaded, err := d.Download(ctx, dataS.SourceURL)
+			if err != nil {
+				return fmt.Errorf("while getting data: %w", err)
+			}
+
+			if downloaded {
+				logger.Infof("Downloaded %s", destPath)
+				// a check on stdout is used while scripting to know if the hub has been upgraded
+				// and a configuration reload is required
+				// TODO: use a better way to communicate this
+				fmt.Printf("updated %s\n", destPath)
 			}
 		}
 	}

pkg/cwhub/dataset_test.go

@@ -1,56 +0,0 @@
-package cwhub
-
-import (
-	"io"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/crowdsecurity/go-cs-lib/cstest"
-)
-
-func TestDownloadFile(t *testing.T) {
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		switch r.URL.Path {
-		case "/xx":
-			w.WriteHeader(http.StatusOK)
-			_, _ = io.WriteString(w, "example content oneoneone")
-		default:
-			w.WriteHeader(http.StatusNotFound)
-			_, _ = io.WriteString(w, "not found")
-		}
-	}))
-	defer ts.Close()
-
-	dest := filepath.Join(t.TempDir(), "example.txt")
-	defer os.Remove(dest)
-
-	err := downloadFile(ts.URL+"/xx", dest)
-	require.NoError(t, err)
-
-	content, err := os.ReadFile(dest)
-	assert.Equal(t, "example content oneoneone", string(content))
-	require.NoError(t, err)
-
-	// bad uri
-	err = downloadFile("https://zz.com", dest)
-	cstest.RequireErrorContains(t, err, "lookup zz.com")
-	cstest.RequireErrorContains(t, err, "no such host")
-
-	// 404
-	err = downloadFile(ts.URL+"/x", dest)
-	cstest.RequireErrorContains(t, err, "bad http code 404")
-
-	// bad target
-	err = downloadFile(ts.URL+"/xx", "")
-	cstest.RequireErrorContains(t, err, cstest.PathNotFoundMessage)
-
-	// destination directory does not exist
-	err = downloadFile(ts.URL+"/xx", filepath.Join(t.TempDir(), "missing/example.txt"))
-	cstest.RequireErrorContains(t, err, cstest.PathNotFoundMessage)
-}

pkg/cwhub/hub.go

@@ -1,7 +1,6 @@
 package cwhub
 
 import (
-	"bytes"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -21,8 +20,8 @@ type Hub struct {
 	items    HubItems // Items read from HubDir and InstallDir
 	local    *csconfig.LocalHubCfg
 	remote   *RemoteHubCfg
-	Warnings []string // Warnings encountered during sync
 	logger   *logrus.Logger
+	Warnings []string // Warnings encountered during sync
 }
 
 // GetDataDir returns the data directory, where data sets are installed.
@@ -150,27 +149,17 @@ func (h *Hub) ItemStats() []string {
 
 // updateIndex downloads the latest version of the index and writes it to disk if it changed.
 func (h *Hub) updateIndex() error {
-	body, err := h.remote.fetchIndex()
+	downloaded, err := h.remote.fetchIndex(h.local.HubIndexFile)
 	if err != nil {
 		return err
 	}
 
-	oldContent, err := os.ReadFile(h.local.HubIndexFile)
-	if err != nil {
-		if !os.IsNotExist(err) {
-			h.logger.Warningf("failed to read hub index: %s", err)
-		}
-	} else if bytes.Equal(body, oldContent) {
+	if downloaded {
+		h.logger.Infof("Wrote index to %s", h.local.HubIndexFile)
+	} else {
 		h.logger.Info("hub index is up to date")
-		return nil
 	}
 
-	if err = os.WriteFile(h.local.HubIndexFile, body, 0o644); err != nil {
-		return fmt.Errorf("failed to write hub index: %w", err)
-	}
-
-	h.logger.Infof("Wrote index to %s, %d bytes", h.local.HubIndexFile, len(body))
-
 	return nil
 }
 

pkg/cwhub/hub_test.go

@@ -29,6 +29,10 @@ func TestUpdateIndex(t *testing.T) {
 	tmpIndex, err := os.CreateTemp("", "index.json")
 	require.NoError(t, err)
 
+	// close the file to avoid preventing the rename on windows
+	err = tmpIndex.Close()
+	require.NoError(t, err)
+
 	t.Cleanup(func() {
 		os.Remove(tmpIndex.Name())
 	})
@@ -72,5 +76,5 @@ func TestUpdateIndex(t *testing.T) {
 	hub.local.HubIndexFile = "/does/not/exist/index.json"
 
 	err = hub.updateIndex()
-	cstest.RequireErrorContains(t, err, "failed to write hub index: open /does/not/exist/index.json:")
+	cstest.RequireErrorContains(t, err, "failed to create temporary download file for /does/not/exist/index.json:")
 }

pkg/cwhub/item.go

@@ -29,10 +29,8 @@ const (
 	versionFuture                 // local version is higher latest, but is included in the index: should not happen
 )
 
-var (
-	// The order is important, as it is used to range over sub-items in collections.
-	ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, CONTEXTS, APPSEC_CONFIGS, APPSEC_RULES, COLLECTIONS}
-)
+// The order is important, as it is used to range over sub-items in collections.
+var ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, CONTEXTS, APPSEC_CONFIGS, APPSEC_RULES, COLLECTIONS}
 
 type HubItems map[string]map[string]*Item
 

pkg/cwhub/iteminstall.go

@@ -48,13 +48,13 @@ func (i *Item) Install(force bool, downloadOnly bool) error {
 		}
 	}
 
-	filePath, err := i.downloadLatest(force, true)
+	downloaded, err := i.downloadLatest(force, true)
 	if err != nil {
 		return err
 	}
 
-	if downloadOnly {
-		i.hub.logger.Infof("Downloaded %s to %s", i.Name, filePath)
+	if downloadOnly && downloaded {
+		i.hub.logger.Infof("Downloaded %s", i.Name)
 		return nil
 	}
 
@@ -62,6 +62,11 @@ func (i *Item) Install(force bool, downloadOnly bool) error {
 		return fmt.Errorf("while enabling %s: %w", i.Name, err)
 	}
 
+	// a check on stdout is used while scripting to know if the hub has been upgraded
+	// and a configuration reload is required
+	// TODO: use a better way to communicate this
+	fmt.Printf("installed %s\n", i.Name)
+
 	i.hub.logger.Infof("Enabled %s", i.Name)
 
 	return nil

pkg/cwhub/iteminstall_test.go

@@ -35,7 +35,8 @@ func testTaint(hub *Hub, t *testing.T, item *Item) {
 	// truncate the file
 	f, err := os.Create(item.State.LocalPath)
 	require.NoError(t, err)
-	f.Close()
+	err = f.Close()
+	require.NoError(t, err)
 
 	// Local sync and check status
 	err = hub.localSync()

pkg/cwhub/itemupgrade.go

@@ -3,23 +3,20 @@ package cwhub
 // Install, upgrade and remove items from the hub to the local configuration
 
 import (
-	"bytes"
-	"crypto/sha256"
-	"encoding/hex"
+	"context"
 	"errors"
 	"fmt"
-	"io"
-	"net/http"
 	"os"
-	"path/filepath"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/crowdsecurity/go-cs-lib/downloader"
 
 	"github.com/crowdsecurity/crowdsec/pkg/emoji"
 )
 
 // Upgrade downloads and applies the last version of the item from the hub.
 func (i *Item) Upgrade(force bool) (bool, error) {
-	updated := false
-
 	if i.State.IsLocal() {
 		i.hub.logger.Infof("not upgrading %s: local item", i.Name)
 		return false, nil
@@ -54,21 +51,21 @@ func (i *Item) Upgrade(force bool) (bool, error) {
 		if i.State.Tainted {
 			i.hub.logger.Warningf("%v %s is tainted, --force to overwrite", emoji.Warning, i.Name)
 		}
-	} else {
-		// a check on stdout is used while scripting to know if the hub has been upgraded
-		// and a configuration reload is required
-		// TODO: use a better way to communicate this
-		fmt.Printf("updated %s\n", i.Name)
-		i.hub.logger.Infof("%v %s: updated", emoji.Package, i.Name)
 
-		updated = true
+		return false, nil
 	}
 
-	return updated, nil
+	// a check on stdout is used while scripting to know if the hub has been upgraded
+	// and a configuration reload is required
+	// TODO: use a better way to communicate this
+	fmt.Printf("updated %s\n", i.Name)
+	i.hub.logger.Infof("%v %s: updated", emoji.Package, i.Name)
+
+	return true, nil
 }
 
 // downloadLatest downloads the latest version of the item to the hub directory.
-func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) {
+func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (bool, error) {
 	i.hub.logger.Debugf("Downloading %s %s", i.Type, i.Name)
 
 	for _, sub := range i.SubItems() {
@@ -84,98 +81,84 @@ func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) {
 			i.hub.logger.Tracef("collection, recurse")
 
 			if _, err := sub.downloadLatest(overwrite, updateOnly); err != nil {
-				return "", err
+				return false, err
 			}
 		}
 
 		downloaded := sub.State.Downloaded
 
 		if _, err := sub.download(overwrite); err != nil {
-			return "", err
+			return false, err
 		}
 
 		// We need to enable an item when it has been added to a collection since latest release of the collection.
 		// We check if sub.Downloaded is false because maybe the item has been disabled by the user.
 		if !sub.State.Installed && !downloaded {
 			if err := sub.enable(); err != nil {
-				return "", fmt.Errorf("enabling '%s': %w", sub.Name, err)
+				return false, fmt.Errorf("enabling '%s': %w", sub.Name, err)
 			}
 		}
 	}
 
 	if !i.State.Installed && updateOnly && i.State.Downloaded && !overwrite {
 		i.hub.logger.Debugf("skipping upgrade of %s: not installed", i.Name)
-		return "", nil
+		return false, nil
 	}
 
-	ret, err := i.download(overwrite)
-	if err != nil {
-		return "", err
-	}
-
-	return ret, nil
+	return i.download(overwrite)
 }
 
-// FetchLatest downloads the latest item from the hub, verifies the hash and returns the content and the used url.
-func (i *Item) FetchLatest() ([]byte, string, error) {
-	if i.latestHash() == "" {
-		return nil, "", errors.New("latest hash missing from index")
-	}
-
+// FetchContentTo downloads the last version of the item's YAML file to the specified path.
+func (i *Item) FetchContentTo(destPath string) (bool, string, error) {
 	url, err := i.hub.remote.urlTo(i.RemotePath)
 	if err != nil {
-		return nil, "", fmt.Errorf("failed to build request: %w", err)
+		return false, "", fmt.Errorf("failed to build request: %w", err)
 	}
 
-	resp, err := hubClient.Get(url)
+	wantHash := i.latestHash()
+	if wantHash == "" {
+		return false, "", errors.New("latest hash missing from index")
+	}
+
+	d := downloader.
+		New().
+		WithHTTPClient(hubClient).
+		ToFile(destPath).
+		WithMakeDirs(true).
+		WithLogger(logrus.WithFields(logrus.Fields{"url": url})).
+		CompareContent().
+		VerifyHash("sha256", wantHash)
+
+	// TODO: recommend hub update if hash does not match
+
+	ctx := context.TODO()
+
+	downloaded, err := d.Download(ctx, url)
 	if err != nil {
-		return nil, "", err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return nil, "", fmt.Errorf("bad http code %d", resp.StatusCode)
+		return false, "", fmt.Errorf("while downloading %s to %s: %w", i.Name, url, err)
 	}
 
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, "", err
-	}
-
-	hash := sha256.New()
-	if _, err = hash.Write(body); err != nil {
-		return nil, "", fmt.Errorf("while hashing %s: %w", i.Name, err)
-	}
-
-	meow := hex.EncodeToString(hash.Sum(nil))
-	if meow != i.Versions[i.Version].Digest {
-		i.hub.logger.Errorf("Downloaded version doesn't match index, please 'hub update'")
-		i.hub.logger.Debugf("got %s, expected %s", meow, i.Versions[i.Version].Digest)
-
-		return nil, "", errors.New("invalid download hash")
-	}
-
-	return body, url, nil
+	return downloaded, url, nil
 }
 
 // download downloads the item from the hub and writes it to the hub directory.
-func (i *Item) download(overwrite bool) (string, error) {
+func (i *Item) download(overwrite bool) (bool, error) {
 	// ensure that target file is within target dir
 	finalPath, err := i.downloadPath()
 	if err != nil {
-		return "", err
+		return false, err
 	}
 
 	if i.State.IsLocal() {
 		i.hub.logger.Warningf("%s is local, can't download", i.Name)
-		return finalPath, nil
+		return false, nil
 	}
 
 	// if user didn't --force, don't overwrite local, tainted, up-to-date files
 	if !overwrite {
 		if i.State.Tainted {
 			i.hub.logger.Debugf("%s: tainted, not updated", i.Name)
-			return "", nil
+			return false, nil
 		}
 
 		if i.State.UpToDate {
@@ -184,45 +167,32 @@ func (i *Item) download(overwrite bool) (string, error) {
 		}
 	}
 
-	body, url, err := i.FetchLatest()
+	downloaded, _, err := i.FetchContentTo(finalPath)
 	if err != nil {
-		what := i.Name
-		if url != "" {
-			what += " from " + url
-		}
-
-		return "", fmt.Errorf("while downloading %s: %w", what, err)
+		return false, fmt.Errorf("while downloading %s: %w", i.Name, err)
 	}
 
-	// all good, install
-
-	parentDir := filepath.Dir(finalPath)
-
-	if err = os.MkdirAll(parentDir, os.ModePerm); err != nil {
-		return "", fmt.Errorf("while creating %s: %w", parentDir, err)
-	}
-
-	// check actual file
-	if _, err = os.Stat(finalPath); !os.IsNotExist(err) {
-		i.hub.logger.Warningf("%s: overwrite", i.Name)
-		i.hub.logger.Debugf("target: %s", finalPath)
-	} else {
-		i.hub.logger.Infof("%s: OK", i.Name)
-	}
-
-	if err = os.WriteFile(finalPath, body, 0o644); err != nil {
-		return "", fmt.Errorf("while writing %s: %w", finalPath, err)
+	if downloaded {
+		i.hub.logger.Infof("Downloaded %s", i.Name)
 	}
 
 	i.State.Downloaded = true
 	i.State.Tainted = false
 	i.State.UpToDate = true
 
-	if err = downloadDataSet(i.hub.local.InstallDataDir, overwrite, bytes.NewReader(body), i.hub.logger); err != nil {
-		return "", fmt.Errorf("while downloading data for %s: %w", i.FileName, err)
+	// read content to get the list of data files
+	reader, err := os.Open(finalPath)
+	if err != nil {
+		return false, fmt.Errorf("while opening %s: %w", finalPath, err)
 	}
 
-	return finalPath, nil
+	defer reader.Close()
+
+	if err = downloadDataSet(i.hub.local.InstallDataDir, overwrite, reader, i.hub.logger); err != nil {
+		return false, fmt.Errorf("while downloading data for %s: %w", i.FileName, err)
+	}
+
+	return true, nil
 }
 
 // DownloadDataIfNeeded downloads the data set for the item.

pkg/cwhub/remote.go

@@ -1,9 +1,12 @@
 package cwhub
 
 import (
+	"context"
 	"fmt"
-	"io"
-	"net/http"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/crowdsecurity/go-cs-lib/downloader"
 )
 
 // RemoteHubCfg is used to retrieve index and items from the remote hub.
@@ -28,34 +31,28 @@ func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) {
 }
 
 // fetchIndex downloads the index from the hub and returns the content.
-func (r *RemoteHubCfg) fetchIndex() ([]byte, error) {
+func (r *RemoteHubCfg) fetchIndex(destPath string) (bool, error) {
 	if r == nil {
-		return nil, ErrNilRemoteHub
+		return false, ErrNilRemoteHub
 	}
 
 	url, err := r.urlTo(r.IndexPath)
 	if err != nil {
-		return nil, fmt.Errorf("failed to build hub index request: %w", err)
+		return false, fmt.Errorf("failed to build hub index request: %w", err)
 	}
 
-	resp, err := hubClient.Get(url)
+	ctx := context.TODO()
+
+	downloaded, err := downloader.
+		New().
+		WithHTTPClient(hubClient).
+		ToFile(destPath).
+		CompareContent().
+		WithLogger(logrus.WithFields(logrus.Fields{"url": url})).
+		Download(ctx, url)
 	if err != nil {
-		return nil, fmt.Errorf("failed http request for hub index: %w", err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		if resp.StatusCode == http.StatusNotFound {
-			return nil, IndexNotFoundError{url, r.Branch}
-		}
-
-		return nil, fmt.Errorf("bad http code %d for %s", resp.StatusCode, url)
+		return false, err
 	}
 
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read request answer for hub index: %w", err)
-	}
-
-	return body, nil
+	return downloaded, nil
 }

pkg/cwhub/sync.go

@@ -1,10 +1,7 @@
 package cwhub
 
 import (
-	"crypto/sha256"
-	"encoding/hex"
 	"fmt"
-	"io"
 	"os"
 	"path/filepath"
 	"slices"
@@ -12,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/Masterminds/semver/v3"
+	"github.com/crowdsecurity/go-cs-lib/downloader"
 	"github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v3"
 )
@@ -38,29 +36,13 @@ func linkTarget(path string, logger *logrus.Logger) (string, error) {
 	return hubpath, nil
 }
 
-func getSHA256(filepath string) (string, error) {
-	f, err := os.Open(filepath)
-	if err != nil {
-		return "", fmt.Errorf("unable to open '%s': %w", filepath, err)
-	}
-
-	defer f.Close()
-
-	h := sha256.New()
-	if _, err := io.Copy(h, f); err != nil {
-		return "", fmt.Errorf("unable to calculate sha256 of '%s': %w", filepath, err)
-	}
-
-	return hex.EncodeToString(h.Sum(nil)), nil
-}
-
 // information used to create a new Item, from a file path.
 type itemFileInfo struct {
-	inhub   bool
 	fname   string
 	stage   string
 	ftype   string
 	fauthor string
+	inhub   bool
 }
 
 func (h *Hub) getItemFileInfo(path string, logger *logrus.Logger) (*itemFileInfo, error) {
@@ -466,7 +448,7 @@ func (h *Hub) localSync() error {
 func (i *Item) setVersionState(path string, inhub bool) error {
 	var err error
 
-	i.State.LocalHash, err = getSHA256(path)
+	i.State.LocalHash, err = downloader.SHA256(path)
 	if err != nil {
 		return fmt.Errorf("failed to get sha256 of %s: %w", path, err)
 	}

test/bats/20_hub.bats

@@ -125,13 +125,19 @@ teardown() {
     assert_stderr --partial "Upgraded 0 contexts"
     assert_stderr --partial "Upgrading collections"
     assert_stderr --partial "Upgraded 0 collections"
+    assert_stderr --partial "Upgrading appsec-configs"
+    assert_stderr --partial "Upgraded 0 appsec-configs"
+    assert_stderr --partial "Upgrading appsec-rules"
+    assert_stderr --partial "Upgraded 0 appsec-rules"
+    assert_stderr --partial "Upgrading collections"
+    assert_stderr --partial "Upgraded 0 collections"
 
     rune -0 cscli parsers install crowdsecurity/syslog-logs
     rune -0 cscli hub upgrade
     assert_stderr --partial "crowdsecurity/syslog-logs: up-to-date"
 
     rune -0 cscli hub upgrade --force
-    assert_stderr --partial "crowdsecurity/syslog-logs: overwrite"
+    assert_stderr --partial "crowdsecurity/syslog-logs: up-to-date"
     assert_stderr --partial "crowdsecurity/syslog-logs: updated"
     assert_stderr --partial "Upgraded 1 parsers"
     # this is used by the cron script to know if the hub was updated

test/bats/20_hub_collections.bats

@@ -180,7 +180,6 @@ teardown() {
     assert_stderr --partial "error while installing 'crowdsecurity/sshd': while enabling crowdsecurity/sshd: crowdsecurity/sshd is tainted, won't enable unless --force"
 
     rune -0 cscli collections install crowdsecurity/sshd --force
-    assert_stderr --partial "crowdsecurity/sshd: overwrite"
     assert_stderr --partial "Enabled crowdsecurity/sshd"
 }
 

test/bats/20_hub_parsers.bats

@@ -180,7 +180,6 @@ teardown() {
     assert_stderr --partial "error while installing 'crowdsecurity/whitelists': while enabling crowdsecurity/whitelists: crowdsecurity/whitelists is tainted, won't enable unless --force"
 
     rune -0 cscli parsers install crowdsecurity/whitelists --force
-    assert_stderr --partial "crowdsecurity/whitelists: overwrite"
     assert_stderr --partial "Enabled crowdsecurity/whitelists"
 }
 

test/bats/20_hub_postoverflows.bats

@@ -180,7 +180,6 @@ teardown() {
     assert_stderr --partial "error while installing 'crowdsecurity/rdns': while enabling crowdsecurity/rdns: crowdsecurity/rdns is tainted, won't enable unless --force"
 
     rune -0 cscli postoverflows install crowdsecurity/rdns --force
-    assert_stderr --partial "crowdsecurity/rdns: overwrite"
     assert_stderr --partial "Enabled crowdsecurity/rdns"
 }
 

test/bats/20_hub_scenarios.bats

@@ -181,7 +181,6 @@ teardown() {
     assert_stderr --partial "error while installing 'crowdsecurity/ssh-bf': while enabling crowdsecurity/ssh-bf: crowdsecurity/ssh-bf is tainted, won't enable unless --force"
 
     rune -0 cscli scenarios install crowdsecurity/ssh-bf --force
-    assert_stderr --partial "crowdsecurity/ssh-bf: overwrite"
     assert_stderr --partial "Enabled crowdsecurity/ssh-bf"
 }