pkg/cwhub documentation (#2607)
* pkg/cwhub: package documentation * Don't repeat local state in "cscli... inspect" * lint * use proper name of the hub item instead of the filename for local items * hub update: avoid reporting local items as tainted
This commit is contained in:
mmetc
GitHub
parent
1509c2d97c
commit
2c652ef92f
15 changed files with 299 additions and 118 deletions
|
|
@ -268,7 +268,7 @@ func NewItemsInstallCmd(typeName string) *cobra.Command {
|
|||
func istalledParentNames(item *cwhub.Item) []string {
|
||||
ret := make([]string, 0)
|
||||
|
||||
for _, parent := range item.ParentCollections() {
|
||||
for _, parent := range item.AncestorCollections() {
|
||||
if parent.State.Installed {
|
||||
ret = append(ret, parent.Name)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,3 @@
|
|||
// Package cwhub is responsible for installing and upgrading the local hub files.
|
||||
//
|
||||
// This includes retrieving the index, the items to install (parsers, scenarios, data files...)
|
||||
// and managing the dependencies and taints.
|
||||
package cwhub
|
||||
|
||||
import (
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ const mockURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s"
|
|||
|
||||
var responseByPath map[string]string
|
||||
|
||||
// testHub initializes a temporary hub with an empty json file, optionally updating it
|
||||
// testHub initializes a temporary hub with an empty json file, optionally updating it.
|
||||
func testHub(t *testing.T, update bool) *Hub {
|
||||
tmpDir, err := os.MkdirTemp("", "testhub")
|
||||
require.NoError(t, err)
|
||||
|
|
@ -67,7 +67,7 @@ func testHub(t *testing.T, update bool) *Hub {
|
|||
return hub
|
||||
}
|
||||
|
||||
// envSetup initializes the temporary hub and mocks the http client
|
||||
// envSetup initializes the temporary hub and mocks the http client.
|
||||
func envSetup(t *testing.T) *Hub {
|
||||
setResponseByPath()
|
||||
log.SetLevel(log.DebugLevel)
|
||||
|
|
@ -92,7 +92,7 @@ func newMockTransport() http.RoundTripper {
|
|||
return &mockTransport{}
|
||||
}
|
||||
|
||||
// Implement http.RoundTripper
|
||||
// Implement http.RoundTripper.
|
||||
func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
// Create mocked http.Response
|
||||
response := &http.Response{
|
||||
|
|
|
|||
|
|
@ -13,11 +13,12 @@ import (
|
|||
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||
)
|
||||
|
||||
// The DataSet is a list of data sources required by an item (built from the data: section in the yaml).
|
||||
type DataSet struct {
|
||||
Data []types.DataSource `yaml:"data,omitempty"`
|
||||
}
|
||||
|
||||
// downloadFile downloads a file and writes it to disk, with no hash verification
|
||||
// downloadFile downloads a file and writes it to disk, with no hash verification.
|
||||
func downloadFile(url string, destPath string) error {
|
||||
log.Debugf("downloading %s in %s", url, destPath)
|
||||
|
||||
|
|
@ -50,7 +51,7 @@ func downloadFile(url string, destPath string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// downloadDataSet downloads all the data files for an item
|
||||
// downloadDataSet downloads all the data files for an item.
|
||||
func downloadDataSet(dataFolder string, force bool, reader io.Reader) error {
|
||||
dec := yaml.NewDecoder(reader)
|
||||
|
||||
|
|
|
|||
113
pkg/cwhub/doc.go
Normal file
113
pkg/cwhub/doc.go
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
// Package cwhub is responsible for installing and upgrading the local hub files for CrowdSec.
|
||||
//
|
||||
// # Definitions
|
||||
//
|
||||
// - A hub ITEM is a file that defines a parser, a scenario, a collection... in the case of a collection, it has dependencies on other hub items.
|
||||
// - The hub INDEX is a JSON file that contains a tree of available hub items.
|
||||
// - A REMOTE HUB is an HTTP server that hosts the hub index and the hub items. It can serve from several branches, usually linked to the CrowdSec version.
|
||||
// - A LOCAL HUB is a directory that contains a copy of the hub index and the downloaded hub items.
|
||||
//
|
||||
// Once downloaded, hub items can be installed by linking to them from the configuration directory.
|
||||
// If an item is present in the configuration directory but it's not a link to the local hub, it is
|
||||
// considered as a LOCAL ITEM and won't be removed or upgraded.
|
||||
//
|
||||
// # Directory Structure
|
||||
//
|
||||
// A typical directory layout is the following:
|
||||
//
|
||||
// For the local hub (HubDir = /etc/crowdsec/hub):
|
||||
//
|
||||
// - /etc/crowdsec/hub/.index.json
|
||||
// - /etc/crowdsec/hub/parsers/{stage}/{author}/{parser-name}.yaml
|
||||
// - /etc/crowdsec/hub/scenarios/{author}/{scenario-name}.yaml
|
||||
//
|
||||
// For the configuration directory (InstallDir = /etc/crowdsec):
|
||||
//
|
||||
// - /etc/crowdsec/parsers/{stage}/{parser-name.yaml} -> /etc/crowdsec/hub/parsers/{stage}/{author}/{parser-name}.yaml
|
||||
// - /etc/crowdsec/scenarios/{scenario-name.yaml} -> /etc/crowdsec/hub/scenarios/{author}/{scenario-name}.yaml
|
||||
// - /etc/crowdsec/scenarios/local-scenario.yaml
|
||||
//
|
||||
// Note that installed items are not grouped by author, this may change in the future if we want to
|
||||
// support items with the same name from different authors.
|
||||
//
|
||||
// Only parsers and postoverflows have the concept of stage.
|
||||
//
|
||||
// Additionally, an item can reference a DATA SET that is installed in a different location than
|
||||
// the item itself. These files are stored in the data directory (InstallDataDir = /var/lib/crowdsec/data).
|
||||
//
|
||||
// - /var/lib/crowdsec/data/http_path_traversal.txt
|
||||
// - /var/lib/crowdsec/data/jira_cve_2021-26086.txt
|
||||
// - /var/lib/crowdsec/data/log4j2_cve_2021_44228.txt
|
||||
// - /var/lib/crowdsec/data/sensitive_data.txt
|
||||
//
|
||||
//
|
||||
// # Using the package
|
||||
//
|
||||
// The main entry point is the Hub struct. You can create a new instance with NewHub().
|
||||
// This constructor takes three parameters, but only the LOCAL HUB configuration is required:
|
||||
//
|
||||
// import (
|
||||
// "fmt"
|
||||
// "github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||
// "github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||
// )
|
||||
//
|
||||
// localHub := csconfig.LocalHubCfg{
|
||||
// HubIndexFile: "/etc/crowdsec/hub/.index.json",
|
||||
// HubDir: "/etc/crowdsec/hub",
|
||||
// InstallDir: "/etc/crowdsec",
|
||||
// InstallDataDir: "/var/lib/crowdsec/data",
|
||||
// }
|
||||
// hub, err := cwhub.NewHub(localHub, nil, false)
|
||||
// if err != nil {
|
||||
// return fmt.Errorf("unable to initialize hub: %w", err)
|
||||
// }
|
||||
//
|
||||
// Now you can use the hub to access the existing items:
|
||||
//
|
||||
// // list all the parsers
|
||||
// for _, parser := range hub.GetItemMap(cwhub.PARSERS) {
|
||||
// fmt.Printf("parser: %s\n", parser.Name)
|
||||
// }
|
||||
//
|
||||
// // retrieve a specific collection
|
||||
// coll := hub.GetItem(cwhub.COLLECTIONS, "crowdsecurity/linux")
|
||||
// if coll == nil {
|
||||
// return fmt.Errorf("collection not found")
|
||||
// }
|
||||
//
|
||||
// You can also install items if they have already been downloaded:
|
||||
//
|
||||
// // install a parser
|
||||
// force := false
|
||||
// downloadOnly := false
|
||||
// err := parser.Install(force, downloadOnly)
|
||||
// if err != nil {
|
||||
// return fmt.Errorf("unable to install parser: %w", err)
|
||||
// }
|
||||
//
|
||||
// As soon as you try to install an item that is not downloaded or is not up-to-date (meaning its computed hash
|
||||
// does not correspond to the latest version available in the index), a download will be attempted and you'll
|
||||
// get the error "remote hub configuration is not provided".
|
||||
//
|
||||
// To provide the remote hub configuration, use the second parameter of NewHub():
|
||||
//
|
||||
// remoteHub := cwhub.RemoteHubCfg{
|
||||
// URLTemplate: "https://hub-cdn.crowdsec.net/%s/%s",
|
||||
// Branch: "master",
|
||||
// IndexPath: ".index.json",
|
||||
// }
|
||||
// updateIndex := false
|
||||
// hub, err := cwhub.NewHub(localHub, remoteHub, updateIndex)
|
||||
// if err != nil {
|
||||
// return fmt.Errorf("unable to initialize hub: %w", err)
|
||||
// }
|
||||
//
|
||||
// The URLTemplate is a string that will be used to build the URL of the remote hub. It must contain two
|
||||
// placeholders: the branch and the file path (it will be an index or an item).
|
||||
//
|
||||
// Setting the third parameter to true will download the latest version of the index, if available on the
|
||||
// specified branch.
|
||||
// There is no exported method to update the index once the hub struct is created.
|
||||
//
|
||||
package cwhub
|
||||
|
|
@ -11,8 +11,8 @@ import (
|
|||
)
|
||||
|
||||
// installPath returns the location of the symlink to the item in the hub, or the path of the item itself if it's local
|
||||
// (eg. /etc/crowdsec/collections/xyz.yaml)
|
||||
// raises an error if the path goes outside of the install dir
|
||||
// (eg. /etc/crowdsec/collections/xyz.yaml).
|
||||
// Raises an error if the path goes outside of the install dir.
|
||||
func (i *Item) installPath() (string, error) {
|
||||
p := i.Type
|
||||
if i.Stage != "" {
|
||||
|
|
@ -23,8 +23,8 @@ func (i *Item) installPath() (string, error) {
|
|||
}
|
||||
|
||||
// downloadPath returns the location of the actual config file in the hub
|
||||
// (eg. /etc/crowdsec/hub/collections/author/xyz.yaml)
|
||||
// raises an error if the path goes outside of the hub dir
|
||||
// (eg. /etc/crowdsec/hub/collections/author/xyz.yaml).
|
||||
// Raises an error if the path goes outside of the hub dir.
|
||||
func (i *Item) downloadPath() (string, error) {
|
||||
ret, err := safePath(i.hub.local.HubDir, i.RemotePath)
|
||||
if err != nil {
|
||||
|
|
@ -34,7 +34,7 @@ func (i *Item) downloadPath() (string, error) {
|
|||
return ret, nil
|
||||
}
|
||||
|
||||
// makeLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir
|
||||
// makeLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir.
|
||||
func (i *Item) createInstallLink() error {
|
||||
dest, err := i.installPath()
|
||||
if err != nil {
|
||||
|
|
@ -63,7 +63,7 @@ func (i *Item) createInstallLink() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// enable enables the item by creating a symlink to the downloaded content, and also enables sub-items
|
||||
// enable enables the item by creating a symlink to the downloaded content, and also enables sub-items.
|
||||
func (i *Item) enable() error {
|
||||
if i.State.Installed {
|
||||
if i.State.Tainted {
|
||||
|
|
@ -97,7 +97,7 @@ func (i *Item) enable() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// purge removes the actual config file that was downloaded
|
||||
// purge removes the actual config file that was downloaded.
|
||||
func (i *Item) purge() error {
|
||||
if !i.State.Downloaded {
|
||||
log.Infof("removing %s: not downloaded -- no need to remove", i.Name)
|
||||
|
|
@ -124,7 +124,7 @@ func (i *Item) purge() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// removeInstallLink removes the symlink to the downloaded content
|
||||
// removeInstallLink removes the symlink to the downloaded content.
|
||||
func (i *Item) removeInstallLink() error {
|
||||
syml, err := i.installPath()
|
||||
if err != nil {
|
||||
|
|
@ -166,7 +166,7 @@ func (i *Item) removeInstallLink() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// disable removes the install link, and optionally the downloaded content
|
||||
// disable removes the install link, and optionally the downloaded content.
|
||||
func (i *Item) disable(purge bool, force bool) error {
|
||||
// XXX: should return the number of disabled/purged items to inform the upper layer whether to reload or not
|
||||
err := i.removeInstallLink()
|
||||
|
|
|
|||
|
|
@ -7,10 +7,10 @@ import (
|
|||
|
||||
var (
|
||||
// ErrNilRemoteHub is returned when the remote hub configuration is not provided to the NewHub constructor.
|
||||
// All attempts to download index or items will return this error.
|
||||
ErrNilRemoteHub = errors.New("remote hub configuration is not provided. Please report this issue to the developers")
|
||||
)
|
||||
|
||||
// IndexNotFoundError is returned when the remote hub index is not found.
|
||||
type IndexNotFoundError struct {
|
||||
URL string
|
||||
Branch string
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import (
|
|||
"slices"
|
||||
)
|
||||
|
||||
// Install installs the item from the hub, downloading it if needed
|
||||
// Install installs the item from the hub, downloading it if needed.
|
||||
func (i *Item) Install(force bool, downloadOnly bool) error {
|
||||
if downloadOnly && i.State.Downloaded && i.State.UpToDate {
|
||||
log.Infof("%s is already downloaded and up-to-date", i.Name)
|
||||
|
|
@ -52,7 +52,7 @@ func (i *Item) Install(force bool, downloadOnly bool) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// allDependencies returns a list of all (direct or indirect) dependencies of the item
|
||||
// allDependencies returns a list of all (direct or indirect) dependencies of the item.
|
||||
func (i *Item) allDependencies() ([]*Item, error) {
|
||||
var collectSubItems func(item *Item, visited map[*Item]bool, result *[]*Item) error
|
||||
|
||||
|
|
@ -94,7 +94,7 @@ func (i *Item) allDependencies() ([]*Item, error) {
|
|||
return ret, nil
|
||||
}
|
||||
|
||||
// Remove disables the item, optionally removing the downloaded content
|
||||
// Remove disables the item, optionally removing the downloaded content.
|
||||
func (i *Item) Remove(purge bool, force bool) (bool, error) {
|
||||
if i.IsLocal() {
|
||||
return false, fmt.Errorf("%s isn't managed by hub. Please delete manually", i.Name)
|
||||
|
|
@ -123,7 +123,7 @@ func (i *Item) Remove(purge bool, force bool) (bool, error) {
|
|||
|
||||
// if the sub depends on a collection that is not a direct or indirect dependency
|
||||
// of the current item, it is not removed
|
||||
for _, subParent := range sub.ParentCollections() {
|
||||
for _, subParent := range sub.AncestorCollections() {
|
||||
if !purge && !subParent.State.Installed {
|
||||
continue
|
||||
}
|
||||
|
|
@ -156,7 +156,7 @@ func (i *Item) Remove(purge bool, force bool) (bool, error) {
|
|||
return removed, nil
|
||||
}
|
||||
|
||||
// Upgrade downloads and applies the last version from the hub
|
||||
// Upgrade downloads and applies the last version of the item from the hub.
|
||||
func (i *Item) Upgrade(force bool) (bool, error) {
|
||||
updated := false
|
||||
|
||||
|
|
@ -203,7 +203,7 @@ func (i *Item) Upgrade(force bool) (bool, error) {
|
|||
return updated, nil
|
||||
}
|
||||
|
||||
// downloadLatest downloads the latest version of the item to the hub directory
|
||||
// downloadLatest downloads the latest version of the item to the hub directory.
|
||||
func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) {
|
||||
// XXX: should return the path of the downloaded file (taken from download())
|
||||
log.Debugf("Downloading %s %s", i.Type, i.Name)
|
||||
|
|
@ -253,7 +253,7 @@ func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) {
|
|||
return ret, nil
|
||||
}
|
||||
|
||||
// fetch downloads the item from the hub, verifies the hash and returns the content
|
||||
// fetch downloads the item from the hub, verifies the hash and returns the content.
|
||||
func (i *Item) fetch() ([]byte, error) {
|
||||
url, err := i.hub.remote.urlTo(i.RemotePath)
|
||||
if err != nil {
|
||||
|
|
@ -291,7 +291,7 @@ func (i *Item) fetch() ([]byte, error) {
|
|||
return body, nil
|
||||
}
|
||||
|
||||
// download downloads the item from the hub and writes it to the hub directory
|
||||
// download downloads the item from the hub and writes it to the hub directory.
|
||||
func (i *Item) download(overwrite bool) (string, error) {
|
||||
// if user didn't --force, don't overwrite local, tainted, up-to-date files
|
||||
if !overwrite {
|
||||
|
|
@ -348,7 +348,7 @@ func (i *Item) download(overwrite bool) (string, error) {
|
|||
return finalPath, nil
|
||||
}
|
||||
|
||||
// DownloadDataIfNeeded downloads the data files for the item
|
||||
// DownloadDataIfNeeded downloads the data set for the item.
|
||||
func (i *Item) DownloadDataIfNeeded(force bool) error {
|
||||
itemFilePath, err := i.installPath()
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -8,8 +8,8 @@ import (
|
|||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||
)
|
||||
|
||||
// Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection
|
||||
// We expect the new scenario to be installed
|
||||
// Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection.
|
||||
// We expect the new scenario to be installed.
|
||||
func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
|
||||
hub := envSetup(t)
|
||||
|
||||
|
|
@ -110,7 +110,7 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
|
|||
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
|
||||
}
|
||||
|
||||
// getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test
|
||||
// getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test.
|
||||
func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *RemoteHubCfg) *Hub {
|
||||
hub, err := NewHub(local, remote, false)
|
||||
require.NoError(t, err, "failed to load hub index")
|
||||
|
|
|
|||
|
|
@ -13,19 +13,22 @@ import (
|
|||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||
)
|
||||
|
||||
// Hub is the main structure for the package.
|
||||
type Hub struct {
|
||||
Items HubItems
|
||||
Items HubItems // Items read from HubDir and InstallDir
|
||||
local *csconfig.LocalHubCfg
|
||||
remote *RemoteHubCfg
|
||||
Warnings []string
|
||||
Warnings []string // Warnings encountered during sync
|
||||
}
|
||||
|
||||
// GetDataDir returns the data directory, where data sets are installed.
|
||||
func (h *Hub) GetDataDir() string {
|
||||
return h.local.InstallDataDir
|
||||
}
|
||||
|
||||
// NewHub returns a new Hub instance with local and (optionally) remote configuration, and syncs the local state
|
||||
// It also downloads the index if updateIndex is true
|
||||
// NewHub returns a new Hub instance with local and (optionally) remote configuration, and syncs the local state.
|
||||
// If updateIndex is true, the local index file is updated from the remote before reading the state of the items.
|
||||
// All download operations (including updateIndex) return ErrNilRemoteHub if the remote configuration is not set.
|
||||
func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, updateIndex bool) (*Hub, error) {
|
||||
if local == nil {
|
||||
return nil, fmt.Errorf("no hub configuration found")
|
||||
|
|
@ -55,7 +58,7 @@ func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, updateIndex bool)
|
|||
return hub, nil
|
||||
}
|
||||
|
||||
// parseIndex takes the content of an index file and fills the map of associated parsers/scenarios/collections
|
||||
// parseIndex takes the content of an index file and fills the map of associated parsers/scenarios/collections.
|
||||
func (h *Hub) parseIndex() error {
|
||||
bidx, err := os.ReadFile(h.local.HubIndexFile)
|
||||
if err != nil {
|
||||
|
|
@ -91,7 +94,7 @@ func (h *Hub) parseIndex() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// ItemStats returns total counts of the hub items
|
||||
// ItemStats returns total counts of the hub items, including local and tainted.
|
||||
func (h *Hub) ItemStats() []string {
|
||||
loaded := ""
|
||||
local := 0
|
||||
|
|
@ -131,7 +134,7 @@ func (h *Hub) ItemStats() []string {
|
|||
return ret
|
||||
}
|
||||
|
||||
// updateIndex downloads the latest version of the index and writes it to disk if it changed
|
||||
// updateIndex downloads the latest version of the index and writes it to disk if it changed.
|
||||
func (h *Hub) updateIndex() error {
|
||||
body, err := h.remote.fetchIndex()
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import (
|
|||
)
|
||||
|
||||
const (
|
||||
// managed item types
|
||||
// managed item types.
|
||||
COLLECTIONS = "collections"
|
||||
PARSERS = "parsers"
|
||||
POSTOVERFLOWS = "postoverflows"
|
||||
|
|
@ -20,59 +20,57 @@ const (
|
|||
)
|
||||
|
||||
const (
|
||||
VersionUpToDate = iota
|
||||
VersionUpdateAvailable
|
||||
VersionUnknown
|
||||
VersionFuture
|
||||
versionUpToDate = iota // the latest version from index is installed
|
||||
versionUpdateAvailable // not installed, or lower than latest
|
||||
versionUnknown // local file with no version, or invalid version number
|
||||
versionFuture // local version is higher latest, but is included in the index: should not happen
|
||||
)
|
||||
|
||||
// The order is important, as it is used to range over sub-items in collections
|
||||
var ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS}
|
||||
var (
|
||||
// The order is important, as it is used to range over sub-items in collections.
|
||||
ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS}
|
||||
)
|
||||
|
||||
type HubItems map[string]map[string]*Item
|
||||
|
||||
// ItemVersion is used to detect the version of a given item
|
||||
// by comparing the hash of each version to the local file.
|
||||
// If the item does not match any known version, it is considered tainted.
|
||||
// If the item does not match any known version, it is considered tainted (modified).
|
||||
type ItemVersion struct {
|
||||
Digest string `json:"digest,omitempty"` // meow
|
||||
Deprecated bool `json:"deprecated,omitempty"`
|
||||
}
|
||||
|
||||
// ItemState is used to keep the local state (i.e. at runtime) of an item
|
||||
// This data is not stored in the index, but is displayed in the output of "cscli ... inspect"
|
||||
// ItemState is used to keep the local state (i.e. at runtime) of an item.
|
||||
// This data is not stored in the index, but is displayed with "cscli ... inspect".
|
||||
type ItemState struct {
|
||||
LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR}
|
||||
LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"`
|
||||
LocalVersion string `json:"local_version,omitempty"`
|
||||
LocalHash string `json:"local_hash,omitempty"` // the local meow
|
||||
LocalHash string `json:"local_hash,omitempty"`
|
||||
Installed bool `json:"installed"`
|
||||
Downloaded bool `json:"downloaded"`
|
||||
UpToDate bool `json:"up_to_date"`
|
||||
Tainted bool `json:"tainted"` // has it been locally modified?
|
||||
BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any
|
||||
Tainted bool `json:"tainted"`
|
||||
BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"`
|
||||
}
|
||||
|
||||
// Item represents an object managed in the hub. It can be a parser, scenario, collection..
|
||||
// Item is created from an index file and enriched with local info.
|
||||
type Item struct {
|
||||
// back pointer to the hub, to retrieve subitems and call install/remove methods
|
||||
hub *Hub
|
||||
hub *Hub // back pointer to the hub, to retrieve other items and call install/remove methods
|
||||
|
||||
// local (deployed) info
|
||||
State ItemState
|
||||
State ItemState `json:"-" yaml:"-"` // local state, not stored in the index
|
||||
|
||||
// descriptive info
|
||||
Type string `json:"type,omitempty" yaml:"type,omitempty"` // can be any of the ItemTypes
|
||||
Type string `json:"type,omitempty" yaml:"type,omitempty"` // one of the ItemTypes
|
||||
Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-...
|
||||
Name string `json:"name,omitempty"` // as seen in .index.json, usually "author/name"
|
||||
FileName string `json:"file_name,omitempty"` // the filename, ie. apache2-logs.yaml
|
||||
Description string `json:"description,omitempty" yaml:"description,omitempty"` // as seen in .index.json
|
||||
Author string `json:"author,omitempty"` // as seen in .index.json
|
||||
References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .index.json
|
||||
Name string `json:"name,omitempty"` // usually "author/name"
|
||||
FileName string `json:"file_name,omitempty"` // eg. apache2-logs.yaml
|
||||
Description string `json:"description,omitempty" yaml:"description,omitempty"`
|
||||
Author string `json:"author,omitempty"`
|
||||
References []string `json:"references,omitempty" yaml:"references,omitempty"`
|
||||
|
||||
// remote (hub) info
|
||||
RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // the path relative to (git | hub API) ie. /parsers/stage/author/file.yaml
|
||||
Version string `json:"version,omitempty"` // the last version
|
||||
Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // the list of existing versions
|
||||
RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // path relative to the base URL eg. /parsers/stage/author/file.yaml
|
||||
Version string `json:"version,omitempty"` // the last available version
|
||||
Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // all the known versions
|
||||
|
||||
// if it's a collection, it can have sub items
|
||||
Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"`
|
||||
|
|
@ -81,17 +79,18 @@ type Item struct {
|
|||
Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"`
|
||||
}
|
||||
|
||||
// HasSubItems returns true if items of this type can have sub-items. Currently only collections.
|
||||
func (i *Item) HasSubItems() bool {
|
||||
return i.Type == COLLECTIONS
|
||||
}
|
||||
|
||||
// IsLocal returns true if the item has been create by a user (not downloaded from the hub).
|
||||
func (i *Item) IsLocal() bool {
|
||||
return i.State.Installed && !i.State.Downloaded
|
||||
}
|
||||
|
||||
// MarshalJSON is used to add the "local" field to the json output
|
||||
// (i.e. with cscli ... inspect -o json)
|
||||
// It must not use a pointer receiver
|
||||
// MarshalJSON is used to prepare the output for "cscli ... inspect -o json".
|
||||
// It must not use a pointer receiver.
|
||||
func (i Item) MarshalJSON() ([]byte, error) {
|
||||
type Alias Item
|
||||
|
||||
|
|
@ -121,9 +120,8 @@ func (i Item) MarshalJSON() ([]byte, error) {
|
|||
})
|
||||
}
|
||||
|
||||
// MarshalYAML is used to add the "local" field to the yaml output
|
||||
// (i.e. with cscli ... inspect -o raw)
|
||||
// It must not use a pointer receiver
|
||||
// MarshalYAML is used to prepare the output for "cscli ... inspect -o raw".
|
||||
// It must not use a pointer receiver.
|
||||
func (i Item) MarshalYAML() (interface{}, error) {
|
||||
type Alias Item
|
||||
|
||||
|
|
@ -138,7 +136,7 @@ func (i Item) MarshalYAML() (interface{}, error) {
|
|||
}, nil
|
||||
}
|
||||
|
||||
// SubItems returns a slice of sub-item pointers, excluding the ones that were not found
|
||||
// SubItems returns a slice of sub-items, excluding the ones that were not found.
|
||||
func (i *Item) SubItems() []*Item {
|
||||
sub := make([]*Item, 0)
|
||||
|
||||
|
|
@ -211,8 +209,8 @@ func (i *Item) logMissingSubItems() {
|
|||
}
|
||||
}
|
||||
|
||||
// ParentCollections returns the list of items (collections) that have this item as a direct dependency
|
||||
func (i *Item) ParentCollections() []*Item {
|
||||
// AncestorCollections returns a slice of items (collections) that have this item as a direct or indirect dependency.
|
||||
func (i *Item) AncestorCollections() []*Item {
|
||||
ret := make([]*Item, 0)
|
||||
|
||||
for _, parentName := range i.State.BelongsToCollections {
|
||||
|
|
@ -228,7 +226,7 @@ func (i *Item) ParentCollections() []*Item {
|
|||
}
|
||||
|
||||
// Status returns the status of the item as a string and an emoji
|
||||
// ie. "enabled,update-available" and emoji.Warning
|
||||
// (eg. "enabled,update-available" and emoji.Warning).
|
||||
func (i *Item) Status() (string, emoji.Emoji) {
|
||||
status := "disabled"
|
||||
ok := false
|
||||
|
|
@ -269,47 +267,47 @@ func (i *Item) Status() (string, emoji.Emoji) {
|
|||
return status, emo
|
||||
}
|
||||
|
||||
// versionStatus: semver requires 'v' prefix
|
||||
// versionStatus returns the status of the item version compared to the hub version.
|
||||
// semver requires the 'v' prefix.
|
||||
func (i *Item) versionStatus() int {
|
||||
local, err := semver.NewVersion(i.State.LocalVersion)
|
||||
if err != nil {
|
||||
return VersionUnknown
|
||||
return versionUnknown
|
||||
}
|
||||
|
||||
// hub versions are already validated while syncing, ignore errors
|
||||
latest, _ := semver.NewVersion(i.Version)
|
||||
|
||||
if local.LessThan(latest) {
|
||||
return VersionUpdateAvailable
|
||||
return versionUpdateAvailable
|
||||
}
|
||||
|
||||
if local.Equal(latest) {
|
||||
return VersionUpToDate
|
||||
return versionUpToDate
|
||||
}
|
||||
|
||||
return VersionFuture
|
||||
return versionFuture
|
||||
}
|
||||
|
||||
// validPath returns true if the (relative) path is allowed for the item
|
||||
// dirNname: the directory name (ie. crowdsecurity)
|
||||
// fileName: the filename (ie. apache2-logs.yaml)
|
||||
// validPath returns true if the (relative) path is allowed for the item.
|
||||
// dirNname: the directory name (ie. crowdsecurity).
|
||||
// fileName: the filename (ie. apache2-logs.yaml).
|
||||
func (i *Item) validPath(dirName, fileName string) bool {
|
||||
return (dirName+"/"+fileName == i.Name+".yaml") || (dirName+"/"+fileName == i.Name+".yml")
|
||||
}
|
||||
|
||||
// GetItemMap returns the map of items for a given type
|
||||
// GetItemMap returns the map of items for a given type.
|
||||
func (h *Hub) GetItemMap(itemType string) map[string]*Item {
|
||||
return h.Items[itemType]
|
||||
}
|
||||
|
||||
// GetItem returns the item from hub based on its type and full name (author/name)
|
||||
// GetItem returns an item from hub based on its type and full name (author/name).
|
||||
func (h *Hub) GetItem(itemType string, itemName string) *Item {
|
||||
return h.GetItemMap(itemType)[itemName]
|
||||
}
|
||||
|
||||
// GetItemNames returns the list of (full) item names for a given type
|
||||
// ie. for collections: crowdsecurity/apache2 crowdsecurity/nginx
|
||||
// The names can be used to retrieve the item with GetItem()
|
||||
// GetItemNames returns a slice of (full) item names for a given type
|
||||
// (eg. for collections: crowdsecurity/apache2 crowdsecurity/nginx).
|
||||
func (h *Hub) GetItemNames(itemType string) []string {
|
||||
m := h.GetItemMap(itemType)
|
||||
if m == nil {
|
||||
|
|
@ -324,7 +322,7 @@ func (h *Hub) GetItemNames(itemType string) []string {
|
|||
return names
|
||||
}
|
||||
|
||||
// GetAllItems returns a slice of all the items, installed or not
|
||||
// GetAllItems returns a slice of all the items of a given type, installed or not.
|
||||
func (h *Hub) GetAllItems(itemType string) ([]*Item, error) {
|
||||
items, ok := h.Items[itemType]
|
||||
if !ok {
|
||||
|
|
@ -343,7 +341,7 @@ func (h *Hub) GetAllItems(itemType string) ([]*Item, error) {
|
|||
return ret, nil
|
||||
}
|
||||
|
||||
// GetInstalledItems returns the list of installed items
|
||||
// GetInstalledItems returns a slice of the installed items of a given type.
|
||||
func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) {
|
||||
items, ok := h.Items[itemType]
|
||||
if !ok {
|
||||
|
|
@ -361,7 +359,7 @@ func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) {
|
|||
return retItems, nil
|
||||
}
|
||||
|
||||
// GetInstalledItemNames returns the names of the installed items
|
||||
// GetInstalledItemNames returns the names of the installed items of a given type.
|
||||
func (h *Hub) GetInstalledItemNames(itemType string) ([]string, error) {
|
||||
items, err := h.GetInstalledItems(itemType)
|
||||
if err != nil {
|
||||
|
|
@ -377,7 +375,7 @@ func (h *Hub) GetInstalledItemNames(itemType string) ([]string, error) {
|
|||
return retStr, nil
|
||||
}
|
||||
|
||||
// SortItemSlice sorts a slice of items by name, case insensitive
|
||||
// SortItemSlice sorts a slice of items by name, case insensitive.
|
||||
func SortItemSlice(items []*Item) {
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import (
|
|||
"strings"
|
||||
)
|
||||
|
||||
// itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary
|
||||
// itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary.
|
||||
func itemKey(itemPath string) (string, error) {
|
||||
f, err := os.Lstat(itemPath)
|
||||
if err != nil {
|
||||
|
|
@ -37,7 +37,7 @@ func itemKey(itemPath string) (string, error) {
|
|||
return fmt.Sprintf("%s/%s", author, fname), nil
|
||||
}
|
||||
|
||||
// GetItemByPath retrieves the item from the hub index based on its path.
|
||||
// GetItemByPath retrieves an item from the hub index based on its local path.
|
||||
func (h *Hub) GetItemByPath(itemType string, itemPath string) (*Item, error) {
|
||||
itemKey, err := itemKey(itemPath)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -6,14 +6,14 @@ import (
|
|||
"net/http"
|
||||
)
|
||||
|
||||
// RemoteHubCfg contains where to find the remote hub, which branch etc.
|
||||
// RemoteHubCfg is used to retrieve index and items from the remote hub.
|
||||
type RemoteHubCfg struct {
|
||||
Branch string
|
||||
URLTemplate string
|
||||
IndexPath string
|
||||
}
|
||||
|
||||
// urlTo builds the URL to download a file from the remote hub
|
||||
// urlTo builds the URL to download a file from the remote hub.
|
||||
func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) {
|
||||
if r == nil {
|
||||
return "", ErrNilRemoteHub
|
||||
|
|
@ -27,7 +27,7 @@ func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) {
|
|||
return fmt.Sprintf(r.URLTemplate, r.Branch, remotePath), nil
|
||||
}
|
||||
|
||||
// fetchIndex downloads the index from the hub and returns the content
|
||||
// fetchIndex downloads the index from the hub and returns the content.
|
||||
func (r *RemoteHubCfg) fetchIndex() ([]byte, error) {
|
||||
if r == nil {
|
||||
return nil, ErrNilRemoteHub
|
||||
|
|
|
|||
|
|
@ -12,13 +12,14 @@ import (
|
|||
|
||||
"github.com/Masterminds/semver/v3"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func isYAMLFileName(path string) bool {
|
||||
return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")
|
||||
}
|
||||
|
||||
// linkTarget returns the target of a symlink, or empty string if it's dangling
|
||||
// linkTarget returns the target of a symlink, or empty string if it's dangling.
|
||||
func linkTarget(path string) (string, error) {
|
||||
hubpath, err := os.Readlink(path)
|
||||
if err != nil {
|
||||
|
|
@ -52,7 +53,7 @@ func getSHA256(filepath string) (string, error) {
|
|||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// information used to create a new Item, from a file path
|
||||
// information used to create a new Item, from a file path.
|
||||
type itemFileInfo struct {
|
||||
inhub bool
|
||||
fname string
|
||||
|
|
@ -127,7 +128,7 @@ func (h *Hub) getItemFileInfo(path string) (*itemFileInfo, error) {
|
|||
return ret, nil
|
||||
}
|
||||
|
||||
// sortedVersions returns the input data, sorted in reverse order (new, old) by semver
|
||||
// sortedVersions returns the input data, sorted in reverse order (new, old) by semver.
|
||||
func sortedVersions(raw []string) ([]string, error) {
|
||||
vs := make([]*semver.Version, len(raw))
|
||||
|
||||
|
|
@ -150,10 +151,14 @@ func sortedVersions(raw []string) ([]string, error) {
|
|||
return ret, nil
|
||||
}
|
||||
|
||||
func newLocalItem(h *Hub, path string, info *itemFileInfo) *Item {
|
||||
func newLocalItem(h *Hub, path string, info *itemFileInfo) (*Item, error) {
|
||||
type localItemName struct {
|
||||
Name string `yaml:"name"`
|
||||
}
|
||||
|
||||
_, fileName := filepath.Split(path)
|
||||
|
||||
return &Item{
|
||||
item := &Item{
|
||||
hub: h,
|
||||
Name: info.fname,
|
||||
Stage: info.stage,
|
||||
|
|
@ -165,6 +170,25 @@ func newLocalItem(h *Hub, path string, info *itemFileInfo) *Item {
|
|||
UpToDate: true,
|
||||
},
|
||||
}
|
||||
|
||||
// try to read the name from the file
|
||||
itemName := localItemName{}
|
||||
|
||||
itemContent, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read %s: %w", path, err)
|
||||
}
|
||||
|
||||
err = yaml.Unmarshal(itemContent, &itemName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal %s: %w", path, err)
|
||||
}
|
||||
|
||||
if itemName.Name != "" {
|
||||
item.Name = itemName.Name
|
||||
}
|
||||
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
|
||||
|
|
@ -198,7 +222,11 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
|
|||
|
||||
if !info.inhub {
|
||||
log.Tracef("%s is a local file, skip", path)
|
||||
h.Items[info.ftype][info.fname] = newLocalItem(h, path, info)
|
||||
item, err := newLocalItem(h, path, info)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
h.Items[info.ftype][item.Name] = item
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -269,13 +297,13 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// checkSubItems checks for the presence, taint and version state of sub-items
|
||||
// checkSubItems checks for the presence, taint and version state of sub-items.
|
||||
func (h *Hub) checkSubItems(v *Item) error {
|
||||
if !v.HasSubItems() {
|
||||
return nil
|
||||
}
|
||||
|
||||
if v.versionStatus() != VersionUpToDate {
|
||||
if v.versionStatus() != versionUpToDate {
|
||||
log.Debugf("%s dependencies not checked: not up-to-date", v.Name)
|
||||
return nil
|
||||
}
|
||||
|
|
@ -321,7 +349,7 @@ func (h *Hub) checkSubItems(v *Item) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// syncDir scans a directory for items, and updates the Hub state accordingly
|
||||
// syncDir scans a directory for items, and updates the Hub state accordingly.
|
||||
func (h *Hub) syncDir(dir string) error {
|
||||
// For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last
|
||||
for _, scan := range ItemTypes {
|
||||
|
|
@ -347,7 +375,7 @@ func (h *Hub) syncDir(dir string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// insert a string in a sorted slice, case insensitive, and return the new slice
|
||||
// insert a string in a sorted slice, case insensitive, and return the new slice.
|
||||
func insertInOrderNoCase(sl []string, value string) []string {
|
||||
i := sort.Search(len(sl), func(i int) bool {
|
||||
return strings.ToLower(sl[i]) >= strings.ToLower(value)
|
||||
|
|
@ -356,7 +384,7 @@ func insertInOrderNoCase(sl []string, value string) []string {
|
|||
return append(sl[:i], append([]string{value}, sl[i:]...)...)
|
||||
}
|
||||
|
||||
// localSync updates the hub state with downloaded, installed and local items
|
||||
// localSync updates the hub state with downloaded, installed and local items.
|
||||
func (h *Hub) localSync() error {
|
||||
err := h.syncDir(h.local.InstallDir)
|
||||
if err != nil {
|
||||
|
|
@ -387,17 +415,19 @@ func (h *Hub) localSync() error {
|
|||
|
||||
vs := item.versionStatus()
|
||||
switch vs {
|
||||
case VersionUpToDate: // latest
|
||||
case versionUpToDate: // latest
|
||||
if err := h.checkSubItems(item); err != nil {
|
||||
warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err))
|
||||
}
|
||||
case VersionUpdateAvailable: // not up-to-date
|
||||
case versionUpdateAvailable: // not up-to-date
|
||||
warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version))
|
||||
case VersionFuture:
|
||||
case versionFuture:
|
||||
warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version))
|
||||
case VersionUnknown:
|
||||
case versionUnknown:
|
||||
if !item.IsLocal() {
|
||||
warnings = append(warnings, fmt.Sprintf("collection %s is tainted (latest:%s)", item.Name, item.Version))
|
||||
}
|
||||
}
|
||||
|
||||
log.Debugf("installed (%s) - status: %d | installed: %s | latest: %s | full: %+v", item.Name, vs, item.State.LocalVersion, item.Version, item.Versions)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -107,3 +107,43 @@ teardown() {
|
|||
refute_output
|
||||
refute_stderr
|
||||
}
|
||||
|
||||
@test "a local item is not tainted" {
|
||||
# not from cscli... inspect
|
||||
rune -0 mkdir -p "$CONFIG_DIR/collections"
|
||||
rune -0 touch "$CONFIG_DIR/collections/foobar.yaml"
|
||||
rune -0 cscli collections inspect foobar.yaml -o json
|
||||
rune -0 jq -e '.tainted==false' <(output)
|
||||
|
||||
rune -0 cscli collections install crowdsecurity/sshd
|
||||
rune -0 truncate -s0 "$CONFIG_DIR/collections/sshd.yaml"
|
||||
rune -0 cscli collections inspect crowdsecurity/sshd -o json
|
||||
rune -0 jq -e '.tainted==true' <(output)
|
||||
|
||||
# and not from hub update
|
||||
rune -0 cscli hub update
|
||||
assert_stderr --partial "collection crowdsecurity/sshd is tainted"
|
||||
refute_stderr --partial "collection foobar.yaml is tainted"
|
||||
}
|
||||
|
||||
@test "a local item's name defaults to its filename" {
|
||||
rune -0 mkdir -p "$CONFIG_DIR/collections"
|
||||
rune -0 touch "$CONFIG_DIR/collections/foobar.yaml"
|
||||
rune -0 cscli collections list -o json
|
||||
rune -0 jq -r '.[][].name' <(output)
|
||||
assert_output "foobar.yaml"
|
||||
rune -0 cscli collections list foobar.yaml
|
||||
rune -0 cscli collections inspect foobar.yaml -o json
|
||||
rune -0 jq -e '.installed==true' <(output)
|
||||
}
|
||||
|
||||
@test "a local item can provide its own name" {
|
||||
rune -0 mkdir -p "$CONFIG_DIR/collections"
|
||||
echo "name: hi-its-me" > "$CONFIG_DIR/collections/foobar.yaml"
|
||||
rune -0 cscli collections list -o json
|
||||
rune -0 jq -r '.[][].name' <(output)
|
||||
assert_output "hi-its-me"
|
||||
rune -0 cscli collections list hi-its-me
|
||||
rune -0 cscli collections inspect hi-its-me -o json
|
||||
rune -0 jq -e '.installed==true' <(output)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue