From 17cd792826c6711ed08b5f188c7f165c9840de9e Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Thu, 29 Jun 2023 16:35:19 +0200 Subject: [PATCH] CI: update ansible tests for re2 (#2318) --- Dockerfile | 3 +- Makefile | 10 +- go.mod | 5 +- go.sum | 15 +- mk/gmsl.html | 733 ++++++++++++++++++ mk/platform/unix_common.mk | 8 +- mk/platform/windows.mk | 8 +- pkg/cwhub/dataset_test.go | 2 +- test/ansible/provision_dependencies.yml | 3 + test/ansible/requirements.yml | 4 +- .../ansible/roles/make_fixture/tasks/main.yml | 14 +- test/ansible/vagrant/alma-8/Vagrantfile | 1 + test/ansible/vagrant/alma-8/bootstrap | 5 + test/ansible/vagrant/alma-9/Vagrantfile | 1 + test/ansible/vagrant/alma-9/bootstrap | 5 + test/ansible/vagrant/centos-8/Vagrantfile | 3 +- test/ansible/vagrant/centos-8/bootstrap | 6 + test/ansible/vagrant/centos-9/Vagrantfile | 1 + test/ansible/vagrant/centos-9/bootstrap | 6 + test/ansible/vagrant/fedora-37/Vagrantfile | 1 + test/ansible/vagrant/fedora-37/bootstrap | 5 + test/ansible/vagrant/freebsd-12/Vagrantfile | 1 + test/ansible/vagrant/freebsd-12/bootstrap | 6 + test/ansible/vagrant/freebsd-13/Vagrantfile | 1 + test/ansible/vagrant/freebsd-13/bootstrap | 6 + .../vagrant/ubuntu-20.04-focal/bootstrap | 5 + test/bats.mk | 2 +- test/bats/01_cscli.bats | 6 +- test/bin/assert-crowdsec-not-running | 11 +- 29 files changed, 838 insertions(+), 39 deletions(-) create mode 100644 mk/gmsl.html create mode 100755 test/ansible/vagrant/alma-8/bootstrap create mode 100755 test/ansible/vagrant/alma-9/bootstrap create mode 100755 test/ansible/vagrant/centos-8/bootstrap create mode 100755 test/ansible/vagrant/centos-9/bootstrap create mode 100755 test/ansible/vagrant/fedora-37/bootstrap create mode 100644 test/ansible/vagrant/freebsd-12/bootstrap create mode 100644 test/ansible/vagrant/freebsd-13/bootstrap create mode 100644 test/ansible/vagrant/ubuntu-20.04-focal/bootstrap diff --git a/Dockerfile b/Dockerfile index d5a3ee273..2073d3c5a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,11 +9,10 @@ WORKDIR /go/src/crowdsec ENV RE2_VERSION=2023-03-01 # wizard.sh requires GNU coreutils -RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold coreutils icu-static pkgconfig && \ +RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold coreutils pkgconfig && \ wget https://github.com/google/re2/archive/refs/tags/${RE2_VERSION}.tar.gz && \ tar -xzf ${RE2_VERSION}.tar.gz && \ cd re2-${RE2_VERSION} && \ - make && \ make install && \ echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \ go install github.com/mikefarah/yq/v4@v4.34.1 diff --git a/Makefile b/Makefile index 04bb12c3f..8943c6782 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ BUILD_RE2_WASM ?= 0 # To build static binaries, run "make BUILD_STATIC=1". # On some platforms, this requires -# additional packages (e.g. glibc-static on fedora, centos..). +# additional packages (e.g. glibc-static and libstdc++-static on fedora, centos..). # If the static build fails at the link stage, it might be because the static library is not provided # for your distribution (look for libre2.a). See the Dockerfile for an example of how to build it. BUILD_STATIC ?= 0 @@ -77,10 +77,14 @@ LD_OPTS_VARS += -X '$(GO_MODULE_NAME)/pkg/cwversion.System=docker' endif GO_TAGS := netgo,osusergo,sqlite_omit_load_extension +# this will be used by Go in the make target +export PKG_CONFIG_PATH:=/usr/local/lib/pkgconfig:$(PKG_CONFIG_PATH) ifeq ($(call bool,$(BUILD_RE2_WASM)),0) -# see if we have libre2-dev installed for C++ optimizations -RE2_CHECK := $(shell pkg-config --libs re2 2>/dev/null) +ifeq ($(PKG_CONFIG),) + $(error "pkg-config is not available. Please install pkg-config.") +endif + ifeq ($(RE2_CHECK),) # we could detect the platform and suggest the command to install RE2_FAIL := "libre2-dev is not installed, please install it or set BUILD_RE2_WASM=1 to use the WebAssembly version" diff --git a/go.mod b/go.mod index 7541b8f28..a92924c27 100644 --- a/go.mod +++ b/go.mod @@ -65,7 +65,7 @@ require ( github.com/prometheus/prom2json v1.3.0 github.com/r3labs/diff/v2 v2.14.1 github.com/segmentio/kafka-go v0.4.34 - github.com/shirou/gopsutil/v3 v3.22.12 + github.com/shirou/gopsutil/v3 v3.23.5 github.com/sirupsen/logrus v1.9.2 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.3 @@ -163,6 +163,7 @@ require ( github.com/rivo/uniseg v0.2.0 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/shopspring/decimal v1.2.0 // indirect github.com/spf13/cast v1.3.1 // indirect github.com/spf13/pflag v1.0.5 // indirect @@ -172,7 +173,7 @@ require ( github.com/tklauser/numcpus v0.6.0 // indirect github.com/ugorji/go/codec v1.2.6 // indirect github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect - github.com/yusufpapurcu/wmi v1.2.2 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect github.com/zclconf/go-cty v1.8.0 // indirect go.mongodb.org/mongo-driver v1.9.4 // indirect golang.org/x/net v0.7.0 // indirect diff --git a/go.sum b/go.sum index 1fa4000aa..41bff5d47 100644 --- a/go.sum +++ b/go.sum @@ -831,8 +831,12 @@ github.com/segmentio/kafka-go v0.4.34 h1:Dm6YlLMiVSiwwav20KY0AoY63s661FXevwJ3CVH github.com/segmentio/kafka-go v0.4.34/go.mod h1:GAjxBQJdQMB5zfNA21AhpaqOB2Mu+w3De4ni3Gbm8y0= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shirou/gopsutil/v3 v3.22.12 h1:oG0ns6poeUSxf78JtOsfygNWuEHYYz8hnnNg7P04TJs= -github.com/shirou/gopsutil/v3 v3.22.12/go.mod h1:Xd7P1kwZcp5VW52+9XsirIKd/BROzbb2wdX3Kqlz9uI= +github.com/shirou/gopsutil/v3 v3.23.5 h1:5SgDCeQ0KW0S4N0znjeM/eFHXXOKyv2dVNgRq/c9P6Y= +github.com/shirou/gopsutil/v3 v3.23.5/go.mod h1:Ng3Maa27Q2KARVJ0SPZF5NdrQSC3XHKP8IIWrHgMeLY= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= @@ -883,7 +887,6 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= @@ -937,8 +940,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= -github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zclconf/go-cty v1.8.0 h1:s4AvqaeQzJIu3ndv4gVIhplVD0krU+bgrcLSVUnaWuA= github.com/zclconf/go-cty v1.8.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= @@ -1203,7 +1206,7 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= diff --git a/mk/gmsl.html b/mk/gmsl.html new file mode 100644 index 000000000..a8da46987 --- /dev/null +++ b/mk/gmsl.html @@ -0,0 +1,733 @@ + + + + GNU Make Standard Library + + +

GNU Make Standard Library

+The GNU Make Standard Library (GMSL) is a collection of functions +implemented using native GNU Make functionality that provide list and +string manipulation, integer arithmetic, associative arrays, stacks, +and debugging facilities.  The GMSL is released under the BSD License.
+
+[Project Page] +[Releases] +
+

Using GMSL

+The two files needed are gmsl +and __gmsl.  To +include the GMSL in your Makefile do
+
include gmsl
+gmsl automatically includes __gmsl.  To check that +you have the right version of gmsl +use the gmsl_compatible +function (see +below). The current version is 1 2 0.
+
+The GMSL package also includes a test suite for GMSL.  Just run make -f gmsl-tests.
+

Logical Operators

GMSL has boolean $(true) (a non-empty string) +and $(false) (an empty string).  The following operators can be +used with those variables.
+
+
not
+ +
+ +Arguments: A boolean value
+ +Returns:   Returns $(true) if the boolean is $(false) and vice versa
+ +
and
+
+Arguments: Two boolean values
+Returns:   Returns $(true) if both of the booleans are true
+
or
+
+Arguments: Two boolean values
+Returns:   Returns $(true) if either of the booleans is true
+
xor
+
+Arguments: Two boolean values
+Returns:   Returns $(true) if exactly one of the booleans is true
+
nand
+
+Arguments: Two boolean values
+Returns:   Returns value of 'not and'
+
nor
+
+Arguments: Two boolean values
+Returns:   Returns value of 'not or'
+
xnor
+
+Arguments: Two boolean values
+Returns:   Returns value of 'not xor'
+
+

List Manipulation Functions

+ A list is a string of characters; the list separator is a space.
+ +
+
first
+
+Arguments: 1: A list
+Returns:   Returns the first element of a list
+ +
last
+
+Arguments: 1: A list
+Returns:   Returns the last element of a list
+ +
rest
+
+Arguments: 1: A list
+Returns:   Returns the list with the first element +removed
+ +
chop
+
+Arguments: 1: A list
+Returns:   Returns the list with the last element removed
+ +
map
+
+Arguments: 1: Name of function to +$(call) for each element of list
+           2: List to +iterate over calling the function in 1
+Returns:   The list after calling the function on each +element
+ +
pairmap
+
+Arguments: 1: Name of function to +$(call) for each pair of elements
+           2: List to +iterate over calling the function in 1
+           3: Second +list to iterate over calling the function in 1
+Returns:   The list after calling the function on each +pair of elements
+ +
leq
+
+Arguments: 1: A list to compare +against...
+           2: ...this +list
+Returns:   Returns $(true) if the two lists are identical
+ +
lne
+
+Arguments: 1: A list to compare +against...
+           2: ...this +list
+Returns:   Returns $(true) if the two lists are different
+ +
reverse
+
+Arguments: 1: A list to reverse
+Returns:   The list with its elements in reverse order
+ +
uniq
+
+Arguments: 1: A list to deduplicate
+Returns:   The list with elements in order without duplicates
+ +
length
+
+Arguments: 1: A list
+Returns:   The number of elements in the list
+ +
+

String Manipulation Functions

+A string is any sequence of characters.
+
+
seq
+
+Arguments: 1: A string to compare +against...
+           2: ...this +string
+Returns:   Returns $(true) if the two strings are +identical
+ +
sne
+
+Arguments: 1: A string to compare +against...
+           2: ...this +string
+Returns:   Returns $(true) if the two strings are not +the same
+ +
strlen
+
+Arguments: 1: A string
+Returns:   Returns the length of the string
+ +
substr
+
+Arguments: 1: A string
+           2: Start offset (first character is 1)
+           3: Ending offset (inclusive)
Returns:   Returns a substring
+ +
split
+
+Arguments: 1: The character to +split on
+           2: A +string to split
+Returns:   Splits a string into a list separated by +spaces at the split
+           character +in the first argument
+ +
merge
+
+Arguments: 1: The character to +put between fields
+           2: A list +to merge into a string
+Returns:   Merges a list into a single string, list +elements are separated
+           by the +character in the first argument
+ +
tr
+
+Arguments: 1: The list of +characters to translate from
+           2: The +list of characters to translate to
+           3: The +text to translate
+Returns:   Returns the text after translating characters
+ +
uc
+
+Arguments: 1: Text to upper case
+Returns:   Returns the text in upper case
+ +
lc
+
+Arguments: 1: Text to lower case
+Returns:   Returns the text in lower case
+ +
+

Set Manipulation Functions

+Sets are represented by sorted, deduplicated lists. To create a set +from a list use set_create, or start with the empty_set and set_insert individual elements. +The empty set is defined as empty_set.

+ +

set_create
+
+Arguments: 1: A list of set elements
+Returns:   Returns the newly created set
+ + +
set_insert
+
+Arguments: 1: A single element to add to a set
+           2: A set
+Returns:   Returns the set with the element added
+ + +
set_remove
+
+Arguments: 1: A single element to remove from a set
+           2: A set
+Returns:   Returns the set with the element removed
+ + +
set_is_member
+
+Arguments: 1: A single element
+           2: A set
+Returns:   Returns $(true) if the element is in the set
+ + +
set_is_not_member
+
+Arguments: 1: A single element
+           2: A set
+Returns:   Returns $(false) if the element is in the set
+ + +
set_union
+
+Arguments: 1: A set
+           2: Another set
+Returns:   Returns the union of the two sets
+ + +
set_intersection
+
+Arguments: 1: A set
+           2: Another set
+Returns:   Returns the intersection of the two sets
+ + +
set_is_subset
+
+Arguments: 1: A set
+           2: Another set
+Returns:   Returns $(true) if the first set is a subset of the second
+ + +
set_equal
+
+Arguments: 1: A set
+           2: Another set
+Returns:   Returns $(true) if the two sets are identical
+ + +
+

Integer Arithmetic Functions

+Integers are represented by lists with the equivalent number of +x's.  For example the number 4 is x x x x.  The maximum +integer that the library can handle as input (i.e. as the argument to a +call to int_encode) is +65536. There is no limit on integer size for internal computations or +output.
+
+The arithmetic library functions come in two forms: one form of each +function takes integers as arguments and the other form takes the +encoded form (x's created by a call to int_encode).  For example, +there are two plus functions: plus +(called with integer arguments and returns an integer) and int_plus (called with encoded +arguments and returns an encoded result).
+
+plus will be slower than int_plus because its arguments +and result have to be translated between the x's format and +integers.  If doing a complex calculation use the int_* forms with a single +encoding of inputs and single decoding of the output.  For simple +calculations the direct forms can be used.
+
+
int_decode
+
+Arguments: 1: A number of x's +representation
+Returns:   Returns the integer for human consumption +that is represented
+           by the +string of x's
+ +
int_encode
+
+Arguments: 1: A number in +human-readable integer form
+Returns:   Returns the integer encoded as a string of x's
+ +
int_plus
+
+Arguments: 1: A number in x's +representation
+           2: Another +number in x's represntation
+Returns:   Returns the sum of the two numbers in x's +representation
+ +
plus (wrapped version of int_plus)
+
+Arguments: 1: An integer
+           2: Another +integer
+Returns:   Returns the sum of the two integers
+ +
int_subtract
+
+Arguments: 1: A number in x's +representation
+           2: Another +number in x's represntation
+Returns:   Returns the difference of the two numbers in +x's representation,
+           or outputs +an error on a numeric underflow
+ +
subtract (wrapped version of int_subtract)
+
+Arguments: 1: An integer
+           2: Another +integer
+Returns:   Returns the difference of the two integers,
+           or outputs +an error on a numeric underflow
+ +
int_multiply
+
+Arguments: 1: A number in x's +representation
+           2: Another +number in x's represntation
+Returns:   Returns the product of the two numbers in x's +representation
+ +
multiply (wrapped version of int_multiply)
+
+Arguments: 1: An integer
+           2: Another +integer
+Returns:   Returns the product of the two integers
+ +
int_divide
+
+Arguments: 1: A number in x's +representation
+           2: Another +number in x's represntation
+Returns:   Returns the result of integer division of +argument 1 divided
+           by +argument 2 in x's representation
+ +
divide (wrapped version of int_divide)
+
+Arguments: 1: An integer
+           2: Another +integer
+Returns:   Returns the integer division of the first +argument by the second
+ +
int_modulo
+
+Arguments: 1: A number in x's +representation
+           2: Another +number in x's represntation
+Returns:   Returns the remainder of integer division of +argument 1 divided
+           by +argument 2 in x's representation
+ +
modulo (wrapped version of int_modulo)
+
+Arguments: 1: An integer
+           2: Another +integer
+Returns:   Returns the remainder of integer division of the first +argument by the second
+ +
int_max, int_min
+
+Arguments: 1: A number in x's +representation
+           2: Another +number in x's represntation
+Returns:   Returns the maximum or minimum of its +arguments in x's
+           +representation
+ +
max, min
+
+Arguments: 1: An integer
+           2: Another +integer
+Returns:   Returns the maximum or minimum of its integer +arguments
+ +
int_gt, int_gte, int_lt, int_lte, int_eq, int_ne
+
+Arguments: Two x's representation +numbers to be compared
+Returns:   $(true) or $(false)
+
+int_gt First argument greater than second argument
+int_gte First argument greater than or equal to second argument
+int_lt First argument less than second argument
+int_lte First argument less than or equal to second argument
+int_eq First argument is numerically equal to the second argument
+int_ne First argument is not numerically equal to the second argument
+ +
gt, gte, lt, lte, eq, ne
+
+Arguments: Two integers to be +compared
+Returns:   $(true) or $(false)
+
+gt First argument greater than second argument
+gte First argument greater than or equal to second argument
+lt First argument less than second argument
+lte First argument less than or equal to second argument
+eq First argument is numerically equal to the second argument
+ne First argument is not numerically equal to the second argument
+ +increment adds 1 to its argument, decrement subtracts 1. Note that
+decrement does not range check and hence will not underflow, but
+will incorrectly say that 0 - 1 = 0
+
int_inc
+
+Arguments: 1: A number in x's +representation
+Returns:   The number incremented by 1 in x's +representation
+ +
inc
+
+Arguments: 1: An integer
+Returns:   The argument incremented by 1
+ +
int_dec
+
+Arguments: 1: A number in x's +representation
+Returns:   The number decremented by 1 in x's +representation
+ +
dec
+
+Arguments: 1: An integer
+Returns:   The argument decremented by 1
+ +
int_double
+
+Arguments: 1: A number in x's +representation
+Returns:   The number doubled (i.e. * 2) and returned in +x's representation
+ +
double
+
+Arguments: 1: An integer
+Returns:   The integer times 2
+ +
int_halve
+
+Arguments: 1: A number in x's +representation
+Returns:   The number halved (i.e. / 2) and returned in +x's representation
+ +
halve
+
+Arguments: 1: An integer
+Returns:   The integer divided by 2
+ +
sequence
+
+Arguments: 1: An integer
+           2: An integer
+Returns:   The sequence [arg1 arg2] if arg1 >= arg2 or [arg2 arg1] if arg2 > arg1
+ +
dec2hex, dec2bin, dec2oct
+
+Arguments: 1: An integer
+Returns:   The decimal argument converted to hexadecimal, binary or octal
+ +
+

Associative Arrays

+An associate array maps a key value (a string with no spaces in it) to +a single value (any string).   
+
+ +
set
+
+Arguments: 1: Name of associative +array
+           2: The key +value to associate
+           3: The +value associated with the key
+Returns:   Nothing
+ +
get
+
+Arguments: 1: Name of associative +array
+           2: The key +to retrieve
+Returns:   The value stored in the array for that key
+ +
keys
+
+Arguments: 1: Name of associative +array
+Returns:   Returns a list of all defined keys in the +array
+ +
defined
+
+Arguments: 1: Name of associative +array
+           2: The key +to test
+Returns:   Returns true if the key is defined (i.e. not +empty)
+ +
+

Named Stacks

+A stack is an ordered list of strings (with no spaces in them).
+
+
push
+
+Arguments: 1: Name of stack
+           2: Value +to push onto the top of the stack (must not contain
+           a space)
+Returns:   None
+ +
pop
+
+Arguments: 1: Name of stack
+Returns:   Top element from the stack after removing it
+ +
peek
+
+Arguments: 1: Name of stack
+Returns:   Top element from the stack without removing it
+ +
depth
+
+Arguments: 1: Name of stack
+Returns:   Number of items on the stack
+ +
+

Function memoization

+To reduce the number of calls to slow functions (such as $(shell) a single memoization function is provided.
+
+
memoize
+
+Arguments: 1: Name of function to memoize
+           2: String argument for the function
+Returns:   Result of $1 applied to $2 but only calls $1 once for each unique $2
+ + +
+

Miscellaneous and Debugging Facilities

+GMSL defines the following constants; all are accessed as normal GNU +Make variables by wrapping them in $() or ${}.
+
+ + + + + + + + + + + + + + + + + + + + + + + +
Constant
+ 		Value
+ 		Purpose
+
true
+ 		T
+ 		Boolean for $(if) +and return from  GMSL functions
+
false
+
+ 		Boolean for $(if) +and return from GMSL functions
+
gmsl_version
+ 		1 0 0
+ 		GMSL version number as list: major minor revision
+
+
+gmsl_compatible
+
+Arguments: List containing the desired library version number (maj min +rev)
+Returns:   +$(true) if this version of the library is compatible
+           +with the requested version number, otherwise $(false) +
gmsl-print-% (target not a function)
+
+Arguments: The % should be +replaced by the name of a variable that you
+           wish to +print out.
+Action:    Echos the name of the variable that matches +the % and its value.
+           For +example, 'make gmsl-print-SHELL' will output the value of
+           the SHELL +variable
+ +
gmsl-echo-% (target not a function)
+
+Arguments: The % should be +replaced by the name of a variable that you
+           wish to +print out.
+Action:    Echos the value of the variable that matches +the %.
+           For +example, 'make gmsl-echo-SHELL' will output the value of
+           the SHELL +variable
+ +
assert
+
+Arguments: 1: A boolean that must +be true or the assertion will fail
+           2: The +message to print with the assertion
+Returns:   None
+ +
assert_exists
+
+Arguments: 1: Name of file that +must exist, if it is missing an assertion
+           will be +generated
+Returns:   None
+ +

+GMSL has a number of environment variables (or command-line overrides) +that control various bits of functionality:
+
+ + + + + + + + + + + + + + + + + + + +
Variable
+ 		Purpose
+
GMSL_NO_WARNINGS
+ 		If set prevents GMSL from outputting warning messages: +artithmetic functions generate underflow warnings.
+
GMSL_NO_ERRORS
+ 		If set prevents GMSL from generating fatal errors: division +by zero or failed assertions are fatal.
+
GMSL_TRACE
+ 		Enables function tracing.  Calls to GMSL functions will +result in name and arguments being traced.
+
+
+
+Copyright (c) 2005-2022 John Graham-Cumming.
+
+ diff --git a/mk/platform/unix_common.mk b/mk/platform/unix_common.mk index f611693f4..8f06c9328 100644 --- a/mk/platform/unix_common.mk +++ b/mk/platform/unix_common.mk @@ -7,8 +7,14 @@ MKDIR=mkdir -p # Go should not be required to run functional tests GOOS ?= $(shell go env GOOS) -#Current versioning information from env +# Current versioning information from env BUILD_VERSION?=$(shell git describe --tags) BUILD_TIMESTAMP=$(shell date +%F"_"%T) DEFAULT_CONFIGDIR?=/etc/crowdsec DEFAULT_DATADIR?=/var/lib/crowdsec/data + +PKG_CONFIG:=$(shell command -v pkg-config 2>/dev/null) + +# See if we have libre2-dev installed for C++ optimizations. +# In fedora and other distros, we need to tell where to find re2.pc +RE2_CHECK := $(shell PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$(PKG_CONFIG_PATH) pkg-config --libs re2 2>/dev/null) diff --git a/mk/platform/windows.mk b/mk/platform/windows.mk index 8e2cdf19b..68abc6853 100644 --- a/mk/platform/windows.mk +++ b/mk/platform/windows.mk @@ -4,9 +4,9 @@ MAKE=make GOOS=windows PREFIX=$(shell $$env:TEMP) -#Current versioning information from env -#BUILD_VERSION?=$(shell (Invoke-WebRequest -UseBasicParsing -Uri https://api.github.com/repos/crowdsecurity/crowdsec/releases/latest).Content | jq -r '.tag_name') -#hardcode it till i find a workaround +# Current versioning information from env +# BUILD_VERSION?=$(shell (Invoke-WebRequest -UseBasicParsing -Uri https://api.github.com/repos/crowdsecurity/crowdsec/releases/latest).Content | jq -r '.tag_name') +# hardcode it till I find a workaround BUILD_VERSION?=$(shell git describe --tags $$(git rev-list --tags --max-count=1)) BUILD_TIMESTAMP?=$(shell Get-Date -Format "yyyy-MM-dd_HH:mm:ss") DEFAULT_CONFIGDIR?=C:\\ProgramData\\CrowdSec\\config @@ -18,3 +18,5 @@ CP=Copy-Item CPR=Copy-Item -Recurse MKDIR=New-Item -ItemType directory WIN_IGNORE_ERR=; exit 0 + +PKG_CONFIG:=$(shell Get-Command pkg-config -ErrorAction SilentlyContinue) diff --git a/pkg/cwhub/dataset_test.go b/pkg/cwhub/dataset_test.go index b903915ac..106268c01 100644 --- a/pkg/cwhub/dataset_test.go +++ b/pkg/cwhub/dataset_test.go @@ -9,7 +9,7 @@ import ( "github.com/jarcoal/httpmock" ) -func TestDownladFile(t *testing.T) { +func TestDownloadFile(t *testing.T) { examplePath := "./example.txt" defer os.Remove(examplePath) diff --git a/test/ansible/provision_dependencies.yml b/test/ansible/provision_dependencies.yml index 891bcc16e..9c85e29ed 100644 --- a/test/ansible/provision_dependencies.yml +++ b/test/ansible/provision_dependencies.yml @@ -10,9 +10,12 @@ - crowdsecurity.testing.go - crowdsecurity.testing.machine_id - crowdsecurity.testing.epel + - crowdsecurity.testing.powertools - crowdsecurity.testing.git - crowdsecurity.testing.gcc - crowdsecurity.testing.gnu_make + - crowdsecurity.testing.pkg_config + - crowdsecurity.testing.re2 - crowdsecurity.testing.bats_requirements - name: "Install Postgres" diff --git a/test/ansible/requirements.yml b/test/ansible/requirements.yml index b1a28b70a..adf7c27f8 100644 --- a/test/ansible/requirements.yml +++ b/test/ansible/requirements.yml @@ -9,13 +9,13 @@ roles: # these should be included as dependencies of crowdsecurity.testing, but sometime are not - src: geerlingguy.repo-epel - src: gantsign.golang + - src: robertdebock.powertools collections: - name: https://github.com/crowdsecurity/ansible-collection-crowdsecurity.testing.git type: git - version: main + version: v0.0.3 # - name: crowdsecurity.testing # source: ../../../crowdsecurity.testing # type: dir - diff --git a/test/ansible/roles/make_fixture/tasks/main.yml b/test/ansible/roles/make_fixture/tasks/main.yml index 39f3e1785..bb255a04c 100644 --- a/test/ansible/roles/make_fixture/tasks/main.yml +++ b/test/ansible/roles/make_fixture/tasks/main.yml @@ -1,23 +1,13 @@ # vim: set ft=yaml.ansible: --- -- name: "Set make_cmd = make (!bsd)" - ansible.builtin.set_fact: - make_cmd: make - when: - - ansible_facts.system not in ['FreeBSD', 'OpenBSD'] - -- name: "Set make_cmd = gmake (bsd)" - ansible.builtin.set_fact: - make_cmd: gmake - when: - - ansible_facts.system in ['FreeBSD', 'OpenBSD'] - name: "Build crowdsec from sources, prepare test environment and fixture" become: false block: - name: "Make bats-build bats-fixture" ansible.builtin.command: - cmd: "{{ make_cmd }} bats-build bats-fixture" + # static build and we don't have to mess with LD_LIBRARY_PATH + cmd: "{{ make_cmd }} bats-build bats-fixture BUILD_STATIC=1" chdir: "{{ ansible_env.HOME }}/crowdsec" creates: "{{ ansible_env.HOME }}/crowdsec/test/local-init/init-config-data.tar" environment: diff --git a/test/ansible/vagrant/alma-8/Vagrantfile b/test/ansible/vagrant/alma-8/Vagrantfile index 4b42adb3a..a1837affb 100644 --- a/test/ansible/vagrant/alma-8/Vagrantfile +++ b/test/ansible/vagrant/alma-8/Vagrantfile @@ -2,6 +2,7 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/alma8' + config.vm.provision 'shell', path: 'bootstrap' end common = '../common' diff --git a/test/ansible/vagrant/alma-8/bootstrap b/test/ansible/vagrant/alma-8/bootstrap new file mode 100755 index 000000000..f714b2c7c --- /dev/null +++ b/test/ansible/vagrant/alma-8/bootstrap @@ -0,0 +1,5 @@ +#!/bin/sh +unset IFS +set -euf + +sudo dnf -y install kitty-terminfo diff --git a/test/ansible/vagrant/alma-9/Vagrantfile b/test/ansible/vagrant/alma-9/Vagrantfile index 0ac3e5fe3..d15917602 100644 --- a/test/ansible/vagrant/alma-9/Vagrantfile +++ b/test/ansible/vagrant/alma-9/Vagrantfile @@ -2,6 +2,7 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/alma9' + config.vm.provision 'shell', path: 'bootstrap' end common = '../common' diff --git a/test/ansible/vagrant/alma-9/bootstrap b/test/ansible/vagrant/alma-9/bootstrap new file mode 100755 index 000000000..f714b2c7c --- /dev/null +++ b/test/ansible/vagrant/alma-9/bootstrap @@ -0,0 +1,5 @@ +#!/bin/sh +unset IFS +set -euf + +sudo dnf -y install kitty-terminfo diff --git a/test/ansible/vagrant/centos-8/Vagrantfile b/test/ansible/vagrant/centos-8/Vagrantfile index 24c37ada9..5ef3ec721 100644 --- a/test/ansible/vagrant/centos-8/Vagrantfile +++ b/test/ansible/vagrant/centos-8/Vagrantfile @@ -1,7 +1,8 @@ # frozen_string_literal: true Vagrant.configure('2') do |config| - config.vm.box = 'centos/stream8' + config.vm.box = 'generic/centos8s' + config.vm.provision 'shell', path: 'bootstrap' end common = '../common' diff --git a/test/ansible/vagrant/centos-8/bootstrap b/test/ansible/vagrant/centos-8/bootstrap new file mode 100755 index 000000000..6e1ce0409 --- /dev/null +++ b/test/ansible/vagrant/centos-8/bootstrap @@ -0,0 +1,6 @@ +#!/bin/sh +unset IFS +set -euf + +sudo dnf -y install dnf-plugins-core +dnf config-manager --set-enabled powertools diff --git a/test/ansible/vagrant/centos-9/Vagrantfile b/test/ansible/vagrant/centos-9/Vagrantfile index 412354f3d..a1fe2b5f7 100644 --- a/test/ansible/vagrant/centos-9/Vagrantfile +++ b/test/ansible/vagrant/centos-9/Vagrantfile @@ -2,6 +2,7 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/centos9s' + config.vm.provision 'shell', path: 'bootstrap' end common = '../common' diff --git a/test/ansible/vagrant/centos-9/bootstrap b/test/ansible/vagrant/centos-9/bootstrap new file mode 100755 index 000000000..ea7a5fbc6 --- /dev/null +++ b/test/ansible/vagrant/centos-9/bootstrap @@ -0,0 +1,6 @@ +#!/bin/sh +unset IFS +set -euf + +sudo dnf -y install dnf-plugins-core +dnf config-manager --set-enabled crb diff --git a/test/ansible/vagrant/fedora-37/Vagrantfile b/test/ansible/vagrant/fedora-37/Vagrantfile index cb10c2d78..73e9cd4e7 100644 --- a/test/ansible/vagrant/fedora-37/Vagrantfile +++ b/test/ansible/vagrant/fedora-37/Vagrantfile @@ -2,6 +2,7 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/fedora37' + config.vm.provision 'shell', path: 'bootstrap' end common = '../common' diff --git a/test/ansible/vagrant/fedora-37/bootstrap b/test/ansible/vagrant/fedora-37/bootstrap new file mode 100755 index 000000000..f714b2c7c --- /dev/null +++ b/test/ansible/vagrant/fedora-37/bootstrap @@ -0,0 +1,5 @@ +#!/bin/sh +unset IFS +set -euf + +sudo dnf -y install kitty-terminfo diff --git a/test/ansible/vagrant/freebsd-12/Vagrantfile b/test/ansible/vagrant/freebsd-12/Vagrantfile index 33e6b473f..65aa98aeb 100644 --- a/test/ansible/vagrant/freebsd-12/Vagrantfile +++ b/test/ansible/vagrant/freebsd-12/Vagrantfile @@ -2,6 +2,7 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/freebsd12' + config.vm.provision 'shell', path: 'bootstrap' end common = '../common' diff --git a/test/ansible/vagrant/freebsd-12/bootstrap b/test/ansible/vagrant/freebsd-12/bootstrap new file mode 100644 index 000000000..1a4adafdb --- /dev/null +++ b/test/ansible/vagrant/freebsd-12/bootstrap @@ -0,0 +1,6 @@ +#!/bin/sh + +set -eu + +pkg install -y gtar + diff --git a/test/ansible/vagrant/freebsd-13/Vagrantfile b/test/ansible/vagrant/freebsd-13/Vagrantfile index 851c04254..d162c86f8 100644 --- a/test/ansible/vagrant/freebsd-13/Vagrantfile +++ b/test/ansible/vagrant/freebsd-13/Vagrantfile @@ -2,6 +2,7 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/freebsd13' + config.vm.provision 'shell', path: 'bootstrap' end common = '../common' diff --git a/test/ansible/vagrant/freebsd-13/bootstrap b/test/ansible/vagrant/freebsd-13/bootstrap new file mode 100644 index 000000000..1a4adafdb --- /dev/null +++ b/test/ansible/vagrant/freebsd-13/bootstrap @@ -0,0 +1,6 @@ +#!/bin/sh + +set -eu + +pkg install -y gtar + diff --git a/test/ansible/vagrant/ubuntu-20.04-focal/bootstrap b/test/ansible/vagrant/ubuntu-20.04-focal/bootstrap new file mode 100644 index 000000000..57f40cc2e --- /dev/null +++ b/test/ansible/vagrant/ubuntu-20.04-focal/bootstrap @@ -0,0 +1,5 @@ +#!/bin/sh +unset IFS +set -euf + +sudo apt install -y kitty-terminfo diff --git a/test/bats.mk b/test/bats.mk index 0f96be8e1..4eb7abcbf 100644 --- a/test/bats.mk +++ b/test/bats.mk @@ -10,7 +10,7 @@ ifdef PACKAGE_TESTING INIT_BACKEND = systemd CONFIG_BACKEND = global else - # LOCAL_DIR will contain contains a local instance of crowdsec, complete with + # LOCAL_DIR contains a local instance of crowdsec, complete with # configuration and data LOCAL_DIR = $(TEST_DIR)/local BIN_DIR = $(LOCAL_DIR)/bin diff --git a/test/bats/01_cscli.bats b/test/bats/01_cscli.bats index a01d936b7..4f6d7b71d 100644 --- a/test/bats/01_cscli.bats +++ b/test/bats/01_cscli.bats @@ -101,9 +101,9 @@ teardown() { # check that LAPI configuration is loaded (human and json, not shows in raw) rune -0 cscli config show -o human - assert_line --regexp ".*- URL\s+: http://127.0.0.1:8080/" - assert_line --regexp ".*- Login\s+: githubciXXXXXXXXXXXXXXXXXXXXXXXX" - assert_line --regexp ".*- Credentials File\s+: .*/local_api_credentials.yaml" + assert_line --regexp ".*- URL +: http://127.0.0.1:8080/" + assert_line --regexp ".*- Login +: githubciXXXXXXXXXXXXXXXXXXXXXXXX" + assert_line --regexp ".*- Credentials File +: .*/local_api_credentials.yaml" rune -0 cscli config show -o json rune -0 jq -c '.API.Client.Credentials | [.url,.login]' <(output) diff --git a/test/bin/assert-crowdsec-not-running b/test/bin/assert-crowdsec-not-running index b545ebf0a..3171287d0 100755 --- a/test/bin/assert-crowdsec-not-running +++ b/test/bin/assert-crowdsec-not-running @@ -1,8 +1,15 @@ #!/usr/bin/env bash is_crowdsec_running() { - # ignore processes in containers - PIDS=$(pgrep --ns $$ -x 'crowdsec') + case $(uname) in + "Linux") + # ignore processes in containers + PIDS=$(pgrep --ns $$ -x 'crowdsec') + ;; + *) + PIDS=$(pgrep -x 'crowdsec') + ;; + esac } # The process can be slow, especially on CI and during test coverage.