package appsec import ( "bytes" "encoding/json" "fmt" "io" "net" "net/http" "net/url" "os" "regexp" "github.com/google/uuid" "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" ) const ( URIHeaderName = "X-Crowdsec-Appsec-Uri" VerbHeaderName = "X-Crowdsec-Appsec-Verb" HostHeaderName = "X-Crowdsec-Appsec-Host" IPHeaderName = "X-Crowdsec-Appsec-Ip" APIKeyHeaderName = "X-Crowdsec-Appsec-Api-Key" UserAgentHeaderName = "X-Crowdsec-Appsec-User-Agent" ) type ParsedRequest struct { RemoteAddr string `json:"remote_addr,omitempty"` Host string `json:"host,omitempty"` ClientIP string `json:"client_ip,omitempty"` URI string `json:"uri,omitempty"` Args url.Values `json:"args,omitempty"` ClientHost string `json:"client_host,omitempty"` Headers http.Header `json:"headers,omitempty"` URL *url.URL `json:"url,omitempty"` Method string `json:"method,omitempty"` Proto string `json:"proto,omitempty"` Body []byte `json:"body,omitempty"` TransferEncoding []string `json:"transfer_encoding,omitempty"` UUID string `json:"uuid,omitempty"` Tx ExtendedTransaction `json:"-"` ResponseChannel chan AppsecTempResponse `json:"-"` IsInBand bool `json:"-"` IsOutBand bool `json:"-"` AppsecEngine string `json:"appsec_engine,omitempty"` RemoteAddrNormalized string `json:"normalized_remote_addr,omitempty"` HTTPRequest *http.Request `json:"-"` } type ReqDumpFilter struct { req *ParsedRequest HeadersContentFilters []string HeadersNameFilters []string HeadersDrop bool BodyDrop bool //BodyContentFilters []string TBD ArgsContentFilters []string ArgsNameFilters []string ArgsDrop bool } func (r *ParsedRequest) DumpRequest(params ...any) *ReqDumpFilter { filter := ReqDumpFilter{} filter.BodyDrop = true filter.HeadersNameFilters = []string{"cookie", "authorization"} filter.req = r return &filter } // clear filters func (r *ReqDumpFilter) NoFilters() *ReqDumpFilter { r2 := ReqDumpFilter{} r2.req = r.req return &r2 } func (r *ReqDumpFilter) WithEmptyHeadersFilters() *ReqDumpFilter { r.HeadersContentFilters = []string{} return r } func (r *ReqDumpFilter) WithHeadersContentFilter(filter string) *ReqDumpFilter { r.HeadersContentFilters = append(r.HeadersContentFilters, filter) return r } func (r *ReqDumpFilter) WithHeadersNameFilter(filter string) *ReqDumpFilter { r.HeadersNameFilters = append(r.HeadersNameFilters, filter) return r } func (r *ReqDumpFilter) WithNoHeaders() *ReqDumpFilter { r.HeadersDrop = true return r } func (r *ReqDumpFilter) WithHeaders() *ReqDumpFilter { r.HeadersDrop = false r.HeadersNameFilters = []string{} return r } func (r *ReqDumpFilter) WithBody() *ReqDumpFilter { r.BodyDrop = false return r } func (r *ReqDumpFilter) WithNoBody() *ReqDumpFilter { r.BodyDrop = true return r } func (r *ReqDumpFilter) WithEmptyArgsFilters() *ReqDumpFilter { r.ArgsContentFilters = []string{} return r } func (r *ReqDumpFilter) WithArgsContentFilter(filter string) *ReqDumpFilter { r.ArgsContentFilters = append(r.ArgsContentFilters, filter) return r } func (r *ReqDumpFilter) WithArgsNameFilter(filter string) *ReqDumpFilter { r.ArgsNameFilters = append(r.ArgsNameFilters, filter) return r } func (r *ReqDumpFilter) FilterBody(out *ParsedRequest) error { if r.BodyDrop { return nil } out.Body = r.req.Body return nil } func (r *ReqDumpFilter) FilterArgs(out *ParsedRequest) error { if r.ArgsDrop { return nil } if len(r.ArgsContentFilters) == 0 && len(r.ArgsNameFilters) == 0 { out.Args = r.req.Args return nil } out.Args = make(url.Values) for k, vals := range r.req.Args { reject := false //exclude by match on name for _, filter := range r.ArgsNameFilters { ok, err := regexp.MatchString("(?i)"+filter, k) if err != nil { log.Debugf("error while matching string '%s' with '%s': %s", filter, k, err) continue } if ok { reject = true break } } for _, v := range vals { //exclude by content for _, filter := range r.ArgsContentFilters { ok, err := regexp.MatchString("(?i)"+filter, v) if err != nil { log.Debugf("error while matching string '%s' with '%s': %s", filter, v, err) continue } if ok { reject = true break } } } //if it was not rejected, let's add it if !reject { out.Args[k] = vals } } return nil } func (r *ReqDumpFilter) FilterHeaders(out *ParsedRequest) error { if r.HeadersDrop { return nil } if len(r.HeadersContentFilters) == 0 && len(r.HeadersNameFilters) == 0 { out.Headers = r.req.Headers return nil } out.Headers = make(http.Header) for k, vals := range r.req.Headers { reject := false //exclude by match on name for _, filter := range r.HeadersNameFilters { ok, err := regexp.MatchString("(?i)"+filter, k) if err != nil { log.Debugf("error while matching string '%s' with '%s': %s", filter, k, err) continue } if ok { reject = true break } } for _, v := range vals { //exclude by content for _, filter := range r.HeadersContentFilters { ok, err := regexp.MatchString("(?i)"+filter, v) if err != nil { log.Debugf("error while matching string '%s' with '%s': %s", filter, v, err) continue } if ok { reject = true break } } } //if it was not rejected, let's add it if !reject { out.Headers[k] = vals } } return nil } func (r *ReqDumpFilter) GetFilteredRequest() *ParsedRequest { //if there are no filters, we return the original request if len(r.HeadersContentFilters) == 0 && len(r.HeadersNameFilters) == 0 && len(r.ArgsContentFilters) == 0 && len(r.ArgsNameFilters) == 0 && !r.BodyDrop && !r.HeadersDrop && !r.ArgsDrop { log.Warningf("no filters, returning original request") return r.req } r2 := ParsedRequest{} r.FilterHeaders(&r2) r.FilterBody(&r2) r.FilterArgs(&r2) return &r2 } func (r *ReqDumpFilter) ToJSON() error { fd, err := os.CreateTemp("", "crowdsec_req_dump_*.json") if err != nil { return fmt.Errorf("while creating temp file: %w", err) } defer fd.Close() enc := json.NewEncoder(fd) enc.SetIndent("", " ") req := r.GetFilteredRequest() log.Tracef("dumping : %+v", req) if err := enc.Encode(req); err != nil { //Don't clobber the temp directory with empty files err2 := os.Remove(fd.Name()) if err2 != nil { log.Errorf("while removing temp file %s: %s", fd.Name(), err) } return fmt.Errorf("while encoding request: %w", err) } log.Infof("request dumped to %s", fd.Name()) return nil } // Generate a ParsedRequest from a http.Request. ParsedRequest can be consumed by the App security Engine func NewParsedRequestFromRequest(r *http.Request, logger *logrus.Entry) (ParsedRequest, error) { var err error contentLength := r.ContentLength if contentLength < 0 { contentLength = 0 } body := make([]byte, contentLength) if r.Body != nil { _, err = io.ReadFull(r.Body, body) if err != nil { return ParsedRequest{}, fmt.Errorf("unable to read body: %s", err) } // reset the original body back as it's been read, i'm not sure its needed? r.Body = io.NopCloser(bytes.NewBuffer(body)) } clientIP := r.Header.Get(IPHeaderName) if clientIP == "" { return ParsedRequest{}, fmt.Errorf("missing '%s' header", IPHeaderName) } clientURI := r.Header.Get(URIHeaderName) if clientURI == "" { return ParsedRequest{}, fmt.Errorf("missing '%s' header", URIHeaderName) } clientMethod := r.Header.Get(VerbHeaderName) if clientMethod == "" { return ParsedRequest{}, fmt.Errorf("missing '%s' header", VerbHeaderName) } clientHost := r.Header.Get(HostHeaderName) if clientHost == "" { //this might be empty logger.Debugf("missing '%s' header", HostHeaderName) } userAgent := r.Header.Get(UserAgentHeaderName) //This one is optional // delete those headers before coraza process the request delete(r.Header, IPHeaderName) delete(r.Header, HostHeaderName) delete(r.Header, URIHeaderName) delete(r.Header, VerbHeaderName) delete(r.Header, UserAgentHeaderName) delete(r.Header, APIKeyHeaderName) originalHTTPRequest := r.Clone(r.Context()) originalHTTPRequest.Body = io.NopCloser(bytes.NewBuffer(body)) originalHTTPRequest.RemoteAddr = clientIP originalHTTPRequest.RequestURI = clientURI originalHTTPRequest.Method = clientMethod originalHTTPRequest.Host = clientHost if userAgent != "" { originalHTTPRequest.Header.Set("User-Agent", userAgent) r.Header.Set("User-Agent", userAgent) //Override the UA in the original request, as this is what will be used by the waf engine } else { //If we don't have a forwarded UA, delete the one that was set by the bouncer originalHTTPRequest.Header.Del("User-Agent") } parsedURL, err := url.Parse(clientURI) if err != nil { return ParsedRequest{}, fmt.Errorf("unable to parse url '%s': %s", clientURI, err) } var remoteAddrNormalized string if r.RemoteAddr == "@" { r.RemoteAddr = "127.0.0.1:65535" } // TODO we need to implement forwrded headers host, _, err := net.SplitHostPort(r.RemoteAddr) if err != nil { log.Errorf("Invalid appsec remote IP source %v: %s", r.RemoteAddr, err.Error()) remoteAddrNormalized = r.RemoteAddr } else { ip := net.ParseIP(host) if ip == nil { log.Errorf("Invalid appsec remote IP address source %v", r.RemoteAddr) remoteAddrNormalized = r.RemoteAddr } else { remoteAddrNormalized = ip.String() } } return ParsedRequest{ RemoteAddr: r.RemoteAddr, UUID: uuid.New().String(), ClientHost: clientHost, ClientIP: clientIP, URI: clientURI, Method: clientMethod, Host: clientHost, Headers: r.Header, URL: parsedURL, Proto: r.Proto, Body: body, Args: ParseQuery(parsedURL.RawQuery), TransferEncoding: r.TransferEncoding, ResponseChannel: make(chan AppsecTempResponse), RemoteAddrNormalized: remoteAddrNormalized, HTTPRequest: originalHTTPRequest, }, nil }