Merge remote-tracking branch 'origin/datatype_prediction' into perftest

experimental/collection/collection.go

@@ -0,0 +1,66 @@
+// Copyright 2022 Juan Pablo Tosso and the OWASP Coraza contributors
+// SPDX-License-Identifier: Apache-2.0
+
+package collection
+
+import (
+	"regexp"
+
+	"github.com/corazawaf/coraza/v3/experimental/types"
+)
+
+// Collection are used to store VARIABLE data
+// for transactions, this data structured is designed
+// to store slices of data for keys
+// Important: CollectionMaps ARE NOT concurrent safe
+type Collection interface {
+	// FindAll returns matches for all the items in this Collection.
+	FindAll() []types.MatchData
+
+	// Name returns the name for the current CollectionMap
+	Name() string
+}
+
+// Single is a Collection with a single element.
+type Single interface {
+	Collection
+
+	// Get returns the value of this Single
+	Get() string
+}
+
+// Keyed is a Collection with elements that can be selected by key.
+type Keyed interface {
+	Collection
+
+	// Get returns a slice of strings for a key
+	Get(key string) []string
+
+	// FindRegex returns a slice of MatchData for the regex
+	FindRegex(key *regexp.Regexp) []types.MatchData
+
+	// FindString returns a slice of MatchData for the string
+	FindString(key string) []types.MatchData
+}
+
+// Map are used to store VARIABLE data
+// for transactions, this data structured is designed
+// to store slices of data for keys
+// Important: CollectionMaps ARE NOT concurrent safe
+type Map interface {
+	Keyed
+
+	// Add a value to some key
+	Add(key string, value string)
+
+	// Set will replace the key's value with this slice
+	Set(key string, values []string)
+
+	// SetIndex will place the value under the index
+	// If the index is higher than the current size of the CollectionMap
+	// it will be appended
+	SetIndex(key string, index int, value string)
+
+	// Remove deletes the key from the CollectionMap
+	Remove(key string)
+}

experimental/plugins/macro/macro.go

@@ -8,7 +8,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/corazawaf/coraza/v3/collection"
+	"github.com/corazawaf/coraza/v3/experimental/collection"
 	"github.com/corazawaf/coraza/v3/experimental/plugins/plugintypes"
 	"github.com/corazawaf/coraza/v3/types/variables"
 )

experimental/plugins/plugintypes/rule.go

@@ -3,7 +3,9 @@
 
 package plugintypes
 
-import "github.com/corazawaf/coraza/v3/types"
+import (
+	"github.com/corazawaf/coraza/v3/types"
+)
 
 // Rule is a rule executed against a transaction.
 type Rule interface {

experimental/plugins/plugintypes/transaction.go

@@ -4,8 +4,8 @@
 package plugintypes
 
 import (
-	"github.com/corazawaf/coraza/v3/collection"
 	"github.com/corazawaf/coraza/v3/debuglog"
+	"github.com/corazawaf/coraza/v3/experimental/collection"
 	"github.com/corazawaf/coraza/v3/types"
 	"github.com/corazawaf/coraza/v3/types/variables"
 )

experimental/types/rule_match.go

@@ -0,0 +1,27 @@
+// Copyright 2022 Juan Pablo Tosso and the OWASP Coraza contributors
+// SPDX-License-Identifier: Apache-2.0
+
+package types
+
+import (
+	"github.com/corazawaf/coraza/v3/types/variables"
+)
+
+// MatchData works like VariableKey but is used for logging,
+// so it contains the collection as a string, and it's value
+type MatchData interface {
+	// Variable
+	Variable() variables.RuleVariable
+	// Key of the variable, blank if no key is required
+	Key() string
+	// Value of the current VARIABLE:KEY
+	Value() string
+	// Message is the expanded macro message
+	Message() string
+	// Data is the expanded logdata of the macro
+	Data() string
+	// Chain depth of variable match
+	ChainLevel() int
+	// Metadata of the matched data
+	Metadata() DataMetadataList
+}

experimental/types/value_metadata.go

@@ -0,0 +1,140 @@
+// Copyright 2023 Juan Pablo Tosso and the OWASP Coraza contributors
+// SPDX-License-Identifier: Apache-2.0
+package types
+
+import (
+	"unicode"
+)
+
+// DataMetadata is the type of metadata that a value can have.
+type DataMetadata int
+
+const (
+	// ValueMetadataAlphanumeric represents an alphanumeric value.
+	ValueMetadataAlphanumeric DataMetadata = iota
+	// ValueMetadataAscii represents an ASCII value.
+	ValueMetadataAscii
+	// ValueMetadataBase64 represents a base64 value.
+	ValueMetadataBase64
+	// ValueMetadataURI represents a URI value.
+	ValueMetadataURI
+	// ValueMetadataDomain represents a domain value.
+	ValueMetadataDomain
+	// ValueMetadataNumeric represents a numeric value, either integer or float.
+	ValueMetadataNumeric
+	// ValueMetadataBoolean represents a boolean value.
+	ValueMetadataBoolean
+	// ValueMetadataUnicode represents a unicode value.
+	ValueMetadataUnicode
+)
+
+// NewValueMetadata returns a new ValueMetadata from a string.
+func NewValueMetadata(metadata string) (DataMetadata, bool) {
+	switch metadata {
+	case "numeric":
+		return ValueMetadataNumeric, true
+	case "boolean":
+		return ValueMetadataBoolean, true
+	case "alphanumeric":
+		return ValueMetadataAlphanumeric, true
+	case "ascii":
+		return ValueMetadataAscii, true
+	case "base64":
+		return ValueMetadataBase64, true
+	case "uri":
+		return ValueMetadataURI, true
+	case "domain":
+		return ValueMetadataDomain, true
+	case "unicode":
+		return ValueMetadataUnicode, true
+	}
+	return 0, false
+}
+
+// DataMetadataList is a list of ValueMetadata.
+type DataMetadataList struct {
+	metadata map[DataMetadata]bool
+}
+
+func (v *DataMetadataList) Evaluate(data string) {
+	// we do the analysis only once
+	if v.metadata == nil {
+		v.metadata = make(map[DataMetadata]bool)
+		v.evaluateNumeric(data)
+		v.evaluateBoolean(data)
+		v.evaluateAlphanumeric(data)
+		v.evaluateAscii(data)
+		v.evaluateBase64(data)
+		// v.evaluateURI(data)
+		// v.evaluateDomain(data)
+		// v.evaluateUnicode(data)
+	}
+}
+
+func (v *DataMetadataList) evaluateAlphanumeric(data string) bool {
+	for _, c := range data {
+		if !unicode.IsLetter(c) && !unicode.IsNumber(c) {
+			v.metadata[ValueMetadataAlphanumeric] = false
+			break
+		}
+	}
+	return v.metadata[ValueMetadataAlphanumeric]
+}
+
+func (v *DataMetadataList) evaluateAscii(data string) bool {
+	res := true
+	for i := 0; i < len(data); i++ {
+		if data[i] > unicode.MaxASCII {
+			res = false
+			break
+		}
+	}
+	v.metadata[ValueMetadataAscii] = res
+	return res
+}
+
+func isBase64(c byte) bool {
+	return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '+' || c == '/'
+}
+
+func (v *DataMetadataList) evaluateBase64(data string) bool {
+	res := true
+	for i := 0; i < len(data); i++ {
+		if !isBase64(data[i]) {
+			res = false
+			break
+		}
+	}
+	v.metadata[ValueMetadataBase64] = res
+	return res
+}
+
+func (v *DataMetadataList) evaluateNumeric(data string) bool {
+	res := true
+	for _, c := range data {
+		if !unicode.IsNumber(c) {
+			res = false
+			break
+		}
+	}
+	v.metadata[ValueMetadataNumeric] = res
+	return res
+}
+
+func (v *DataMetadataList) evaluateBoolean(data string) bool {
+	res := false
+	if data == "true" || data == "false" {
+		res = true
+	}
+	v.metadata[ValueMetadataBoolean] = res
+	return res
+}
+
+func (v *DataMetadataList) IsInScope(metadataTypes []DataMetadata) bool {
+	for _, metadataType := range metadataTypes {
+		if v.metadata[metadataType] {
+			return true
+		}
+	}
+	return false
+}

internal/actions/setvar.go

@@ -8,7 +8,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/corazawaf/coraza/v3/collection"
+	"github.com/corazawaf/coraza/v3/experimental/collection"
 	"github.com/corazawaf/coraza/v3/experimental/plugins/macro"
 	"github.com/corazawaf/coraza/v3/experimental/plugins/plugintypes"
 	"github.com/corazawaf/coraza/v3/types/variables"

internal/actions/setvar_test.go

@@ -8,8 +8,8 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/corazawaf/coraza/v3/collection"
 	"github.com/corazawaf/coraza/v3/debuglog"
+	"github.com/corazawaf/coraza/v3/experimental/collection"
 	"github.com/corazawaf/coraza/v3/experimental/plugins/plugintypes"
 	"github.com/corazawaf/coraza/v3/internal/corazawaf"
 )

internal/actions/tag.go

@@ -4,6 +4,9 @@
 package actions
 
 import (
+	"fmt"
+	"strings"
+
 	"github.com/corazawaf/coraza/v3/experimental/plugins/plugintypes"
 	"github.com/corazawaf/coraza/v3/internal/corazawaf"
 )
@@ -30,6 +33,16 @@ func (a *tagFn) Init(r plugintypes.RuleMetadata, data string) error {
 		return ErrMissingArguments
 	}
 	r.(*corazawaf.Rule).Tags_ = append(r.(*corazawaf.Rule).Tags_, data)
+	if strings.HasPrefix(data, "metadatafilter/") {
+		filters_string := strings.Split(data, "/")
+		filters := strings.Split(filters_string[1], ",")
+		for _, filter := range filters {
+			ok := r.(*corazawaf.Rule).AddAllowedMetadata(filter)
+			if ok != nil {
+				return fmt.Errorf("invalid metadata filter: %s", filter)
+			}
+		}
+	}
 	return nil
 }
 

internal/collections/concat.go

@@ -7,9 +7,9 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/corazawaf/coraza/v3/collection"
+	"github.com/corazawaf/coraza/v3/experimental/collection"
+	"github.com/corazawaf/coraza/v3/experimental/types"
 	"github.com/corazawaf/coraza/v3/internal/corazarules"
-	"github.com/corazawaf/coraza/v3/types"
 	"github.com/corazawaf/coraza/v3/types/variables"
 )
 

internal/collections/concat_test.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/corazawaf/coraza/v3/types"
+	"github.com/corazawaf/coraza/v3/experimental/types"
 	"github.com/corazawaf/coraza/v3/types/variables"
 )
 

internal/collections/map.go

@@ -7,9 +7,9 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/corazawaf/coraza/v3/collection"
+	"github.com/corazawaf/coraza/v3/experimental/collection"
+	"github.com/corazawaf/coraza/v3/experimental/types"
 	"github.com/corazawaf/coraza/v3/internal/corazarules"
-	"github.com/corazawaf/coraza/v3/types"
 	"github.com/corazawaf/coraza/v3/types/variables"
 )
 

internal/collections/named.go

@@ -8,9 +8,9 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/corazawaf/coraza/v3/collection"
+	"github.com/corazawaf/coraza/v3/experimental/collection"
+	"github.com/corazawaf/coraza/v3/experimental/types"
 	"github.com/corazawaf/coraza/v3/internal/corazarules"
-	"github.com/corazawaf/coraza/v3/types"
 	"github.com/corazawaf/coraza/v3/types/variables"
 )
 

internal/collections/noop.go

@@ -4,8 +4,8 @@
 package collections
 
 import (
-	"github.com/corazawaf/coraza/v3/collection"
-	"github.com/corazawaf/coraza/v3/types"
+	"github.com/corazawaf/coraza/v3/experimental/collection"
+	"github.com/corazawaf/coraza/v3/experimental/types"
 )
 
 var Noop collection.Collection = &noop{}

internal/collections/single.go

@@ -7,9 +7,9 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/corazawaf/coraza/v3/collection"
+	"github.com/corazawaf/coraza/v3/experimental/collection"
+	"github.com/corazawaf/coraza/v3/experimental/types"
 	"github.com/corazawaf/coraza/v3/internal/corazarules"
-	"github.com/corazawaf/coraza/v3/types"
 	"github.com/corazawaf/coraza/v3/types/variables"
 )
 

internal/collections/sized.go

@@ -9,9 +9,9 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/corazawaf/coraza/v3/collection"
+	"github.com/corazawaf/coraza/v3/experimental/collection"
+	"github.com/corazawaf/coraza/v3/experimental/types"
 	"github.com/corazawaf/coraza/v3/internal/corazarules"
-	"github.com/corazawaf/coraza/v3/types"
 	"github.com/corazawaf/coraza/v3/types/variables"
 )