From 996c7bbcf67cb8ba527191ef1eadde042572f5a7 Mon Sep 17 00:00:00 2001
From: Nancy Enos <nicymimz@gmail.com>
Date: Sat, 26 Oct 2024 23:36:17 +0300
Subject: [PATCH] createst: Allow to exclude certain fields

Ticket: #4062
---
 README.md   |  2 ++
 createst.py | 22 ++++++++++++++++++++--
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index f11ac3fa6..804f691e5 100644
--- a/README.md
+++ b/README.md
@@ -237,6 +237,8 @@ options:
                         Adds a suricata.yaml to the test
   --features <features>
                         Adds specified features
+  --exclude-fields [EXCLUDE_FIELDS]
+                        Exclude specified fields from filter block
 ```
 
 ### Examples
diff --git a/createst.py b/createst.py
index aac1a9c4a..e317bcab8 100755
--- a/createst.py
+++ b/createst.py
@@ -222,18 +222,34 @@ def is_valid_suri_directory():
 
 def get_manipulated_list():
     """
-    Manipulate eve.json to load json successfully and skip the fields
+    Manipulate eve.json to load JSON successfully and skip the fields
     mentioned in `skip_fields` variable.
     """
     eve_path = os.path.join(test_dir, "output", "eve.json")
+    exclude_fields = args["exclude_fields"].strip().split(",") if args["exclude_fields"] else []
     allow_events = args["allow_events"].strip().split(",") if args["allow_events"] else []
+
+    def exclude_nested_fields(data, base_key=""):
+        """
+        Function to recursively exclude nested fields
+        """
+        if isinstance(data, dict):
+            filtered_data = {}
+            for k, v in data.items():
+                full_key = f"{base_key}.{k}" if base_key else k
+                if full_key not in skip_fields and not any(full_key == excl or full_key.startswith(f"{excl}.") for excl in exclude_fields):
+                    filtered_data[k] = exclude_nested_fields(v, full_key)
+            return filtered_data
+        return data
+
     with open(eve_path, "r") as fp:
         content = fp.read()
     content_list = content.strip().split("\n")
     jcontent_list = [json.loads(e) for e in content_list]
     all_content_list = []
     for e in jcontent_list:
-        md = {k: v for k, v in e.items() if k not in skip_fields}
+        md = exclude_nested_fields(e)
+        
         if "event_type" in md and md["event_type"] == "stats":
             continue
         all_content_list.append(md)
@@ -393,6 +409,8 @@ def parse_args():
                         help="Adds a suricata.yaml to the test")
     parser.add_argument("--features", default=None, metavar="<features>",
                         help="Adds specified features")
+    parser.add_argument("--exclude-fields", nargs="?", default=None,
+                        help="Exclude specified fields from filter block")                   
     # add arg to allow stdout only
     args = parser.parse_args()