add email-message translation to ecs (#1621)

stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json

@@ -43,7 +43,7 @@
   },
   "file": {
     "fields": {
-      "name": ["file.name", "dll.name", "file.path", "process.name", "process.executable", "process.parent.name", "process.parent.executable"],
+      "name": ["file.name", "dll.name", "file.path", "process.name", "process.executable", "process.parent.name", "process.parent.executable", "email.attachments.file.name"],
       "created": ["file.created", "file.ctime"],
       "modified": ["file.mtime"],
       "accessed": ["file.accessed"],
@@ -159,7 +159,15 @@
   },
   "email-addr": {
     "fields": {
-      "value": ["user.email"]
+      "value": ["user.email", "email.from.address", "email.to.address"]
+    }
+  },
+  "email-message": {
+    "fields": {
+      "subject": ["email.subject"],
+      "from_ref.value": ["email.from.address"],
+      "to_refs[*].value": ["email.to.address"],
+      "cc_refs[*].value": ["email.to.address"]
     }
   },
   "x-oca-event": {

stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json

@@ -2616,6 +2616,61 @@
       }
     }
   },
+  "email": {
+    "subject": {
+      "key": "email-message.subject",
+      "object": "mail"
+    },
+    "from": {
+      "address": [
+        {
+          "key": "email-addr.value",
+          "object": "emailfrom"
+        },
+        {
+          "key": "email-message.from_ref",
+          "object": "mail",
+          "references": "emailfrom"
+        }
+      ]
+    },
+    "to": {
+      "address": [
+        {
+          "key": "email-addr.value",
+          "unwrap": true,
+          "object": "emailto"
+        },
+        {
+          "key": "email-message.to_refs",
+          "object": "mail",
+          "references": ["emailto"]
+        }
+      ]
+    },
+    "cc": {
+      "address": [
+        {
+          "key": "email-addr.value",
+          "unwrap": true,
+          "object": "emailcc"
+        },
+        {
+          "key": "email-message.cc_refs",
+          "object": "mail",
+          "references": ["emailcc"]
+        }
+      ]
+    },
+    "attachments": {
+      "file": {
+        "name": {
+          "key": "file.name",
+          "object": "attachment"
+        }
+      }
+    }
+  },
   "tags": {
     "key": "process.x_tags",
     "object": "process"

stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py

@@ -380,6 +380,29 @@
     }
 }
 
+email_data = {
+    "email": {
+        "attachments": [
+            { "file": { "name": "tabby.html", "mime_type": "text/html" } },
+            { "file": { "name": "tabby.zip", "mime_type": "application/zip" } }
+        ],
+        "subject": "Check out this picture of a cat!",
+        "from": { "address": "[email protected]" },
+        "to": {
+            "address": [
+                "[email protected]",
+                "[email protected]"
+            ]
+        },
+        "cc": {
+            "address": [
+                "[email protected]",
+                "[email protected]"
+            ]
+        }
+    }
+}
+
 class TestElasticEcsTransform(unittest.TestCase, object):
     @staticmethod
     def get_first(itr, constraint):
@@ -848,4 +871,26 @@ def test_observer(self):
           observer_ip and 
           observer_ip.get("type") == "ipv4-addr" and
           observer_ip.get("value") == "10.0.0.101" 
-        )
+        )
+
+    def test_email(self):
+        result_bundle = run_in_thread(entry_point.translate_results, data_source, [email_data])
+        assert (result_bundle['type'] == 'bundle')
+        translation_objects = result_bundle.get('objects')
+        assert (translation_objects and len(translation_objects) == 2)
+        observed_data = translation_objects[1]
+        stix_objects = observed_data.get("objects")
+        assert len(stix_objects) == 8
+        for k,v in stix_objects.items():
+            print(f"{k}: {v}")
+        emailmsg = stix_objects.get("2")
+        assert (
+            emailmsg and emailmsg.get("type") == "email-message" and
+            emailmsg.get("subject") == "Check out this picture of a cat!"
+        )
+        from_email = stix_objects.get(emailmsg.get("from_ref"))
+        assert from_email and from_email.get("value") == "[email protected]"
+        to_emails = {stix_objects.get(ref).get("value") for ref in emailmsg.get("to_refs")}
+        assert to_emails == {"[email protected]", "[email protected]"}
+        filenames = {stix_objects[str(ref)]["name"] for ref in range(len(stix_objects)) if stix_objects[str(ref)] and stix_objects[str(ref)]["type"] == "file"}
+        assert filenames == {"tabby.zip", "tabby.html"}

stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_stix_to_query.py

@@ -121,7 +121,7 @@ def test_file_query(self):
         stix_pattern = "[file:name = 'some_file.exe']"
         translated_query = translation.translate('elastic_ecs', 'query', '{}', stix_pattern)
         translated_query['queries'] = _remove_timestamp_from_query(translated_query['queries'])
-        test_query = ['(file.name : "some_file.exe" OR dll.name : "some_file.exe" OR file.path : "some_file.exe" OR process.name : "some_file.exe" OR process.executable : "some_file.exe" OR process.parent.name : "some_file.exe" OR process.parent.executable : "some_file.exe")']
+        test_query = ['(file.name : "some_file.exe" OR dll.name : "some_file.exe" OR file.path : "some_file.exe" OR process.name : "some_file.exe" OR process.executable : "some_file.exe" OR process.parent.name : "some_file.exe" OR process.parent.executable : "some_file.exe" OR email.attachments.file.name : "some_file.exe")']
         _test_query_assertions(translated_query, test_query)
 
     def test_complex_query(self):
@@ -137,7 +137,7 @@ def test_file_not_equal_query(self):
         stix_pattern = "[file:name != 'some_file.exe']"
         translated_query = translation.translate('elastic_ecs', 'query', '{}', stix_pattern)
         translated_query['queries'] = _remove_timestamp_from_query(translated_query['queries'])
-        test_query = ['((NOT file.name : "some_file.exe" AND file.name:*) OR (NOT dll.name : "some_file.exe" AND dll.name:*) OR (NOT file.path : "some_file.exe" AND file.path:*) OR (NOT process.name : "some_file.exe" AND process.name:*) OR (NOT process.executable : "some_file.exe" AND process.executable:*) OR (NOT process.parent.name : "some_file.exe" AND process.parent.name:*) OR (NOT process.parent.executable : "some_file.exe" AND process.parent.executable:*))']
+        test_query = ['((NOT file.name : "some_file.exe" AND file.name:*) OR (NOT dll.name : "some_file.exe" AND dll.name:*) OR (NOT file.path : "some_file.exe" AND file.path:*) OR (NOT process.name : "some_file.exe" AND process.name:*) OR (NOT process.executable : "some_file.exe" AND process.executable:*) OR (NOT process.parent.name : "some_file.exe" AND process.parent.name:*) OR (NOT process.parent.executable : "some_file.exe" AND process.parent.executable:*) OR (NOT email.attachments.file.name : "some_file.exe" AND email.attachments.file.name:*))']
         _test_query_assertions(translated_query, test_query)
 
     def test_port_queries(self):
@@ -383,3 +383,21 @@ def test_start_comparison(self):
         translated_query['queries'] = _remove_timestamp_from_query(translated_query['queries'])
         test_query = ['(event.start:[* TO "2023-12-12T00:00:00.000Z"])']
         _test_query_assertions(translated_query, test_query)
+
+    def test_email_query(self):
+        stix_pattern = "[email-addr:value = '[email protected]']"
+        translated_query = translation.translate('elastic_ecs', 'query', '{}', stix_pattern)
+        translated_query['queries'] = _remove_timestamp_from_query(translated_query['queries'])
+        translated_query["queries"] == ['(user.email : "[email protected]" OR email.from.address : "[email protected]" OR email.to.address : "[email protected]")']
+
+    def test_email_from(self):
+        stix_pattern = "[email-message:from_ref.value = '[email protected]']"
+        translated_query = translation.translate('elastic_ecs', 'query', '{}', stix_pattern)
+        translated_query['queries'] = _remove_timestamp_from_query(translated_query['queries'])
+        translated_query["queries"] == ['(email.from.address : "[email protected]")']
+
+    def test_email_to(self):
+        stix_pattern = "[email-message:to_refs[*].value = '[email protected]']"
+        translated_query = translation.translate('elastic_ecs', 'query', '{}', stix_pattern)
+        translated_query['queries'] = _remove_timestamp_from_query(translated_query['queries'])
+        translated_query["queries"] == ['(email.to.address : "[email protected]")']