[panw] Parse URL from domain_edl category threat logs (#11837)

In PAN-OS threat logs, when threat_category is 'domain_edl', the misc field will contain a URL. This change adds parsing of the URL for this case.
elastic · Dec 7, 2024 · 42bd8ea · 42bd8ea
1 parent 73f0cf3
commit 42bd8ea
Show file tree

Hide file tree

Showing 5 changed files with 194 additions and 1 deletion.
diff --git a/packages/panw/changelog.yml b/packages/panw/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "4.2.0"
+  changes:
+    - description: Parse URL in threat logs with 'domain-edl' threat category
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/11837
 - version: "4.1.1"
   changes:
     - description: Map name for more subtypes, fix CSV escaping

diff --git a/packages/panw/data_stream/panos/_dev/test/pipeline/test-panw-panos-threat-sample.log b/packages/panw/data_stream/panos/_dev/test/pipeline/test-panw-panos-threat-sample.log
@@ -215,3 +215,4 @@ Apr 9 20:43:30 AC-PA5250 1,2024/04/09 20:43:29,123456789012,THREAT,virus,2561,20
 <14>Aug 19 13:58:31 fw0096.example.io 1,2024/08/19 13:58:31,019901001188,THREAT,scan,2562,2024/08/19 13:58:31,10.48.12.171,10.190.160.25,0.0.0.0,0.0.0.0,,,,not-applicable,vsys2,interconnect,public,ae2.1349,,HOST-LOGCOLLECTOR,2024/08/19 13:58:31,0,1,41526,443,0,0,0x2000,tcp,alert,,SCAN: Host Sweep(8002),any,medium,client-to-server,7361590532514024944,0x8000000000000000,10.0.0.0-10.255.255.255,European Union,,,0,,,0,,,,,,,,0,15,23,0,0,az1_vsys_internet,fw0096,,,,,0,,0,,N/A,scan,AppThreat-0-0,0x0,0,4294967295,,,,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2024-08-19T13:58:31.761+00:00,,,0,unknown,unknown,unknown,1,,,not-applicable,no,no,
 <14>Aug 19 13:58:32 fw1034.example.io 1,2024/08/19 13:58:32,007951000353454,THREAT,vulnerability,2562,2024/08/19 13:58:32,10.71.208.15,10.68.15.198,0.0.0.0,0.0.0.0,SectorProxy Browsing my3-user,,,web-browsing,vsys1,interconnect,proxy,ethernet1/2,ethernet1/3,HOST-LOGCOLLECTOR,2024/08/19 13:58:32,577801,1,18830,8097,0,0,0x1102000,tcp,alert,"shadow",Potential HTML Evasion Technique Detected in HTTP Response(91883),ua-generic,low,server-to-client,7395705320518981763,0x8000000000000000,10.0.0.0-10.255.255.255,10.0.0.0-10.255.255.255,,,0,,,1,,,,,,,,0,850,852,0,0,,fw1034,,,,,0,,0,2024/08/19 13:58:26,N/A,protocol-anomaly,AppThreat-8883-8920,0x0,0,4294967295,,,c1b9f945-e213-4cdf-b77d-2700446a3baf,805901,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2024-08-19T13:58:32.880+00:00,,,,internet-utility,general-internet,browser-based,4,"used-by-malware,able-to-transfer-file,has-known-vulnerability,tunnel-other-application,pervasive-use",,web-browsing,no,no,
 <14>Nov 06 14:11:30 pa555 1,2024/11/06 14:11:30,0000000000001,THREAT,file,2562,2024/11/06 14:11:30,192.168.1.2,10.71.208.15,0.0.0.0,0.0.0.0,file download test rule,contoso\\steve,,web-browsing,vsys1,HOMENET,EXTNET,ethernet1/2,ethernet1/1,log-profile1,2024/11/06 14:11:30,994313,2,37268,443,0,0,0x1002000,tcp,alert,"elastic-agent.exe",Windows Executable (EXE)(52020),computer-and-internet-info,low,server-to-client,7367538158076100804,0x8000000000000000,192.168.0.0-192.168.255.255,United States,,,0,,,1,,,,,,,,0,199,479,0,0,,pa555,artifacts.elastic.co/downloads/beats/elastic-agent/elastic-agent-8.15.3+build202411051926-windows-x86_64.zip,,,,7213055707168598,,0,2024/11/06 14:11:30,N/A,N/A,AppThreat-8911-9049,0x0,0,4294967295,,,88e69ca4-8783-4b7c-9982-f73ec6f1a83c,1679420,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2024-11-06T14:11:30.036-05:00,,,,internet-utility,generate-internet,browser-based,2,"used-bymalware,able-to-transfer-file,has-known-vulnerability,tunnel-other-application,pervasive-use",,web-browsing,no,no,
+<14>Nov 22 11:44:08 pa555 1,2024/11/22 11:44:07,013101001308,THREAT,spyware,2561,2024/11/22 11:44:07,67.43.156.0,67.43.156.1,0.0.0.0,0.0.0.0,A_DST_L7D_DNS,domain\user01,,dns-base,vsys1,Group,Servers,abc.123,abd.234,Panorama-Elastic,2024/10/01 10:43:54,34891187,2,59020,53,0,0,0x3000,tcp,sinkhole,"*.domain.dev",Suspicious Domain(12000000),any,medium,client-to-server,7401113521124350246,0x8000000000000000,10.0.0.0-10.255.255.255,10.0.0.0-10.255.255.255,,,0,,,0,,,,,,,,0,0,0,0,0,Core,AC5250,,,,,0,,0,,N/A,domain-edl,AppThreat-0-0,0x0,0,4291167295,,,5e791170-7507-4ab1-a951-79ebed0dad21,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2024-10-01T10:43:55.308+02:00,,,,infrastructure,networking,network-protocol,3,"used-by-malware,has-known-vulnerability,pervasive-use",dns,dns-base,no,no,_reportid
diff --git a/...panw/data_stream/panos/_dev/test/pipeline/test-panw-panos-threat-sample.log-expected.json b/...panw/data_stream/panos/_dev/test/pipeline/test-panw-panos-threat-sample.log-expected.json
@@ -37527,6 +37527,188 @@
                 "domain": "contoso",
                 "name": "steve"
             }
+        },
+        {
+            "@timestamp": "2024-10-01T18:13:55.308+09:30",
+            "destination": {
+                "as": {
+                    "number": 35908
+                },
+                "domain": "*.domain.dev",
+                "geo": {
+                    "continent_name": "Asia",
+                    "country_iso_code": "BT",
+                    "country_name": "Bhutan",
+                    "location": {
+                        "lat": 27.5,
+                        "lon": 90.5
+                    },
+                    "name": "10.0.0.0-10.255.255.255"
+                },
+                "ip": "67.43.156.1",
+                "port": 53
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "action": "spyware_detected",
+                "category": [
+                    "intrusion_detection",
+                    "threat",
+                    "network"
+                ],
+                "kind": "alert",
+                "original": "<14>Nov 22 11:44:08 pa555 1,2024/11/22 11:44:07,013101001308,THREAT,spyware,2561,2024/11/22 11:44:07,67.43.156.0,67.43.156.1,0.0.0.0,0.0.0.0,A_DST_L7D_DNS,domain\\user01,,dns-base,vsys1,Group,Servers,abc.123,abd.234,Panorama-Elastic,2024/10/01 10:43:54,34891187,2,59020,53,0,0,0x3000,tcp,sinkhole,\"*.domain.dev\",Suspicious Domain(12000000),any,medium,client-to-server,7401113521124350246,0x8000000000000000,10.0.0.0-10.255.255.255,10.0.0.0-10.255.255.255,,,0,,,0,,,,,,,,0,0,0,0,0,Core,AC5250,,,,,0,,0,,N/A,domain-edl,AppThreat-0-0,0x0,0,4291167295,,,5e791170-7507-4ab1-a951-79ebed0dad21,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2024-10-01T10:43:55.308+02:00,,,,infrastructure,networking,network-protocol,3,\"used-by-malware,has-known-vulnerability,pervasive-use\",dns,dns-base,no,no,_reportid",
+                "outcome": "failure",
+                "severity": 3,
+                "timezone": "+09:30",
+                "type": [
+                    "denied"
+                ]
+            },
+            "labels": {
+                "temporary_match": true
+            },
+            "log": {
+                "level": "medium"
+            },
+            "message": "67.43.156.0,67.43.156.1,0.0.0.0,0.0.0.0,A_DST_L7D_DNS,domain\\user01,,dns-base,vsys1,Group,Servers,abc.123,abd.234,Panorama-Elastic,2024/10/01 10:43:54,34891187,2,59020,53,0,0,0x3000,tcp,sinkhole,\"*.domain.dev\",Suspicious Domain(12000000),any,medium,client-to-server,7401113521124350246,0x8000000000000000,10.0.0.0-10.255.255.255,10.0.0.0-10.255.255.255,,,0,,,0,,,,,,,,0,0,0,0,0,Core,AC5250,,,,,0,,0,,N/A,domain-edl,AppThreat-0-0,0x0,0,4291167295,,,5e791170-7507-4ab1-a951-79ebed0dad21,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2024-10-01T10:43:55.308+02:00,,,,infrastructure,networking,network-protocol,3,\"used-by-malware,has-known-vulnerability,pervasive-use\",dns,dns-base,no,no,_reportid",
+            "network": {
+                "application": "dns-base",
+                "community_id": "1:Sj3H9LtJYWFdWXjIqpG6+D2O7fs=",
+                "direction": "inbound",
+                "transport": "tcp",
+                "type": "ipv4"
+            },
+            "observer": {
+                "egress": {
+                    "interface": {
+                        "name": "abd.234"
+                    },
+                    "zone": "Servers"
+                },
+                "hostname": "AC5250",
+                "ingress": {
+                    "interface": {
+                        "name": "abc.123"
+                    },
+                    "zone": "Group"
+                },
+                "product": "PAN-OS",
+                "serial_number": "013101001308",
+                "type": "firewall",
+                "vendor": "Palo Alto Networks"
+            },
+            "panw": {
+                "panos": {
+                    "action": "sinkhole",
+                    "action_flags": "0x8000000000000000",
+                    "application": {
+                        "category": "networking",
+                        "characteristics": "used-by-malware,has-known-vulnerability,pervasive-use",
+                        "container": "dns",
+                        "is_saas": "no",
+                        "is_sanctioned": "no",
+                        "risk_level": 3,
+                        "sub_category": "infrastructure",
+                        "technology": "network-protocol",
+                        "tunneled": "dns-base"
+                    },
+                    "cloud_report": {
+                        "id": "_reportid"
+                    },
+                    "content_version": "AppThreat-0-0",
+                    "device_group_hierarchy1": "0",
+                    "device_group_hierarchy2": "0",
+                    "device_group_hierarchy3": "0",
+                    "device_group_hierarchy4": "0",
+                    "flow_id": "34891187",
+                    "generated_time": "2024-11-22T11:44:07.000+09:30",
+                    "high_resolution_timestamp": "2024-10-01T18:13:55.308+09:30",
+                    "http2_connection": "0",
+                    "imsi": "0",
+                    "log_profile": "Panorama-Elastic",
+                    "logged_time": "2024-10-01T10:43:54.000+09:30",
+                    "parent_session": {
+                        "id": "0"
+                    },
+                    "partial_hash": "0",
+                    "payload_protocol_id": "4291167295",
+                    "received_time": "2024-11-22T11:44:07.000+09:30",
+                    "repeat_count": 2,
+                    "ruleset": "A_DST_L7D_DNS",
+                    "sctp": {
+                        "assoc_id": "0"
+                    },
+                    "sequence_number": "7401113521124350246",
+                    "sub_type": "spyware",
+                    "threat": {
+                        "id": "12000000",
+                        "name": "Suspicious Domain"
+                    },
+                    "threat_category": "domain-edl",
+                    "tunnel_type": "N/A",
+                    "type": "THREAT",
+                    "url": {
+                        "category": "any"
+                    },
+                    "url_idx": "0",
+                    "virtual_sys": "vsys1",
+                    "vsys_name": "Core",
+                    "wildfire": {
+                        "report_id": "0"
+                    }
+                }
+            },
+            "related": {
+                "hosts": [
+                    "AC5250"
+                ],
+                "ip": [
+                    "67.43.156.0",
+                    "67.43.156.1"
+                ],
+                "user": [
+                    "user01"
+                ]
+            },
+            "rule": {
+                "name": "A_DST_L7D_DNS",
+                "uuid": "5e791170-7507-4ab1-a951-79ebed0dad21"
+            },
+            "source": {
+                "as": {
+                    "number": 35908
+                },
+                "geo": {
+                    "continent_name": "Asia",
+                    "country_iso_code": "BT",
+                    "country_name": "Bhutan",
+                    "location": {
+                        "lat": 27.5,
+                        "lon": 90.5
+                    },
+                    "name": "10.0.0.0-10.255.255.255"
+                },
+                "ip": "67.43.156.0",
+                "port": 59020,
+                "user": {
+                    "domain": "domain",
+                    "name": "user01"
+                }
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "url": {
+                "domain": "*.domain.dev",
+                "original": "*.domain.dev"
+            },
+            "user": {
+                "domain": "domain",
+                "name": "user01"
+            }
         }
     ]
 }
diff --git a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/threat.yml b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/threat.yml
@@ -318,6 +318,11 @@ processors:
       tag: set_url_from_file
       field: url.original
       copy_from: _temp_.future_use3
+  - set:
+      if: 'ctx.panw?.panos?.threat_category == "domain-edl" && ctx.panw?.panos?.misc instanceof String'
+      tag: set_domain_edl_url
+      field: url.original
+      copy_from: panw.panos.misc
   # Crude implementation of `uri_parts` as its not working well due to lack of scheme.
   # When the scheme of the URL is absent, this script parses the URL in `ctx.panw.panos.misc` into components namely 
   # `url.original`, `url.domain`, `url.port`, `url.path`, `url.query`, `url.extension`

diff --git a/packages/panw/manifest.yml b/packages/panw/manifest.yml
@@ -1,6 +1,6 @@
 name: panw
 title: Palo Alto Next-Gen Firewall
-version: "4.1.1"
+version: "4.2.0"
 description: Collect logs from Palo Alto next-gen firewalls with Elastic Agent.
 type: integration
 format_version: "3.0.3"