From 69259f62e5b34938eb4791da91bf920383e44aed Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Fri, 22 Nov 2024 13:53:30 -0800 Subject: [PATCH 1/2] [panw] Parse URL from domain_edl category threat logs In threat logs, when threat_category is 'domain_edl', the misc field will contain a URL. This adds parsing of the URL for this case. --- packages/panw/changelog.yml | 5 + .../test-panw-panos-threat-sample.log | 1 + ...panw-panos-threat-sample.log-expected.json | 182 ++++++++++++++++++ .../elasticsearch/ingest_pipeline/threat.yml | 5 + packages/panw/manifest.yml | 2 +- 5 files changed, 194 insertions(+), 1 deletion(-) diff --git a/packages/panw/changelog.yml b/packages/panw/changelog.yml index 8b5373d76f9..f619b93871b 100644 --- a/packages/panw/changelog.yml +++ b/packages/panw/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "4.2.0" + changes: + - description: Parse URL in threat logs with 'domain-edl' threat category + type: enhancement + link: https://github.com/elastic/integrations/pull/999999 - version: "4.1.0" changes: - description: Parse URL from threat-file event type diff --git a/packages/panw/data_stream/panos/_dev/test/pipeline/test-panw-panos-threat-sample.log b/packages/panw/data_stream/panos/_dev/test/pipeline/test-panw-panos-threat-sample.log index 1566cca9201..51aaa054826 100644 --- a/packages/panw/data_stream/panos/_dev/test/pipeline/test-panw-panos-threat-sample.log +++ b/packages/panw/data_stream/panos/_dev/test/pipeline/test-panw-panos-threat-sample.log @@ -212,3 +212,4 @@ Apr 9 20:43:30 AC-PA5250 1,2024/04/09 20:43:29,123456789012,THREAT,virus,2561,20 <14>Aug 19 13:58:31 fw0096.example.io 1,2024/08/19 13:58:31,019901001188,THREAT,scan,2562,2024/08/19 13:58:31,10.48.12.171,10.190.160.25,0.0.0.0,0.0.0.0,,,,not-applicable,vsys2,interconnect,public,ae2.1349,,HOST-LOGCOLLECTOR,2024/08/19 13:58:31,0,1,41526,443,0,0,0x2000,tcp,alert,,SCAN: Host Sweep(8002),any,medium,client-to-server,7361590532514024944,0x8000000000000000,10.0.0.0-10.255.255.255,European Union,,,0,,,0,,,,,,,,0,15,23,0,0,az1_vsys_internet,fw0096,,,,,0,,0,,N/A,scan,AppThreat-0-0,0x0,0,4294967295,,,,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2024-08-19T13:58:31.761+00:00,,,0,unknown,unknown,unknown,1,,,not-applicable,no,no, <14>Aug 19 13:58:32 fw1034.example.io 1,2024/08/19 13:58:32,007951000353454,THREAT,vulnerability,2562,2024/08/19 13:58:32,10.71.208.15,10.68.15.198,0.0.0.0,0.0.0.0,SectorProxy Browsing my3-user,,,web-browsing,vsys1,interconnect,proxy,ethernet1/2,ethernet1/3,HOST-LOGCOLLECTOR,2024/08/19 13:58:32,577801,1,18830,8097,0,0,0x1102000,tcp,alert,"shadow",Potential HTML Evasion Technique Detected in HTTP Response(91883),ua-generic,low,server-to-client,7395705320518981763,0x8000000000000000,10.0.0.0-10.255.255.255,10.0.0.0-10.255.255.255,,,0,,,1,,,,,,,,0,850,852,0,0,,fw1034,,,,,0,,0,2024/08/19 13:58:26,N/A,protocol-anomaly,AppThreat-8883-8920,0x0,0,4294967295,,,c1b9f945-e213-4cdf-b77d-2700446a3baf,805901,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2024-08-19T13:58:32.880+00:00,,,,internet-utility,general-internet,browser-based,4,"used-by-malware,able-to-transfer-file,has-known-vulnerability,tunnel-other-application,pervasive-use",,web-browsing,no,no, <14>Nov 06 14:11:30 pa555 1,2024/11/06 14:11:30,0000000000001,THREAT,file,2562,2024/11/06 14:11:30,192.168.1.2,10.71.208.15,0.0.0.0,0.0.0.0,file download test rule,contoso\\steve,,web-browsing,vsys1,HOMENET,EXTNET,ethernet1/2,ethernet1/1,log-profile1,2024/11/06 14:11:30,994313,2,37268,443,0,0,0x1002000,tcp,alert,"elastic-agent.exe",Windows Executable (EXE)(52020),computer-and-internet-info,low,server-to-client,7367538158076100804,0x8000000000000000,192.168.0.0-192.168.255.255,United States,,,0,,,1,,,,,,,,0,199,479,0,0,,pa555,artifacts.elastic.co/downloads/beats/elastic-agent/elastic-agent-8.15.3+build202411051926-windows-x86_64.zip,,,,7213055707168598,,0,2024/11/06 14:11:30,N/A,N/A,AppThreat-8911-9049,0x0,0,4294967295,,,88e69ca4-8783-4b7c-9982-f73ec6f1a83c,1679420,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2024-11-06T14:11:30.036-05:00,,,,internet-utility,generate-internet,browser-based,2,"used-bymalware,able-to-transfer-file,has-known-vulnerability,tunnel-other-application,pervasive-use",,web-browsing,no,no, +<14>Nov 22 11:44:08 pa555 1,2024/11/22 11:44:07,013101001308,THREAT,spyware,2561,2024/11/22 11:44:07,67.43.156.0,67.43.156.1,0.0.0.0,0.0.0.0,A_DST_L7D_DNS,domain\user01,,dns-base,vsys1,Group,Servers,abc.123,abd.234,Panorama-Elastic,2024/10/01 10:43:54,34891187,2,59020,53,0,0,0x3000,tcp,sinkhole,"*.domain.dev",Suspicious Domain(12000000),any,medium,client-to-server,7401113521124350246,0x8000000000000000,10.0.0.0-10.255.255.255,10.0.0.0-10.255.255.255,,,0,,,0,,,,,,,,0,0,0,0,0,Core,AC5250,,,,,0,,0,,N/A,domain-edl,AppThreat-0-0,0x0,0,4291167295,,,5e791170-7507-4ab1-a951-79ebed0dad21,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2024-10-01T10:43:55.308+02:00,,,,infrastructure,networking,network-protocol,3,"used-by-malware,has-known-vulnerability,pervasive-use",dns,dns-base,no,no,_reportid diff --git a/packages/panw/data_stream/panos/_dev/test/pipeline/test-panw-panos-threat-sample.log-expected.json b/packages/panw/data_stream/panos/_dev/test/pipeline/test-panw-panos-threat-sample.log-expected.json index bc2a093d828..6952a140d2f 100644 --- a/packages/panw/data_stream/panos/_dev/test/pipeline/test-panw-panos-threat-sample.log-expected.json +++ b/packages/panw/data_stream/panos/_dev/test/pipeline/test-panw-panos-threat-sample.log-expected.json @@ -37041,6 +37041,188 @@ "domain": "contoso", "name": "steve" } + }, + { + "@timestamp": "2024-10-01T18:13:55.308+09:30", + "destination": { + "as": { + "number": 35908 + }, + "domain": "*.domain.dev", + "geo": { + "continent_name": "Asia", + "country_iso_code": "BT", + "country_name": "Bhutan", + "location": { + "lat": 27.5, + "lon": 90.5 + }, + "name": "10.0.0.0-10.255.255.255" + }, + "ip": "67.43.156.1", + "port": 53 + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "spyware_detected", + "category": [ + "intrusion_detection", + "threat", + "network" + ], + "kind": "alert", + "original": "<14>Nov 22 11:44:08 pa555 1,2024/11/22 11:44:07,013101001308,THREAT,spyware,2561,2024/11/22 11:44:07,67.43.156.0,67.43.156.1,0.0.0.0,0.0.0.0,A_DST_L7D_DNS,domain\\user01,,dns-base,vsys1,Group,Servers,abc.123,abd.234,Panorama-Elastic,2024/10/01 10:43:54,34891187,2,59020,53,0,0,0x3000,tcp,sinkhole,\"*.domain.dev\",Suspicious Domain(12000000),any,medium,client-to-server,7401113521124350246,0x8000000000000000,10.0.0.0-10.255.255.255,10.0.0.0-10.255.255.255,,,0,,,0,,,,,,,,0,0,0,0,0,Core,AC5250,,,,,0,,0,,N/A,domain-edl,AppThreat-0-0,0x0,0,4291167295,,,5e791170-7507-4ab1-a951-79ebed0dad21,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2024-10-01T10:43:55.308+02:00,,,,infrastructure,networking,network-protocol,3,\"used-by-malware,has-known-vulnerability,pervasive-use\",dns,dns-base,no,no,_reportid", + "outcome": "failure", + "severity": 3, + "timezone": "+09:30", + "type": [ + "denied" + ] + }, + "labels": { + "temporary_match": true + }, + "log": { + "level": "medium" + }, + "message": "67.43.156.0,67.43.156.1,0.0.0.0,0.0.0.0,A_DST_L7D_DNS,domain\\user01,,dns-base,vsys1,Group,Servers,abc.123,abd.234,Panorama-Elastic,2024/10/01 10:43:54,34891187,2,59020,53,0,0,0x3000,tcp,sinkhole,\"*.domain.dev\",Suspicious Domain(12000000),any,medium,client-to-server,7401113521124350246,0x8000000000000000,10.0.0.0-10.255.255.255,10.0.0.0-10.255.255.255,,,0,,,0,,,,,,,,0,0,0,0,0,Core,AC5250,,,,,0,,0,,N/A,domain-edl,AppThreat-0-0,0x0,0,4291167295,,,5e791170-7507-4ab1-a951-79ebed0dad21,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2024-10-01T10:43:55.308+02:00,,,,infrastructure,networking,network-protocol,3,\"used-by-malware,has-known-vulnerability,pervasive-use\",dns,dns-base,no,no,_reportid", + "network": { + "application": "dns-base", + "community_id": "1:Sj3H9LtJYWFdWXjIqpG6+D2O7fs=", + "direction": "inbound", + "transport": "tcp", + "type": "ipv4" + }, + "observer": { + "egress": { + "interface": { + "name": "abd.234" + }, + "zone": "Servers" + }, + "hostname": "AC5250", + "ingress": { + "interface": { + "name": "abc.123" + }, + "zone": "Group" + }, + "product": "PAN-OS", + "serial_number": "013101001308", + "type": "firewall", + "vendor": "Palo Alto Networks" + }, + "panw": { + "panos": { + "action": "sinkhole", + "action_flags": "0x8000000000000000", + "application": { + "category": "networking", + "characteristics": "used-by-malware,has-known-vulnerability,pervasive-use", + "container": "dns", + "is_saas": "no", + "is_sanctioned": "no", + "risk_level": 3, + "sub_category": "infrastructure", + "technology": "network-protocol", + "tunneled": "dns-base" + }, + "cloud_report": { + "id": "_reportid" + }, + "content_version": "AppThreat-0-0", + "device_group_hierarchy1": "0", + "device_group_hierarchy2": "0", + "device_group_hierarchy3": "0", + "device_group_hierarchy4": "0", + "flow_id": "34891187", + "generated_time": "2024-11-22T11:44:07.000+09:30", + "high_resolution_timestamp": "2024-10-01T18:13:55.308+09:30", + "http2_connection": "0", + "imsi": "0", + "log_profile": "Panorama-Elastic", + "logged_time": "2024-10-01T10:43:54.000+09:30", + "parent_session": { + "id": "0" + }, + "partial_hash": "0", + "payload_protocol_id": "4291167295", + "received_time": "2024-11-22T11:44:07.000+09:30", + "repeat_count": 2, + "ruleset": "A_DST_L7D_DNS", + "sctp": { + "assoc_id": "0" + }, + "sequence_number": "7401113521124350246", + "sub_type": "spyware", + "threat": { + "id": "12000000", + "name": "Suspicious Domain" + }, + "threat_category": "domain-edl", + "tunnel_type": "N/A", + "type": "THREAT", + "url": { + "category": "any" + }, + "url_idx": "0", + "virtual_sys": "vsys1", + "vsys_name": "Core", + "wildfire": { + "report_id": "0" + } + } + }, + "related": { + "hosts": [ + "AC5250" + ], + "ip": [ + "67.43.156.0", + "67.43.156.1" + ], + "user": [ + "user01" + ] + }, + "rule": { + "name": "A_DST_L7D_DNS", + "uuid": "5e791170-7507-4ab1-a951-79ebed0dad21" + }, + "source": { + "as": { + "number": 35908 + }, + "geo": { + "continent_name": "Asia", + "country_iso_code": "BT", + "country_name": "Bhutan", + "location": { + "lat": 27.5, + "lon": 90.5 + }, + "name": "10.0.0.0-10.255.255.255" + }, + "ip": "67.43.156.0", + "port": 59020, + "user": { + "domain": "domain", + "name": "user01" + } + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "domain": "*.domain.dev", + "original": "*.domain.dev" + }, + "user": { + "domain": "domain", + "name": "user01" + } } ] } diff --git a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/threat.yml b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/threat.yml index 31ecea3b1aa..ca6d8362122 100644 --- a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/threat.yml +++ b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/threat.yml @@ -300,6 +300,11 @@ processors: tag: set_url_from_file field: url.original copy_from: _temp_.future_use3 + - set: + if: 'ctx.panw?.panos?.threat_category == "domain-edl" && ctx.panw?.panos?.misc instanceof String' + tag: set_domain_edl_url + field: url.original + copy_from: panw.panos.misc # Crude implementation of `uri_parts` as its not working well due to lack of scheme. # When the scheme of the URL is absent, this script parses the URL in `ctx.panw.panos.misc` into components namely # `url.original`, `url.domain`, `url.port`, `url.path`, `url.query`, `url.extension` diff --git a/packages/panw/manifest.yml b/packages/panw/manifest.yml index 5a0991822af..96bb14f228c 100644 --- a/packages/panw/manifest.yml +++ b/packages/panw/manifest.yml @@ -1,6 +1,6 @@ name: panw title: Palo Alto Next-Gen Firewall -version: "4.1.0" +version: "4.2.0" description: Collect logs from Palo Alto next-gen firewalls with Elastic Agent. type: integration format_version: "3.0.3" From 312ec7a4c94543421e1a6c1d10d994c88164425f Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Fri, 22 Nov 2024 13:59:48 -0800 Subject: [PATCH 2/2] Update changelog --- packages/panw/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/panw/changelog.yml b/packages/panw/changelog.yml index f619b93871b..0ec206e3dac 100644 --- a/packages/panw/changelog.yml +++ b/packages/panw/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Parse URL in threat logs with 'domain-edl' threat category type: enhancement - link: https://github.com/elastic/integrations/pull/999999 + link: https://github.com/elastic/integrations/pull/11837 - version: "4.1.0" changes: - description: Parse URL from threat-file event type