Parsing Syslog for Cisco FirePower

[gebhard73]

The below file allows parsing of Cisco FirePower / ASA access log and VPN log files. It could be a starting point to be adopted to local needs. In the bottom I've added the template definition for the fields.

It's been in testing for about 13 Million events per hour peak and seems to do its job. Comments, suggestions, and improvements are welcomed.

New version as of 2021-03-31. For changes see comments inline.

For a general how-to see #3175 .

/opt/so/saltstack/local/salt/elasticsearch/files/ingest/syslog

{
  "description" : "syslog, adopted for FirePower parsing",
  "processors" : [
    {
        "dissect": {
                "field": "message",
                "pattern" : "%{message}",
                "on_failure": [ { "drop" : { } } ]
        },
        "remove": {
                "field": [ "type", "agent" ],
                "ignore_failure": true
        }
    },
/*
Dissect FirePower access log lines
- 2021-03-19, v0.5: first beta version (Gebhard)
- 2021-03-23, v0.5a: added ignore_failure to kv (#02) (Gebhard)
- 2021-03-26, v0.6: removed field renaming, use timestamp from event (Gebhard)
- 2021-03-29, v0.7: added parsing for ASA VPN (Gebhard)
*/
/*
00 - Preparation:
- syslogid: will always be set for FirePower log entries so we use this to distinguish them from other lines
            and have to initialize it with 0
*/
    {
      "set": {
        "field": "syslogid",
        "value": 0
      }
    },
/*
01 - Parse the log line before the key-value pairs, should match FirePower lines only
*/
    {
      "dissect": {
        "field": "message",
        "pattern": "<%{?syslogprio}>%{tmptimestamp}  %{?empty->}FTD-1-%{syslogid}: %{real_message}",
        "ignore_failure": true
      }
    },
/*
02 - Parse the key-value section
- field names will be altered afterwards as needed
- temporary field real_message will be deleted directly
- not allowed in content of text fields from FirePower: ", " and ": "
*/
    {
      "kv": {
        "field": "real_message",
        "field_split": ", ",
        "value_split": ": ",
        "if": "ctx.syslogid != 0",
        "ignore_failure": true
      }
    },
    {
      "remove": {
        "field": "real_message",
        "ignore_missing": true,
        "ignore_failure": true
      }
    },
/*
03 - Rename field names as needed
- syslogids currently obeyed: 430002, 430003, 430004, 430005
- field names must match definitions in index template
- removed 2021-03-26 for being too expensive
*/
/*
10 - Paring for ASA VPN Logs
- created 2021-03-29 (Gebhard)
*/
    {
      "dissect": {
        "field": "message",
        "pattern": "<%{?syslogprio}>%{tmptimestamp}: %{?empty->}ASA-4-%{syslogid}: Group <%{VPNGroup}> User <%{VPNUser}> IP <%{VPNIPExt}> IPv4 Address <%{VPNIPInt}> IPv6 address <::> assigned to session",
        "if": "ctx.syslogid == 0",
        "ignore_failure": true
      }
    },
    {
      "dissect": {
        "field": "message",
        "pattern": "<%{?syslogprio}>%{tmptimestamp}: %{?empty->}ASA-6-%{syslogid}: Group <%{VPNGroup}> User <%{VPNUser}> IP <%{VPNIPExt}> %{?dummy}",
        "if": "ctx.syslogid == 0",
        "ignore_failure": true
      }
    },
    {
      "dissect": {
        "field": "message",
        "pattern": "<%{?syslogprio}>%{tmptimestamp}: %{?empty->}ASA-5-%{syslogid}: Group = %{VPNGroup}, IP = %{VPNIPExt}, %{?dummy}",
        "if": "ctx.syslogid == 0",
        "ignore_failure": true
      }
    },
    {
      "dissect": {
        "field": "message",
        "pattern": "<%{?syslogprio}>%{tmptimestamp}: %{?empty->}ASA-4-%{syslogid}: Group = %{VPNGroup}, Username = %{VPNUser}, IP = %{VPNIPExt}, %{?dummy}",
        "if": "ctx.syslogid == 0",
        "ignore_failure": true
      }
    },
/*
98 - Clean up
- add device names for UUIDs
- handle time format
*/
    {
      "set": {
        "field": "DeviceName",
        "value": "system1",
        "if": "ctx.DeviceUUID == 'blahfasel'"
      }
    },
    {
      "set": {
        "field": "DeviceName",
        "value": "system2",
        "if": "ctx.DeviceUUID == 'faselblah'"
      }
    },
    {
      "set": {
        "field": "DeviceName",
        "value": "systemA",
        "if": "ctx.log.source.address == '8.8.8.8'"
      }
    },
    {
      "set": {
        "field": "DeviceName",
        "value": "systemB",
        "if": "ctx.log.source.address == '4.4.4.4'"
      }
    },
    {
      "date": {
        "field": "tmptimestamp",
        "target_field": "syslog.timestamp",
        "formats": ["yyyy-MM-dd'T'HH:mm:ss'Z'"],
        "timezone": "UTC",
        "ignore_failure": true
      }
    },
    {
      "remove": {
        "field": "tmptimestamp",
        "ignore_missing": true,
        "ignore_failure": true
      }
    },
/*
99 - Default handling with expensive grok only for non-FirePower messages
- only when syslogid == 0
*/
    {
      "grok": {
        "field": "message",
        "patterns": [
          "^<%{INT:syslog.priority}>%{DATA:syslog.timestamp} %{WORD:source.application}: %{GREEDYDATA:real_message}$",
          "^%{SYSLOGTIMESTAMP:syslog.timestamp} %{SYSLOGHOST:syslog.host} %{SYSLOGPROG:syslog.program}: CEF:0\\|%{DATA:vendor}\\|%{DATA:product}\\|%{GREEDYDATA:message2}$"
        ],
        "if": "ctx.syslogid == 0",
        "ignore_failure": true
      }
    },
    { "set": { "if": "ctx.source?.application == 'filterlog'", "field": "dataset", "value": "firewall", "ignore_failure": true  } },
    { "set": { "if": "ctx.vendor != null", "field": "module", "value": "{{ vendor }}", "ignore_failure": true } },
    { "set": { "if": "ctx.product != null", "field": "dataset", "value": "{{ product }}", "ignore_failure": true } },
    { "set": { "field": "ingest.timestamp",      "value": "{{ @timestamp }}"       } },
    { "date": { "if": "ctx.syslog?.timestamp != null", "field": "syslog.timestamp",       "target_field": "@timestamp",   "formats": ["MMM  d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601", "UNIX"], "ignore_failure": true  } },
    { "remove":         { "field": ["pid", "program"],  "ignore_missing": true, "ignore_failure": true  } },
    { "pipeline": { "if": "ctx.vendor != null && ctx.product != null", "name": "{{ vendor }}.{{ product }}", "ignore_failure": true } },
    { "pipeline": { "if": "ctx.dataset == 'firewall'", "name": "filterlog", "ignore_failure": true } },
    { "pipeline":       { "name": "common"                                             } }
  ]
}

Index definitions for the new fields:
/opt/so/saltstack/local/salt/elasticsearch/templates/so/so-common-template.json

[...]
       "properties":{
        "syslogid":{
          "type":"keyword"
        },
        "DeviceUUID":{
          "type":"keyword"
        },
        "AccessControlRuleAction":{
          "type":"keyword"
        },
        "DeviceName":{
          "type":"keyword"
        },
        "SrcIP":{
          "type":"ip"
        },
        "SrcPort":{
          "type":"integer"
        },
        "DstIP":{
          "type":"ip"
        },
        "DstPort":{
          "type":"integer"
        },
        "Protocol":{
          "type":"keyword"
        },
        "IngressInterface":{
          "type":"keyword"
        },
        "EgressInterface":{
          "type":"keyword"
        },
        "IngressZone":{
          "type":"keyword"
        },
        "EgressZone":{
          "type":"keyword"
        },
        "ACPolicy":{
          "type":"keyword"
        },
        "AccessControlRuleName":{
          "type":"keyword"
        },
        "Prefilter Policy":{
          "type":"keyword"
        },
        "User":{
          "type":"keyword"
        },
        "UserAgent":{
          "type":"text"
        },
        "Client":{
          "type":"keyword"
        },
        "ClientVersion":{
          "type":"keyword"
        },
        "ApplicationProtocol":{
          "type":"keyword"
        },
        "WebApplication":{
          "type":"keyword"
        },
        "InitiatorPackets":{
          "type":"long"
        },
        "ResponderPackets":{
          "type":"long"
        },
        "InitiatorBytes":{
          "type":"long"
        },
        "ResponderBytes":{
          "type":"long"
        },
        "NAPPolicy":{
          "type":"keyword"
        },
        "ReferencedHost":{
          "type":"text"
        },
        "URL":{
          "type":"text"
        },
        "ICMPType":{
          "type":"keyword"
        },
        "ICMPCode":{
          "type":"keyword"
        },
        "DNSQuery":{
          "type":"text"
        },
        "DNSRecordType":{
          "type":"keyword"
        },
        "ConnectionDuration":{
          "type":"long"
        },
        "FileAction":{
          "type":"keyword"
        },
        "FileDirection":{
          "type":"keyword"
        },
        "FileName":{
          "type":"text"
        },
        "FilePolicy":{
          "type":"keyword"
        },
        "FileSHA256":{
          "type":"keyword"
        },
        "FileSandboxStatus":{
          "type":"keyword"
        },
        "FileSize":{
          "type":"long"
        },
        "FileStorageStatus":{
          "type":"keyword"
        },
        "FileType":{
          "type":"keyword"
        },
        "FirstPacketSecond":{
          "type":"date"
        },
        "SHA_Disposition":{
          "type":"keyword"
        },
        "SperoDisposition":{
          "type":"keyword"
        },
        "ThreatName":{
          "type":"keyword"
        },
        "ArchiveDepth":{
          "type":"text"
        },
        "ArchiveFileName":{
          "type":"text"
        },
        "ArchiveFileStatus":{
          "type":"keyword"
        },
        "FileStaticAnalysisStatus":{
          "type":"keyword"
        },
        "ThreatScore":{
          "type":"integer"
        },
        "URI":{
          "type":"text"
        },
        "syslog.timestamp":{
          "type":"date"
        },
        "originalClientSrcIP":{
          "type":"ip"
        },
        "VPNGroup":{
          "type":"keyword"
        },
        "VPNUser":{
          "type":"text"
        },
        "VPNIPExt":{
          "type":"ip"
        },
        "VPNIPInt":{
          "type":"ip"
        },

[...]

[stijnpieters]

A question: Why did you do this directly in the .../syslog file, and not in a separate, custom pipeline file?

I made a partial parsing pipeline for Sophos UTM and made the syslog file direct the pipeline to my custom one based on certain parameters in de syslog message. However currently they're redirected on basis of the hostname of the syslog sender, which makes my config less dynamic as I would have hoped.

[gebhard73]

it's just because I like so have it simple for a system which is dedicated for storing this kind of logs only (KISS)

[jchenturbo]

Hi.. trying to set this up..will this work with ASA that doesnt have firepower?

[gebhard73]

It depends which fields you have in the syslog row ... you may have to adjust the field names. I would give it a try, at least it's a starting point. Perhaps filebeat modules is also an option starting with SO 2.3.60 (https://docs.securityonion.net/en/2.3/filebeat.html#modules / https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-module-cisco.html)