fluent-bit-filters-bro-http.conf

# Tabular log format parser
[FILTER]
    Name parser
    Match ${observer_product}.http
    Key_Name log
    Parser bro_http_parser
    Preserve_Key on
    Reserve_Data on

# JSON log format parser
[FILTER]
    Name parser
    Match ${observer_product}.http.json
    Key_Name log
    Parser bro_http_parser_json
    Preserve_Key off
    Reserve_Data on

# Rename 4-tuple fields from json
[FILTER]
    Name modify
    Match ${observer_product}.http.json
    Rename id.orig_h orig_h
    Rename id.orig_p orig_p
    Rename id.resp_h resp_h
    Rename id.resp_p resp_p

# Prefix all regular keys ([A-z]*) with 'zeek_http_'
# This will also catch any extended fields to move them under zeek.http
[FILTER]
    Name lua
    Match ${observer_product}.http*
    Script  bro_http_parse.lua
    Call    bro_http_prefix_all

[FILTER]
    Name modify
    Match ${observer_product}.http
    Add zeek_log_file http
    Add zeek_log_format tab

# These are special "_*" keys, rename them separately
[FILTER]
    Name modify
    Match ${observer_product}.http.json
    Rename _path zeek_log_file
    Rename _write_ts zeek_write_ts
    Add zeek_log_format json

# Pre-process HTTP-specific fields
[FILTER]
    Name modify
    Match ${observer_product}.http*
    Add _event_dataset ${observer_product}.http
    Add  url_scheme    http
    Rename zeek_http_uid        zeek_uid
    Rename zeek_http_orig_h     source_ip
    Rename zeek_http_orig_p     source_port
    Rename zeek_http_resp_h     destination_ip
    Rename zeek_http_resp_p     destination_port
    Rename zeek_http_host       url_domain
    Rename zeek_http_username   url_username
    Rename zeek_http_password   url_password
    Rename zeek_http_user_agent user_agent_original
    Rename zeek_http_version           _http_version
    Rename zeek_http_method            _http_request_method
    Rename zeek_http_referrer          _http_request_referrer
    Rename zeek_http_request_body_len  _http_request_body_bytes
    Rename zeek_http_response_body_len _http_response_body_bytes
    Rename zeek_http_status_code       _http_response_status_code
    Rename zeek_http_log               _log_original

# Cleanup - remove keys represending IP addresses if value is "-"
[FILTER]
    Name modify
    Match ${observer_product}.http*
    Condition Key_value_equals source_ip -
    Remove_regex client_ip

[FILTER]
    Name modify
    Match ${observer_product}.http*
    Condition Key_value_equals destination_ip -
    Remove_regex server_ip

[FILTER]
    Name modify
    Match ${observer_product}.http*
    Condition Key_value_equals zeek_http_info_msg -
    Remove_regex zeek_http_info_msg

[FILTER]
    Name modify
    Match ${observer_product}.http*
    Condition Key_value_equals url_username -
    Remove_regex url_username

[FILTER]
    Name modify
    Match ${observer_product}.http*
    Condition Key_value_equals url_password -
    Remove_regex url_password

[FILTER]
    Name modify
    Match ${observer_product}.http*
    Condition Key_value_equals _http_request_referrer -
    Remove_regex _http_request_referrer

[FILTER]
    Name modify
    Match ${observer_product}.http*
    Condition Key_value_equals user_agent_original -
    Remove_regex user_agent_original

# Next keys are arrays for json input format, cleanup only if tabular
[FILTER]
    Name modify
    Match ${observer_product}.http
    Condition Key_value_equals zeek_http_resp_fuids -
    Remove_regex zeek_http_resp_fuids

[FILTER]
    Name modify
    Match ${observer_product}.http
    Condition Key_value_equals zeek_http_resp_filenames -
    Remove_regex zeek_http_resp_filenames

[FILTER]
    Name modify
    Match ${observer_product}.http
    Condition Key_value_equals zeek_http_resp_mime_types -
    Remove_regex zeek_http_resp_mime_types

[FILTER]
    Name modify
    Match ${observer_product}.http
    Condition Key_value_equals zeek_http_orig_fuids -
    Remove_regex zeek_http_orig_fuids

[FILTER]
    Name modify
    Match ${observer_product}.http
    Condition Key_value_equals zeek_http_orig_filenames -
    Remove_regex zeek_http_orig_filenames

[FILTER]
    Name modify
    Match ${observer_product}.http
    Condition Key_value_equals zeek_http_orig_mime_types -
    Remove_regex zeek_http_orig_mime_types

[FILTER]
    Name modify
    Match ${observer_product}.http
    Condition Key_value_equals zeek_http_tags (empty)
    Remove_regex zeek_http_tags

[FILTER]
    Name modify
    Match ${observer_product}.http
    Condition Key_value_equals zeek_http_proxied -
    Remove_regex zeek_http_proxied

# Copies, before they are nested
[FILTER]
    Name modify
    Match ${observer_product}.http*
    Copy zeek_uid        _event_id
    Copy destination_port url_port

# ECS: Parse uri into ECS fields
[FILTER]
    Name    lua
    Match ${observer_product}.http*
    Script  bro_http_parse.lua
    Call    bro_http_parse_uri

# ECS: Parse vectors into arrays
[FILTER]
    Name    lua
    Match ${observer_product}.http
    Script  bro_http_parse.lua
    Call    bro_http_parse_arrays

# Cleanup arrays for JSON log format
[FILTER]
    Name    lua
    Match ${observer_product}.http.json
    Script  bro_http_parse.lua
    Call    bro_http_cleanup_arrays

# Remove no longer needed keys
[FILTER]
    Name modify
    Match ${observer_product}.http*
    Remove zeek_http_uri
    
# CUSTOM: Nest JSON objects to comply with ECS notation: Zeek HTTP - inner layer
[FILTER]
    Name nest
    Match ${observer_product}.http*
    Operation nest
    Wildcard zeek_http_*
    Nest_under http
    Remove_prefix zeek_http_

# CUSTOM: Nest JSON objects to comply with ECS notation: Zeek HTTP - outer layer
[FILTER]
    Name nest
    Match ${observer_product}.http*
    Operation nest
    Wildcard zeek_*
    Wildcard http*
    Nest_under zeek
    Remove_prefix zeek_

# ECS: Nest JSON objects to comply with ECS notation: HTTP.request
[FILTER]
    Name nest
    Match ${observer_product}.http*
    Operation nest
    Wildcard _http_request_*
    Nest_under request
    Remove_prefix _http_request_

# ECS: Nest JSON objects to comply with ECS notation: HTTP.response
[FILTER]
    Name nest
    Match ${observer_product}.http*
    Operation nest
    Wildcard _http_response_*
    Nest_under response
    Remove_prefix _http_response_

# ECS: Nest JSON objects to comply with ECS notation: HTTP
[FILTER]
    Name nest
    Match ${observer_product}.http*
    Operation nest
    Wildcard _http_*
    Wildcard request*
    Wildcard response*
    Nest_under http
    Remove_prefix _http_

# ECS: Nest JSON objects to comply with ECS notation: URL
[FILTER]
    Name nest
    Match ${observer_product}.http*
    Operation nest
    Wildcard url_*
    Nest_under url
    Remove_prefix url_

# ECS: Nest JSON objects to comply with ECS notation: USER_AGENT
[FILTER]
    Name nest
    Match ${observer_product}.http*
    Operation nest
    Wildcard user_agent_*
    Nest_under user_agent
    Remove_prefix user_agent_

# Copies - moved below nesting filters
[FILTER]
    Name modify
    Match ${observer_product}.http*
    Copy source_ip        client_ip
    Copy source_port      client_port
    Copy destination_ip   server_ip
    Copy destination_port server_port