Skip to content

Commit

Permalink
[HOT FIX] v0.1.7-alpha02262019 - Logstash Pipeline
Browse files Browse the repository at this point in the history
helk-logstash
+ Added offline plugins file
+ Updated win security conversion
+ cleaned process-name filter & process-name-split configs
+ cleaned process-id filter & proces-id conversion configs
+ set kafka max poll records to 500
+ updated SOURCE_ & TARGET_ field names from process entity to be renamed process_source_ and process_target. Following the basic `entity_context_property` from OSSEM CIM
  • Loading branch information
Cyb3rWard0g committed Feb 26, 2019
1 parent 65131b2 commit cfb9b98
Show file tree
Hide file tree
Showing 16 changed files with 284 additions and 266 deletions.
3 changes: 2 additions & 1 deletion docker/helk-logstash/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,5 @@ LABEL maintainer="Roberto Rodriguez @Cyb3rWard0g"
LABEL description="Dockerfile base for the HELK Logstash."

RUN mv /usr/share/logstash/config/logstash.yml /usr/share/logstash/config/logstash.yml.backup
COPY --chown=logstash:logstash config/logstash.yml /usr/share/logstash/config/logstash.yml
COPY --chown=logstash:logstash config/logstash.yml /usr/share/logstash/config/logstash.yml
COPY --chown=logstash:logstash plugins/logstash-offline-plugins-6.6.1.zip /usr/share/logstash/logstash-offline-plugins-6.6.1.zip
2 changes: 1 addition & 1 deletion docker/helk-logstash/pipeline/0002-kafka-input.conf
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,6 @@ input {
session_timeout_ms => "30000"
max_poll_interval_ms => "300000"
#############################
max_poll_records => "250"
max_poll_records => "500"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
# HELK process-name-filter filter conf
# HELK build Stage: Alpha
# Author: Roberto Rodriguez (@Cyb3rWard0g)
# License: GPL-3.0

filter {

This comment has been minimized.

Copy link
@neu5ron

neu5ron Feb 26, 2019

Collaborator

recommend moving all the "add_field" into the mutate rename.

if [event_id] {
if [Image] {
mutate { add_field => { "z_logstash_pipeline" => "1523_1" } }
mutate { rename => { "Image" => "process_path" } }
}
if [Application] {
mutate { add_field => { "z_logstash_pipeline" => "1523_2" } }
mutate { rename => { "Application" => "process_path" } }
}
if [NewProcessName] {
mutate { add_field => { "z_logstash_pipeline" => "1523_3" } }
mutate { rename => { "NewProcessName" => "process_path" } }
}
if [ProcessName] {
mutate { add_field => { "z_logstash_pipeline" => "1523_4" } }
mutate { rename => { "ProcessName" => "process_path" }}
}
if [ParentProcessName] {
mutate { add_field => { "z_logstash_pipeline" => "1523_5" } }
mutate { rename => { "ParentProcessName" => "process_parent_path" } }
}
if [ParentImage] {
mutate { add_field => { "z_logstash_pipeline" => "1523_6" } }
mutate { rename => { "ParentImage" => "process_parent_path" } }
}
if [TargetImage] {
mutate { add_field => { "z_logstash_pipeline" => "1523_7" } }
mutate { rename => { "TargetImage" => "process_target_path" } }
}
if [SourceImage] {
mutate { add_field => { "z_logstash_pipeline" => "1523_8" } }
mutate { rename => { "SourceImage" => "process_path" } }
}
if [ProdessName] {
mutate { rename => { "ProdessName" => "process_path" } }
}
}
}
102 changes: 0 additions & 102 deletions docker/helk-logstash/pipeline/1523-winevent-process-name-split.conf

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
# HELK process-ids filter conf
# HELK build Stage: Alpha
# Author: Roberto Rodriguez (@Cyb3rWard0g)
# License: GPL-3.0

filter {

This comment has been minimized.

Copy link
@neu5ron

neu5ron Feb 26, 2019

Collaborator

recommend moving all the add_field's into their corresponding rename mutate

if [event_id] {
if [ProcessId] {
mutate { add_field => { "z_logstash_pipeline" => "1524_2" } }
mutate { rename => { "ProcessId" => "process_id" } }
}
if [NewProcessId] {
mutate { add_field => { "z_logstash_pipeline" => "1524_3" } }
mutate { rename => { "NewProcessId" => "process_id" } }
}
if [ParentProcessId] {
mutate { add_field => { "z_logstash_pipeline" => "1524_5" } }
mutate { rename => { "ParentProcessId" => "process_parent_id" } }
}
if [ProcessGuid] {
mutate { add_field => { "z_logstash_pipeline" => "1524_6" } }
mutate { rename => { "ProcessGuid" => "process_guid" } }
}
if [ParentProcessGuid] {
mutate { add_field => { "z_logstash_pipeline" => "1524_7" } }
mutate { rename => { "ParentProcessGuid" => "process_parent_guid" } }
}
if [SourceProcessGuid] {
mutate { add_field => { "z_logstash_pipeline" => "1524_8" } }
mutate { rename => { "SourceProcessGuid" => "process_guid" } }
}
if [SourceProcessGUID] {
mutate { add_field => { "z_logstash_pipeline" => "1524_9" } }
mutate { rename => { "SourceProcessGUID" => "process_guid" } }
}
if [SourceProcessId] {
mutate { add_field => { "z_logstash_pipeline" => "1524_11" } }
mutate { rename => { "SourceProcessId" => "process_id" } }
}
if [TargetProcessGuid] {
mutate { add_field => { "z_logstash_pipeline" => "1524_12" } }
mutate { rename => { "TargetProcessGuid" => "process_target_guid" } }
}
if [TargetProcessGUID] {
mutate { add_field => { "z_logstash_pipeline" => "1524_13" } }
mutate { rename => { "TargetProcessGUID" => "process_target_guid" } }
}
if [TargetProcessId] {
mutate { add_field => { "z_logstash_pipeline" => "1524_15" } }
mutate { rename => { "TargetProcessId" => "process_target_id" } }
}
}
}
28 changes: 2 additions & 26 deletions docker/helk-logstash/pipeline/1531-winevent-sysmon-filter.conf
Original file line number Diff line number Diff line change
Expand Up @@ -20,19 +20,10 @@ filter {
source => "RuleName"
field_split => ","
value_split => "="
prefix => "mitre_"
prefix => "rule_"
transform_key => "lowercase"
}
}
if [Image] {
mutate {
rename => {
"Image" => "process_path"
"ProcessGuid" => "process_guid"
"ProcessId" => "process_id"
}
}
}
if [Hashes] {
kv {
source => "Hashes"
Expand All @@ -42,18 +33,6 @@ filter {
transform_key => "lowercase"
}
}
if [TargetImage] {
mutate {
rename => {
"SourceImage" => "process_path"
"SourceProcessGUID" => "process_guid"
"SourceProcessId" => "process_id"
"TargetImage" => "target_process_path"
"TargetProcessGUID" => "target_process_guid"
"TargetProcessId" => "target_process_id"
}
}
}
if [User] {
grok {
match => { "User" => "%{GREEDYDATA:user_domain}\\%{GREEDYDATA:user_name}" }
Expand All @@ -66,13 +45,10 @@ filter {
rename => {
"CommandLine" => "process_command_line"
"CurrentDirectory" => "process_current_directory"
"ParentImage" => "process_parent_path"
"ParentCommandLine" => "process_parent_command_line"
"IntegrityLevel" => "process_integrity_level"
"LogonGuid" => "user_logon_guid"
"LogonId" => "user_logon_id"
"ParentProcessGuid" => "process_parent_guid"
"ParentProcessId" => "process_parent_id"
"TerminalSessionId" => "user_session_id"
"FileVersion" => "file_version"
"Description" => "file_description"
Expand Down Expand Up @@ -238,7 +214,7 @@ filter {
date {
timezone => "UTC"
match => [ "CreationUtcTime", "YYYY-MM-dd HH:mm:ss.SSS" ]
target => "@date_creation"
target => "@file_date_creation"
tag_on_failure => [ "_sysmon_CreationUtcTime_datefailure", "_sysmon_datefailure", "_dateparsefailure" ]
}
date {
Expand Down
Loading

0 comments on commit cfb9b98

Please sign in to comment.