Merge pull request #93 from neu5ron/master

Logstash refactoring and many logstash additions
Cyb3rWard0g · Aug 2, 2018 · c7af8e4 · c7af8e4
2 parents 931d567 + 3433425
commit c7af8e4
Show file tree

Hide file tree

Showing 67 changed files with 4,113 additions and 1,599 deletions.
diff --git a/helk-kibana/dashboards/Global_Dashboard.json b/helk-kibana/dashboards/Global_Dashboard.json
diff --git a/helk-kibana/dashboards/Sysmon_Dashboard.json b/helk-kibana/dashboards/Sysmon_Dashboard.json
diff --git a/helk-kibana/dashboards/Sysmon_Network_Dashboard.json b/helk-kibana/dashboards/Sysmon_Network_Dashboard.json
diff --git a/helk-kibana/kibana.yml b/helk-kibana/kibana.yml
@@ -12,7 +12,7 @@ server.host: "helk-kibana"
 #server.basePath: ""
 
 # The maximum payload size in bytes for incoming server requests.
-#server.maxPayloadBytes: 1048576
+server.maxPayloadBytes: 2048576
 
 # The Kibana server's name.  This is used for display purposes.
 server.name: "helk-kibana"
@@ -58,11 +58,11 @@ elasticsearch.url: "http://helk-elasticsearch:9200"
 
 # Time in milliseconds to wait for Elasticsearch to respond to pings. Defaults to the value of
 # the elasticsearch.requestTimeout setting.
-#elasticsearch.pingTimeout: 1500
+elasticsearch.pingTimeout: 7500
 
 # Time in milliseconds to wait for responses from the back end or Elasticsearch. This value
 # must be a positive integer.
-elasticsearch.requestTimeout: 60000
+elasticsearch.requestTimeout: 300000
 
 # List of Kibana client-side headers to send to Elasticsearch. To send *no* client-side
 # headers, set this value to [] (an empty list).

diff --git a/helk-logstash/output_templates/10-logs-all-default.json b/helk-logstash/output_templates/10-logs-all-default.json
@@ -0,0 +1,57 @@
+{
+  "order": 10,
+  "index_patterns": [ "logs-*" ],
+  "version": 2018080201,
+  "settings": {
+    "index": {
+      "mapping": {
+        "ignore_malformed": true,
+        "total_fields.limit": "1000",
+        "coerce": true
+      }
+    },
+    "refresh_interval": "30s"
+  },
+  "mappings": {
+    "_doc": {
+      "dynamic": "true",
+      "dynamic_templates": [
+        {
+          "strings": {
+            "match_mapping_type": "string",
+            "mapping": {
+              "type": "text",
+              "norms": false,
+              "fields": {
+                "keyword": {
+                  "type": "keyword",
+                  "ignore_above": 1024
+                }
+              }
+            }
+          }
+        }
+      ],
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "@version": {
+          "type": "keyword"
+        },
+        "log_ingest_timestamp": {
+          "type": "date"
+        },
+        "z_logstash_type": {
+          "enabled": false
+        },
+        "z_original_message": {
+          "enabled": false
+        },
+        "z_logstash_pipeline": {
+          "enabled": false
+        }
+      }
+    }
+  }
+}
diff --git a/helk-logstash/output_templates/50-logs-winevent-all.json b/helk-logstash/output_templates/50-logs-winevent-all.json
@@ -0,0 +1,153 @@
+{
+  "order": 50,
+  "index_patterns": [ "logs-endpoint-winevent-*" ],
+  "version": 2018080101,
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "cli_n_file_analyzer": {
+          "type": "custom",
+          "filter": [ "case_change_only_delim_filter", "three_or_more_tokenizer_limit_filter", "cli_n_file_word_delim_filter", "two_or_more_tokenizer_limit_filter", "lowercase", "unique" ],
+          "tokenizer": "standard"
+        }
+      },
+      "filter": {
+        "cli_n_file_word_delim_filter": {
+          "type": "word_delimiter",
+          "generate_word_parts": true,
+          "split_on_case_change": true,
+          "split_on_numerics": false,
+          "stem_english_possessive": false,
+          "generate_number_parts": true,
+          "preserve_original": true
+        },
+        "case_change_only_delim_filter": {
+          "type": "word_delimiter",
+          "generate_word_parts": true,
+          "split_on_case_change": true,
+          "split_on_numerics": false,
+          "stem_english_possessive": false,
+          "generate_number_parts": false,
+          "preserve_original": true
+        },
+        "two_or_more_tokenizer_limit_filter": {
+          "type": "length",
+          "min": 2
+        },
+        "three_or_more_tokenizer_limit_filter": {
+          "type": "length",
+          "min": 3
+        }
+      },
+      "normalizer": {
+        "lowercase_normalizer": {
+          "type": "custom",
+          "char_filter": [ ],
+          "filter": [ "lowercase" ]
+        }
+      }
+    },
+    "index": {
+      "mapping": {
+        "total_fields.limit": "3000"
+      }
+    },
+    "refresh_interval": "30s"
+  },
+  "mappings": {
+    "_doc":{
+      "properties":{
+        "process_id":{"type":"integer"},
+        "event_id":{"type":"integer"},
+        "file_name": {
+          "type": "text",
+          "norms": false,
+          "analyzer": "cli_n_file_analyzer",
+          "fields": {
+            "keyword": {
+              "ignore_above": 7500,
+              "type": "keyword"
+            }
+          }
+        },
+        "logon_process_name": {
+          "type": "text",
+          "norms": false,
+          "analyzer": "cli_n_file_analyzer",
+          "fields": {
+            "keyword": {
+              "ignore_above": 7500,
+              "type": "keyword"
+            }
+          }
+        },
+        "object_name": {
+          "type": "text",
+          "norms": false,
+          "analyzer": "cli_n_file_analyzer",
+          "fields": {
+            "keyword": {
+              "ignore_above": 7500,
+              "type": "keyword"
+            }
+          }
+        },
+        "process_command_line": {
+          "type": "text",
+          "norms": false,
+          "analyzer": "cli_n_file_analyzer",
+          "fields": {
+            "keyword": {
+              "ignore_above": 7500,
+              "type": "keyword"
+            }
+          }
+        },
+        "process_current_directory": {
+          "type": "text",
+          "norms": false,
+          "analyzer": "cli_n_file_analyzer",
+          "fields": {
+            "keyword": {
+              "ignore_above": 7500,
+              "type": "keyword"
+            }
+          }
+        },
+        "process_parent_path": {
+          "type": "text",
+          "norms": false,
+          "analyzer": "cli_n_file_analyzer",
+          "fields": {
+            "keyword": {
+              "ignore_above": 7500,
+              "type": "keyword"
+            }
+          }
+        },
+        "process_parent_command_line": {
+          "type": "text",
+          "norms": false,
+          "analyzer": "cli_n_file_analyzer",
+          "fields": {
+            "keyword": {
+              "ignore_above": 7500,
+              "type": "keyword"
+            }
+          }
+        },
+        "process_path": {
+          "type": "text",
+          "norms": false,
+          "analyzer": "cli_n_file_analyzer",
+          "fields": {
+            "keyword": {
+              "ignore_above": 7500,
+              "type": "keyword"
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/helk-logstash/output_templates/60-powershell-direct-template.json b/helk-logstash/output_templates/60-powershell-direct-template.json
@@ -0,0 +1,12 @@
+{
+  "order": 60,
+  "index_patterns" : "logs-endpoint-powershell-direct-*",
+  "version": 2018080101,
+  "mappings":{
+    "_doc":{
+      "properties":{
+        "process_id":{"type":"integer"}
+      }
+    }
+  }
+}
diff --git a/helk-logstash/output_templates/60-winevent-application-template.json b/helk-logstash/output_templates/60-winevent-application-template.json
@@ -0,0 +1,12 @@
+{
+  "order": 60,
+  "index_patterns": [ "logs-endpoint-winevent-application-*" ],
+  "version": 2018080101,
+  "mappings":{
+    "_doc":{
+      "properties":{
+        "spp_restart_scheduled":{"type":"date"}
+      }
+    }
+  }
+}