-
Notifications
You must be signed in to change notification settings - Fork 1
/
6100-httpd.conf
139 lines (119 loc) · 8.32 KB
/
6100-httpd.conf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
# SOF-ELK® Configuration File
# (C)2018 Lewes Technology Consulting, LLC
#
# This file contains filters, transforms, and enrichments for HTTPD access log messages
filter {
if [type] == "httpdlog" {
# this should only occur with syslog-based source records
if [logsource] {
mutate {
add_field => { "path" => "HTTPD log from %{logsource}" }
# the syslog handler populates this with the syslog-based date. we don't want that, though - we'll replace with the date parsed from the http log entry below
remove_field => [ "timestamp" ]
}
} else if "filebeat" in [tags] {
mutate {
add_field => { "path" => "filebeat: %{[host][name]}:%{source}" }
}
} else {
mutate {
add_field => { "path" => "unknown source" }
}
}
grok {
patterns_dir => [ "/usr/local/sof-elk/grok-patterns" ]
match => { "message" =>
[
# syslog-based entries, as used on my own servers (HTTP access log pushed via syslog messages)
"%{SYSLOGTIMESTAMP} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: %{HOSTNAME:hostname} %{COMBINEDPROXYLOG_CUSTOM}",
"%{SYSLOGTIMESTAMP} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: %{HOSTNAME:hostname} %{COMMONPROXYLOG_CUSTOM}",
"%{SYSLOGTIMESTAMP} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: %{HOSTNAME:hostname} %{COMBINEDAPACHELOG_CUSTOM}",
"%{SYSLOGTIMESTAMP} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: %{HOSTNAME:hostname} %{COMMONAPACHELOG_CUSTOM}",
# live syslog-based data, partially parsed via the syslog{} input selector
"%{HOSTNAME:hostname} %{COMBINEDPROXYLOG_CUSTOM}",
"%{HOSTNAME:hostname} %{COMMONPROXYLOG_CUSTOM}",
"%{HOSTNAME:hostname} %{COMBINEDAPACHELOG_CUSTOM}",
"%{HOSTNAME:hostname} %{COMMONAPACHELOG_CUSTOM}",
# straight-out NCSA combined/common formats. combined has to go first, since common will match a combined entry as well - and we break on first match
"%{COMBINEDPROXYLOG_CUSTOM}",
"%{COMMONPROXYLOG_CUSTOM}",
"%{COMBINEDAPACHELOG_CUSTOM}",
"%{COMMONAPACHELOG_CUSTOM}",
# ssl_access_log format
"\[%{HTTPDATE:timestamp}\] %{IPORHOST:source_ip} %{WORD:ssl_proto} %{NOTSPACE:ssl_cipher} \"(?:%{WORD:request_method} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" (?:%{NUMBER:destination_bytes}|-)",
# IIS Logs, with help from http://logz.io/blog/iis-log-analyzer/
# 2015-12-08 06:41:42 GET /handler/someservice.ashx someData=foobar 80 10.223.22.122 HTTP/1.1 Mozilla/5.0+(Windows+NT+6.1;+WOW64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/46.0.2490.86+Safari/537.36 1.2.1005047168.1446986881 http://www.logz.io/testing.aspx www.logz.io 200 638 795 0
"%{TIMESTAMP_ISO8601:timestamp} %{WORD:request_method} %{URIPATH:request} %{NOTSPACE:query_string} %{NUMBER:destination_port} %{IPORHOST:source_ip} HTTP/%{NUMBER:httpversion} %{NOTSPACE:agent} %{NOTSPACE:cookie} %{NOTSPACE:referrer} %{NOTSPACE:hostname} %{NUMBER:response_code} %{NUMBER:destination_bytes} %{NUMBER:source_bytes} %{NUMBER:response_time}",
"%{TIMESTAMP_ISO8601:timestamp} %{IPORHOST:destination_ip} %{WORD:request_method} %{URIPATH:request} %{NOTSPACE:query_string} %{NUMBER:destination_port} %{USER:ident} %{IPORHOST:source_ip} %{NOTSPACE:agent} %{NUMBER:response_code} %{NUMBER:response_sub} %{NUMBER:win_status} %{NUMBER:response_time}",
# 2002-05-24 20:18:01 172.224.24.114 - 206.73.118.24 80 GET /Default.htm - 200 7930 248 31 Mozilla/4.0+(compatible;+MSIE+5.01;+Windows+2000+Server) http://64.224.24.114/
"%{TIMESTAMP_ISO8601:timestamp} %{IPORHOST:source_ip} %{USER:ident} %{IPORHOST:destination_ip} %{NUMBER:destination_port} %{WORD:request_method} %{URIPATH:request} %{NOTSPACE:query_string} %{NUMBER:response_code} %{NUMBER:destination_bytes} %{NUMBER:source_bytes} %{NUMBER:response_time} %{NOTSPACE:agent} %{NOTSPACE:referrer}",
"%{TIMESTAMP_ISO8601:timestamp} %{IPORHOST:destination_ip} %{WORD:request_method} %{URIPATH:request} %{NOTSPACE:query_string} %{NUMBER:destination_port} %{USER:ident} %{IPORHOST:source_ip} %{NOTSPACE:agent} %{NOTSPACE:referrer} %{NUMBER:response_code} %{NUMBER:response_sub} %{NUMBER:win_status} %{NUMBER:response_time}",
"%{TIMESTAMP_ISO8601:timestamp} %{IPORHOST:source_ip} %{WORD:request_method} %{NOTSPACE:request} %{NOTSPACE:query_string} %{NUMBER:destination_port} %{NOTSPACE:ident} %{IPORHOST:destination_ip} %{NOTSPACE:agent} %{NUMBER:response_code} %{NUMBER:response_sub} %{NUMBER:win_status} %{NUMBER:response_time}",
"%{TIMESTAMP_ISO8601:timestamp} %{IPORHOST:source_ip} %{WORD:request_method} %{URIPATH:request} %{NOTSPACE:query_string} %{NUMBER:destination_port} %{NOTSPACE:ident} %{IPORHOST:destination_ip} %{NOTSPACE:agent} %{NUMBER:response_code} %{NUMBER:response_sub} %{NUMBER:win_status} %{NUMBER:response_time}",
"%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:hostname} %{IPORHOST:source_ip} %{WORD:request_method} %{URIPATH:request} %{NOTSPACE:query_string} %{NUMBER:destination_port} %{NOTSPACE:ident} %{IPORHOST:destination_ip} %{NOTSPACE:agent} %{NOTSPACE:cookie} %{NOTSPACE:referrer} %{NUMBER:response_code} %{NUMBER:response_sub} %{NUMBER:win_status} %{NUMBER:source_bytes} %{NUMBER:destination_bytes} %{NUMBER:response_time}",
"%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:hostname} %{IPORHOST:destination_ip} %{WORD:request_method} %{URIPATH:request} %{NOTSPACE:query_string} %{NUMBER:destination_port} %{USER:ident} %{IPORHOST:destination_ip} %{NOTSPACE:agent} %{NOTSPACE:cookie} %{NOTSPACE:referrer} %{NUMBER:response_code} %{NUMBER:response_sub} %{NUMBER:win_status} %{NUMBER:source_bytes} %{NUMBER:destination_bytes} %{NUMBER:response_time}",
"%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:hostname} %{WORD:request_method} %{URIPATH:request} %{NOTSPACE:query_string} %{NUMBER:destination_port} %{USER:ident} %{IPORHOST:source_ip} %{NOTSPACE:agent} %{NOTSPACE:cookie} %{NOTSPACE:referrer} %{NOTSPACE:hostname} %{NUMBER:response_code} %{NUMBER:response_sub} %{NUMBER:win_status} %{NUMBER:destination_bytes} %{NUMBER:source_bytes} %{NUMBER:response_time}"
]
}
add_tag => [ "parse_done" ]
tag_on_failure => [ "_gpfail", "fail_ht01" ]
}
# if nothing above matched, we might have a corrupt or partial record. let's try to squeak out whatever fields we can
if !( "parse_done" in [tags]) {
grok {
patterns_dir => [ "/usr/local/sof-elk/grok-patterns" ]
match => { "message" =>
[
# match first few fields from the syslog-based format
"%{SYSLOGTIMESTAMP} %{SYSLOGHOST} %{SYSLOGPROG}: %{HOSTNAME:hostname} %{IPORHOST:source_ip} %{USER:ident} %{NOTSPACE:username} \[%{HTTPDATE:timestamp}\]",
# client IP address is first entry in NCSA combined/common formats
"^%{IPORHOST:source_ip} %{USER:ident} %{NOTSPACE:username} \[%{HTTPDATE:timestamp}\]",
# first few fields from ssl_access_log format
"^\[%{HTTPDATE:timestamp}\] %{IPORHOST:source_ip}",
# find ANY httplog-formatted date string
"\[%{HTTPDATE:timestamp}\]"
]
}
add_tag => [ "partial_httpd_log", "parse_done" ]
tag_on_failure => [ "_gpfail", "fail_ht02" ]
}
}
# re-assemble an entire URL if all the components are there
if [protocol] and [hostname] and [request] {
if [destination_port] {
mutate {
add_field => { "request_url" => "%{protocol}://%{hostname}:%{destination_port}%{request}" }
}
} else {
mutate {
add_field => { "request_url" => "%{protocol}://%{hostname}%{request}" }
}
}
if [query_string] {
mutate {
replace => { "request_url" => "%{request_url}?%{query_string}" }
}
}
}
if "[timestamp]" {
date {
match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
remove_field => [ "timestamp" ]
}
date {
match => [ "timestamp", "YYYY-MM-dd HH:mm:ss" ] #added for ISS logs
remove_field => [ "timestamp" ]
}
}
if [program] {
mutate {
remove_field => [ "program" ]
}
}
if "parse_done" in [tags] {
mutate {
remove_field => [ "message" ]
}
}
}
}