-
Notifications
You must be signed in to change notification settings - Fork 11
/
icecastlogparser.py
executable file
·145 lines (122 loc) · 5.55 KB
/
icecastlogparser.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
#!/usr/bin/python
# icecastServerLogParser.py
# Alejandro Ferrari <[email protected]>
# version 1.0
"""
Parser for ICECAST server log output, of the form:
190.49.XX.XX - - [25/Jun/2012:04:50:59 -0300]
"GET /Retromix_64.mp3?1340608279543.mp3 HTTP/1.1" 200 19143936
"http://player.domain.com/player/Flash/"
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.5 (KH" 2385
You can then break it up as follows:
IP ADDRESS - -
Server Date / Time [SPACE]
"GET /path/to/stream
HTTP/Type Request"
Success Code
Bytes Sent To Client
Referer
Clien Software/User-Agent
Session Duration Time
"""
from pyparsing import alphas,nums, dblQuotedString, Combine, Word, Group, delimitedList, Suppress, removeQuotes
import string
import glob
import sys
import MySQLdb
import pygeoip
import time
import re
from datetime import datetime
from datetime import timedelta
from socket import gethostname
#################################################
# Configurations
#################################################
# Server Name for identify where the Hit was
server_name = gethostname().lower()
# glob supports Unix style pathname extensions
# Here need to put the Access log file name you need parse
python_files = glob.glob("/var/log/icecast/old/access.log.20150326_180409")
# Put the correct path to your .DAT GeoIP DB
gi = pygeoip.GeoIP('/usr/share/GeoIP/GeoIP.dat')
gic = pygeoip.GeoIP('/usr/share/GeoIP/GeoLiteCity.dat')
# DB Params
db_host = "190.228.29.57"
db_user = "fm8990stats"
db_passwd = "fm8990st4ts"
db_name = "fm8990stats"
# Filters (Skip this lines if match, using regex)
filter_ip = r'54.146.35|10.10'
# Number of inserts per query
HIST_PER_QUERY = 100
#################################################
# Dont modify below this line
#################################################
try:
conn = MySQLdb.connect (host = db_host, user = db_user, passwd = db_passwd, db = db_name)
except MySQLdb.Error, e:
print "Error %d: %s" % (e.args[0], e.args[1])
sys.exit (1)
def getCmdFields( s, l, t ):
t["method"],t["requestURI"],t["protocolVersion"] = t[0].strip('"').split()
logLineBNF = None
def getLogLineBNF():
global logLineBNF
if logLineBNF is None:
integer = Word( nums )
ipAddress = delimitedList( integer, ".", combine=True )
timeZoneOffset = Word("+-",nums)
month = Word(string.uppercase, string.lowercase, exact=3)
serverDateTime = Group( Suppress("[") + Combine( integer + "/" + month + "/" + integer + ":" + integer + ":" + integer + ":" + integer ) + timeZoneOffset + Suppress("]") )
logLineBNF = ( ipAddress.setResultsName("ipAddr") +
Suppress("-") +
("-" | Word( alphas+nums+"@._" )).setResultsName("auth") +
serverDateTime.setResultsName("timestamp") +
dblQuotedString.setResultsName("cmd").setParseAction(getCmdFields) +
(integer | "-").setResultsName("statusCode") +
(integer | "-").setResultsName("numBytesSent") +
dblQuotedString.setResultsName("referer").setParseAction(removeQuotes) +
dblQuotedString.setResultsName("userAgent").setParseAction(removeQuotes) +
(integer | "-").setResultsName("numDurationTime"))
return logLineBNF
# Variable definition
hits_counter = 0
query = ""
values_to_insert = []
for file_name in sorted(python_files):
with open(file_name) as f:
for line in f:
if not line: continue
fields = getLogLineBNF().parseString(line)
countryCode = gi.country_code_by_addr(fields.ipAddr)
streamName = fields.requestURI.strip('/').split('?')
if re.match(filter_ip, fields.ipAddr, flags=0):
continue
else:
datetime_end = datetime.strptime(fields.timestamp[0],"%d/%b/%Y:%H:%M:%S")
datetime_start = datetime_end - timedelta(seconds=int(fields.numDurationTime))
if hits_counter == HIST_PER_QUERY:
# prepare a cursor object using cursor() method
cursor = conn.cursor()
try:
# Execute the SQL command
query = "INSERT INTO icecast_logs (datetime_start, datetime_end, ip, country_code, mount, status_code, duration, sent_bytes, agent, referer, server, user, pass) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
cursor.executemany(query, values_to_insert)
# Commit your changes in the database
conn.commit()
values_to_insert = []
except MySQLdb.Error, e:
# Rollback in case there is any error
print "An error has been passed. %s" % e
conn.rollback()
cursor.close()
hits_counter = 0
query = ""
else:
#query = query + "INSERT INTO icecast_logs (datetime_start, datetime_end, ip, country_code, mount, status_code, duration, sent_bytes, agent, referer, server, user, pass) \
# VALUES('{0}', '{1}', '{2}', '{3}', '{4}', {5}, {6}, {7}, '{8}', '{9}', '{10}', '{11}', '{12}');".format(datetime_start, datetime_end, fields.ipAddr, countryCode, streamName[0], fields.statusCode, fields.numDurationTime, fields.numBytesSent, fields.userAgent, fields.referer, server_name, fields.userName, fields.password)
values_to_insert.append((datetime_start, datetime_end, fields.ipAddr, countryCode, streamName[0], fields.statusCode, fields.numDurationTime, fields.numBytesSent, fields.userAgent, fields.referer, server_name, fields.userName, fields.password))
hits_counter+=1
# print query
conn.close ()