Skip to content

Commit

Permalink
Changed output format
Browse files Browse the repository at this point in the history
  • Loading branch information
@catttam
catttam committed Oct 17, 2024
1 parent 110de70 commit 4a34185
Show file tree
Hide file tree
Showing 2 changed files with 74 additions and 48 deletions.
12 changes: 8 additions & 4 deletions extract_logs_metrics.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,10 @@ metrics(){
echo "[*] Warning: Couldn't process file $LOG_FILE for status code '$code'"
else
if [ $init == 't' ]; then
python3 logs_metric_parser.py -f "${out}_f${code}.json" -p $code
python3 logs_metric_parser.py -f "${out}_f${code}.json" -a $code
init="f"
else
python3 logs_metric_parser.py -f "${out}_f${code}.json" -p $code -u
python3 logs_metric_parser.py -f "${out}_f${code}.json" -a $code -u
fi
fi
fi
Expand Down Expand Up @@ -93,14 +93,18 @@ done
# Generate the html file
if [ ! -f "${HISTORY_LOGS_INFERENCE}" ] || [ ! -s "${HISTORY_LOGS_INFERENCE}" ]; then
goaccess "${LATEST_LOGS_INFERENCE}" --log-format="${LOG_FORMAT}" -o "/app/metrics/dashboard.html"
python3 logs_metric_parser.py -f $LATEST_LOGS_INFERENCE -i 0
else
metrics $HISTORY_LOGS_INFERENCE

cat $LATEST_LOGS_INFERENCE | tee -a $HISTORY_LOGS_INFERENCE >/dev/null
python3 logs_metric_parser.py -f $HISTORY_LOGS_INFERENCE -i 0 -u
goaccess "${HISTORY_LOGS_INFERENCE}" --log-format="${LOG_FORMAT}" -o "/app/metrics/dashboard.html"

fi

if [ ! -f "${HISTORY_LOGS_CREATE}" ] || [ ! -s "${HISTORY_LOGS_CREATE}" ]; then
python3 logs_metric_parser.py -f $LATEST_LOGS_CREATE -c
python3 logs_metric_parser.py -f $LATEST_LOGS_CREATE -c 0
else
python3 logs_metric_parser.py -f $HISTORY_LOGS_CREATE -c -u
python3 logs_metric_parser.py -f $HISTORY_LOGS_CREATE -c 0 -u
fi
110 changes: 66 additions & 44 deletions logs_metric_parser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,11 @@

parser = argparse.ArgumentParser(description="Command-line to retreive Prometheus metrics from OSCAR", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-f", "--file_path", type=str, help="Logfile path/name")
parser.add_argument("-g", "--general", action="store_true", help="Complete logfile")
parser.add_argument("-g", "--geolocation", action="store_true", help="Using as input complete generated GoAccess json")
parser.add_argument("-u", "--use_existing", action="store_true", required=False, help="Use existing output file")
parser.add_argument("-p", "--partial", action="store_true", help="Filtered by status code logfile")
parser.add_argument("-i", "--inference", action="store_true", help="Using as input inference log format file")
parser.add_argument("-c", "--create", action="store_true", help="Using as input created services log format file")
parser.add_argument("-a", "--goaccess", action="store_true", help="Using as input generated GoAccess json filtered")
parser.add_argument("status_code", type=int, help="Complete logfile")


Expand Down Expand Up @@ -56,49 +57,67 @@ def parse_geolocation_info(write_type):
gfile.close()

"""
> Number of AI applications (created services -> POST requests to /system/services)
> Output format: {num_created, start_date, end_date}
> Processed inference executions (POST requests to /run or /job)
> Output format: {service, executions, type, successfull, failed, start_date, end_date}
> Output format: {service_name, status_code, total_visits, unique_visits, start_date, end_date}
"""

def parse_inference_info(status_code, write_type):

inference = dict()
def parse_inference_goaccess(status_code, write_type):
requests = metrics["requests"]["data"]
exec_count = 0

for r in requests:
if r["method"] == "POST":

with open(f'{OUTPUT_PATH}/total_inference_metrics.csv', write_type, newline='') as efile:
writer = csv.writer(efile)
if write_type == "w": writer.writerow(["service_name", "status_code", "total_visits", "unique_visits", "start_metric_date", "end_metric_date"])
for r in requests:
path = r["data"]
if RUN_PATH in path or JOB_PATH in path:
sum_requests = r["hits"]["count"]
total_visits = r["hits"]["count"]
unique_visits = r["visitors"]["count"]
split_path = split(path)
service = split_path[1]
if service in inference.keys():
inference[service].append({"exec_type": split_path[0], "status_code": status_code, "count": sum_requests})
else:
inference[service] = [{"exec_type": split_path[0], "status_code": status_code, "count": sum_requests}]
exec_count+=sum_requests

if exec_count != 0:
with open(f'{OUTPUT_PATH}/total_inference_metrics.csv', write_type, newline='') as efile:
writer = csv.writer(efile)
if write_type == "w": writer.writerow(["inference_count", "status_code", "start_metric_date", "end_metric_date"])
writer.writerow([exec_count, status_code, START_DATE, END_DATE])

with open(f'{OUTPUT_PATH}/services_inference_metrics.csv', write_type, newline='') as sfile:
writer = csv.writer(sfile)
if write_type == "w": writer.writerow(["service_name", "exec_type", "status_code", "inference_count" , "start_metric_date", "end_metric_date"])
for k in inference.keys():
for item in inference[k]:
writer.writerow([k, item["exec_type"], item["status_code"], item["count"], START_DATE, END_DATE])

def parse_create_info(write_type):
service_name = split_path[1]

writer.writerow([service_name, status_code, total_visits, unique_visits, START_DATE, END_DATE])

"""
> Info about all inference executions
> Output format: {service_name, exec_type, status_code, owner_uid, request_date, request_time}
"""
def parse_inference_log(write_type):
with open(f'{OUTPUT_PATH}/services_inference_metrics.csv', write_type, newline='') as sfile:
writer = csv.writer(cfile)
if write_type == "w": write.writerow(["service_name", "exec_type", "status_code", "owner_uid", "request_date", "request_time"])
with open(args.file_path, 'r') as rawfile:
for log in rawfile:
pattern = re.compile(
r'\[GIN-EXECUTIONS-LOGGER\]\s' # Literal text "[GIN-EXECUTIONS-LOGGER]"
r'(?P<date>\d{4}/\d{2}/\d{2})\s-\s(?P<time>\d{2}:\d{2}:\d{2})\s\|\s' # Date and time
r'(?P<status>\d{3})\s\|\s' # HTTP status code
r'(?P<latency>[\d.]+s)\s\|\s' # Latency (time taken)
r'(?P<client_ip>\d{1,3}(?:\.\d{1,3}){3})\s\|\s' # Client IP
r'(?P<method>[A-Z]+)\s+\|\s' # HTTP method (e.g., POST)
r'(?P<path>\S+)\s\|\s' # Request path
r'(?P<user>\S+)' # User (e.g., <uid>@egi.eu)
)
match = pattern.search(log)
if match:
path = match.group('path')
split_path = split(path)
exec_type = split_path[0]
service_name = split_path[1]

status_code = match.group('status')
request_date = match.group('date')
request_time = match.group('time')
owner_uid = match.group('user')
writer.writerow([service_name, exec_type, status_code, owner_uid, request_date, request_time])

"""
> Number of AI applications (created services -> POST requests to /system/services)
> Output format: {service_name, owner_uid, creation_date, creation_time}
"""
def parse_create_log(write_type):
with open(f'{OUTPUT_PATH}/created_services_info.csv', write_type, newline='') as cfile:
writer = csv.writer(cfile)
if write_type == "w": writer.writerow(["service_name", "owner_uid", "creation_time"])
if write_type == "w": writer.writerow(["service_name", "owner_uid", "creation_date", "creation_time"])
with open(args.file_path, 'r') as rawfile:
for log in rawfile:
pattern = re.compile(
Expand All @@ -113,18 +132,21 @@ def parse_create_info(write_type):
)
match = pattern.search(log)
if match:
creation_time = match.group(date)
service_name = match.group(service_name)
owner_uid = match.group(user)
writer.writerow([service_name, owner_uid, creation_time])
creation_date = match.group('date')
creation_time = match.group('time')
service_name = match.group('service_name')
owner_uid = match.group('user')
writer.writerow([service_name, owner_uid, creation_date, creation_time])

wr="w"
if args.use_existing:
wr="a"

if args.create:
parse_create_info(wr)
if args.general:
parse_create_log(wr)
if args.inference:
parse_inference_log(wr)
if args.geolocation:
parse_geolocation_info(wr)
if args.partial:
parse_inference_info(args.status_code, wr)
if args.goaccess:
parse_inference_goaccess(args.status_code, wr)

0 comments on commit 4a34185

Please sign in to comment.