metrics2csv

#!/bin/bash
# Usage:
#
#   metrics2csv test server
#
# Or recursive re-processing of an entire directory tree instead:
#
#   metrics2csv recurse

source ./config
TEST=$1
RESULTPSQL="psql -h $RESULTHOST -U $RESULTUSER -p $RESULTPORT -d $RESULTDB"

# Scaffolding for re-testing the import code
function clear {
$RESULTPSQL -c "TRUNCATE TABLE test_metrics_data;"
}

function show {
$RESULTPSQL -c "SELECT metric,collected,value FROM test_metrics_data WHERE server='${SERVERNAME}' and test=${TEST} ORDER BY collected,metric LIMIT 10"
}

function process {
# In theory I could rely on existing environment variables to figure these both
# out.  In practice that's already caused one bug, so parameters for these two.
TESTNUM="$1"
H="$2"

if [ ! -f "results.txt" ] ; then
  echo Not a test result directory, skipping
  return 1
fi

if [ -s "meminfo.log" ] ; then
    # Linux
    echo Processing Linux formatted log files server ${H} test $TESTNUM
    ../../../csv2gnuplot -i iostat.log -d iostat -s "Device" -t "pgbench disk" --disks=${DISKLIST} -o iostat
    ../../../csv2gnuplot -i vmstat.log -d vmstat -s "cache" -t "pgbench vmstat" -o vmstat
    ../../../dirty-plot < meminfo.log > dirtydata.txt
else
    # Assume Darwin for now, eventually others?
    echo Processing Darwin formatted log files server $H test $TESTNUM
    ../../../csv2gnuplot -i iostat.log -d iostat -s "KB/t" -t "pgbench disk" -o iostat
    ../../../csv2gnuplot -i vmstat.log -d vmstat -s "wired" -t "pgbench vmstat" -o vmstat
fi

$RESULTPSQL -f ../../../export-latency-metric.sql

$RESULTPSQL -c "TRUNCATE TABLE tmp_metric_import;"
$RESULTPSQL -c "\copy tmp_metric_import FROM 'psql.log' WITH DELIMITER '|'"
$RESULTPSQL -c "\copy tmp_metric_import FROM 'latency_metric.csv' WITH CSV HEADER"
$RESULTPSQL -c "\copy tmp_metric_import FROM 'vmstat.csv' WITH CSV HEADER"
$RESULTPSQL -c "\copy tmp_metric_import FROM 'iostat.csv' WITH CSV HEADER"
if [ -f "meminfo.csv" ] ; then
  $RESULTPSQL -c "\copy tmp_metric_import FROM 'meminfo.csv' WITH CSV HEADER"
fi

$RESULTPSQL -c "DELETE FROM test_metrics_data WHERE test=${TESTNUM} AND server='${H}'"
$RESULTPSQL -e -c "INSERT INTO test_metrics_data (collected,value,metric,test,server) SELECT collected,value,metric,${TESTNUM},'${H}'  FROM tmp_metric_import;"

$RESULTPSQL -e -c "
WITH S AS (
  SELECT test,server,date_trunc('second',ts) AS collected,
  round(min(schedule_lag)::numeric / 1000,3) AS min_schedule_lag_ms,
  round(avg(schedule_lag)::numeric / 1000,3) AS avg_schedule_lag_ms,
  round(max(schedule_lag)::numeric / 1000,3) AS max_schedule_lag_ms
  FROM timing GROUP BY test,server,date_trunc('second',ts)),
L AS (
  SELECT test,server,collected,min_schedule_lag_ms AS lag_val,'min_schedule_lag_ms' AS metric FROM S UNION
  SELECT test,server,collected,max_schedule_lag_ms,'max_schedule_lag_ms' FROM S UNION
  SELECT test,server,collected,avg_schedule_lag_ms,'avg_schedule_lag_ms' FROM S)
INSERT INTO test_metrics_data (collected,value,metric,test,server)
  SELECT collected,lag_val,metric,test,server FROM l;"
}

#
# Recurse import an entire directory tree
#
function reprocess {
shopt -s nullglob
pushd results

for serv in *
do
  if [ -d $serv ] ; then
    pushd $serv
    echo "Server directory $serv"

	for t in *
	do
	  if [ -d $t ] ; then
	    pushd $t
	    echo "Result directory $d/$f"
	    process $t $serv
        popd
	  fi
    done

    popd
  fi
done
}

#
# Main
#

if [ "$1" = "recurse" ] ; then
  echo "Recursion mode"
  reprocess
elif [ -n "$2" ] ; then
  # Try to handle being called from test results directory or from
  # toolkit base directory
  if [ ! -f "iostat.log" ] ; then
    pushd results/$2/$1
  fi
  process $1 $2
else
    echo "Usage:  metrics2csv test server."
    echo "Or use 'recurse' from the code directory for automatic mode."
fi