Merge pull request #166 from JeffersonLab/cache_small_file_scripts_MMI

Scripts to produce reports on small files on cache.

cache_small/cache_small_file_report.html

@@ -0,0 +1 @@
+<TABLE BORDER=1><TR><TH>count_millions</TH><TH>user</TH></TR><TR><TD>6.6876</TD><TD>tbritton</TD></TR><TR><TD>3.9519</TD><TD>gxproj3</TD></TR><TR><TD>1.4424</TD><TD>ijaegle</TD></TR><TR><TD>0.6272</TD><TD>gxproj2</TD></TR><TR><TD>0.5766</TD><TD>gxproj1</TD></TR><TR><TD>0.1777</TD><TD>jzarling</TD></TR><TR><TD>0.1764</TD><TD>gxproj5</TD></TR><TR><TD>0.1738</TD><TD>gxproj4</TD></TR><TR><TD>0.0786</TD><TD>nwickjlb</TD></TR><TR><TD>0.0779</TD><TD>scole</TD></TR><TR><TD>0.0396</TD><TD>acernst</TD></TR><TR><TD>0.0151</TD><TD>billlee</TD></TR><TR><TD>0.0119</TD><TD>davidl</TD></TR><TR><TD>0.0082</TD><TD>gxproj6</TD></TR><TR><TD>0.0060</TD><TD>mkamel</TD></TR><TR><TD>0.0052</TD><TD>sdobbs</TD></TR><TR><TD>0.0038</TD><TD>acschick</TD></TR><TR><TD>0.0034</TD><TD>halldata</TD></TR><TR><TD>0.0012</TD><TD>gxproj9</TD></TR><TR><TD>0.0003</TD><TD>gluex</TD></TR></TABLE>

cache_small/find_command.txt

@@ -0,0 +1 @@
+[marki@ifarm1901 ~]$ find /cache/halld -type f -size -1048576c -exec stat -c "%n %U %X %Y %s" {} \; > /u/scratch/marki/cache_small_files.txt

cache_small/load_small_file_info.sql

@@ -0,0 +1,30 @@
+/**********************************************************************
+
+Grants:
+
+grant all on cacheInfo.* to marki@localhost identified by '*****';
+grant file on *.* to marki@localhost;
+
+Invocation:
+
+sed 's/\(.*\)\//\1 /' cache_small_files.txt > csf.tmp
+mysql -umarki -p cacheInfo < load_small_file_info.sql
+
+Example query:
+
+select count(*) as c, user from smallFile group by user order by c desc;
+
+**********************************************************************/
+
+drop table if exists smallFile;
+create table smallFile (
+  dir varchar(1024),
+  name varchar(1024),
+  user varchar(32),
+  atime int,
+  mtime int,
+  size int
+);
+load data local infile
+  './csf.tmp' into table smallFile
+  fields terminated by ' ';

cache_small/make_top.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+rm -rf small_files
+mkdir small_files
+cp -v button_red.png button_blue.png small_files
+cd small_files
+rm -f top_level.tmp
+mysql --silent -umarki -phybrid cacheInfo -e 'select count(*) as count, user from smallFile group by user order by count desc;' > top_level.tmp
+rm -f index.html
+echo \<h1\>Small Files on the Cache Disk\</h1\> > index.html
+echo `date` >> index.html
+echo \<table border\> >> index.html
+echo \<tr\>\<th\>count\</th\>\<th\>user\</th\>\<th\>directories by name\</th\>\<th\>directories by count\</th\>\</tr\> >> index.html
+awk '{print "<tr><td>"$1"</td><td>"$2"</td><td style=\"text-align:center\"><a href=\""$2"_by_dir.html\"><img src=\"button_red.png\"/></a></td><td style=\"text-align:center\"><a href=\""$2"_by_count.html\"><img src=\"button_blue.png\"/></a></td>"}' < top_level.tmp >> index.html
+echo \</table\> >> index.html
+rm -f users.tmp
+mysql --silent -umarki -phybrid cacheInfo -e 'select unique user from smallFile;' > users.tmp
+while read u; do
+    echo "$u"
+    rm -f sql.tmp
+    echo select dir, count\(*\) as count, name as example_file from smallFile where user = \'$u\' group by dir order by dir\; > sql.tmp
+    mysql --html -umarki -phybrid cacheInfo < sql.tmp > ${u}_by_dir.html
+    rm -f sql.tmp
+    echo select dir, count\(*\) as count, name as example_file from smallFile where user = \'$u\' group by dir order by count desc, dir\; > sql.tmp
+    mysql --html -umarki -phybrid cacheInfo < sql.tmp > ${u}_by_count.html
+done < users.tmp