update workload module #86

dbiir · Jan 18, 2018 · eccd603 · eccd603
1 parent c98a254
commit eccd603
Show file tree

Hide file tree

Showing 4 changed files with 115 additions and 6 deletions.
diff --git a/rainbow-workload/README.md b/rainbow-workload/README.md
@@ -0,0 +1,8 @@
+# Rainbow workload
+
+Rainbow Workload.
+
+## Intro
+- LRU
+- Workload Evaluation Client
+- HttpGet
diff --git a/rainbow-workload/src/main/java/cn/edu/ruc/iir/rainbow/workload/cmd/CmdWorkloadEva.java b/rainbow-workload/src/main/java/cn/edu/ruc/iir/rainbow/workload/cmd/CmdWorkloadEva.java
@@ -1,5 +1,6 @@
 package cn.edu.ruc.iir.rainbow.workload.cmd;
 
+import cn.edu.ruc.iir.rainbow.benchmark.util.DateUtil;
 import cn.edu.ruc.iir.rainbow.common.cmd.Command;
 import cn.edu.ruc.iir.rainbow.common.cmd.Receiver;
 import cn.edu.ruc.iir.rainbow.common.exception.ExceptionHandler;
@@ -10,6 +11,7 @@
 
 import java.io.BufferedReader;
 import java.io.FileReader;
+import java.util.Date;
 import java.util.Properties;
 
 /**
@@ -38,6 +40,7 @@ public void execute(Properties params) {
 
             // begin evaluate
             String line;
+            int i = 0;
             while ((line = workloadReader.readLine()) != null) {
                 String columns = line.split("\t")[2];
                 String queryId = line.split("\t")[0];
@@ -51,6 +54,11 @@ public void execute(Properties params) {
                 }
                 PrestoEvaluator instance = PrestoEvaluator.Instance();
                 instance.execute(tableName, columns, orderByColumn);
+                i++;
+                if (i % 300 == 0 || i % 1000 == 0) {
+                    System.out.print(DateUtil.formatTime(new Date()) + "\t");
+                    System.out.println("line: " + i);
+                }
             }
         } catch (Exception e) {
             System.out.println("CmdWorkloadEva Error. \n" + e.getMessage());

diff --git a/rainbow-workload/src/main/java/cn/edu/ruc/iir/rainbow/workload/eva/PrestoEvaluator.java b/rainbow-workload/src/main/java/cn/edu/ruc/iir/rainbow/workload/eva/PrestoEvaluator.java
@@ -44,7 +44,11 @@ public static PrestoEvaluator Instance() {
     }
 
     public void execute(String tableName, String columns, String orderByColumn) {
-
+        try {
+            Thread.sleep(5000);
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
         String sql = "";
         try {
             Statement statement = connection.createStatement();
@@ -54,11 +58,6 @@ public void execute(String tableName, String columns, String orderByColumn) {
                 sql = "select " + columns + " from " + tableName + " order by " + orderByColumn + " limit 10";
             }
             ResultSet resultSet = statement.executeQuery(sql);
-            try {
-                Thread.sleep(3000);
-            } catch (InterruptedException e) {
-                e.printStackTrace();
-            }
             resultSet.next();
             resultSet.close();
             statement.close();

diff --git a/rainbow-workload/src/test/java/cn/edu/ruc/iir/rainbow/workload/util/HttpUtilTest.java b/rainbow-workload/src/test/java/cn/edu/ruc/iir/rainbow/workload/util/HttpUtilTest.java
@@ -10,11 +10,13 @@
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
+import org.apache.commons.collections.map.HashedMap;
 import org.junit.jupiter.api.Test;
 
 import java.io.IOException;
 import java.util.Date;
 import java.util.List;
+import java.util.Map;
 import java.util.Random;
 
 /**
@@ -30,6 +32,98 @@ public class HttpUtilTest {
     int count = 0;
     private boolean flag = false;
 
+    Map<String, Boolean> queryMap = new HashedMap();
+
+    @Test
+    public void LatestTest() {
+        SqlParser parser = new SqlParser();
+        Query q = null;
+        Object o = new Object();
+        JSONArray jsonArray = null;
+
+        int num = 0, cou = 0;
+        while (true) {
+            try {
+                o = HttpUtil.HttpGet(Settings.PRESTO_QUERY);
+            } catch (Exception e) {
+                ExceptionHandler.Instance().log(ExceptionType.ERROR, "http get error", e);
+            }
+            jsonArray = JSON.parseArray(o.toString());
+
+            String queryId = null;
+            String query = null;
+            for (int i = 0; i < jsonArray.size(); i++) {
+                JSONObject jsonObject = (JSONObject) jsonArray.get(i);
+                if (queryMap.get("queryId") != null && !queryMap.get("queryId") && jsonObject.size() == 8) {
+                    queryId = jsonObject.get("queryId").toString();
+                    queryMap.put("queryId", true);
+                    query = jsonObject.get("query").toString();
+//                    System.out.println(queryId + "\t" + i + "\t" + query);
+                    // Parser
+                    try {
+                        q = (Query) parser.createStatement(query);
+                    } catch (Exception e) {
+                        ExceptionHandler.Instance().log(ExceptionType.ERROR, "query error", e);
+                    }
+//                System.out.println(q.toString());
+                    QuerySpecification queryBody = (QuerySpecification) q.getQueryBody();
+                    // get columns
+                    List<SelectItem> selectItemList = queryBody.getSelect().getSelectItems();
+
+                    // tableName
+                    Table t = (Table) queryBody.getFrom().get();
+//                    System.out.println(t.getName());
+                    if (t.getName().toString().equals("text")) {
+                        System.out.println("Text visit: " + cou++);
+                        if (cou >= 4000) {
+                            System.out.println(queryMap.size());
+                        }
+                        int j = 0;
+                        Random random = new Random(System.currentTimeMillis());
+                        AccessPatternCache APC = new AccessPatternCache(100000, 0.1);
+                        String time = DateUtil.formatTime(new Date());
+                        System.out.println(time);
+                        try {
+                            if (!flag) {
+                                flag = true;
+                                FileUtil.writeFile(time + "\tBegin\t" + i + "\r\n", Settings.APC_PATH, true);
+                            }
+                        } catch (IOException e) {
+                            e.printStackTrace();
+                        }
+                        double weight = Double.parseDouble("1");
+                        AccessPattern pattern = new AccessPattern(queryId, weight);
+                        for (SelectItem column : selectItemList) {
+//                        System.out.println(scolumn.toString());
+                            pattern.addColumn(column.toString());
+                        }
+                        if (APC.cache(pattern)) {
+                            System.out.println(i + ", trigger layout optimization.");
+                            j++;
+                            APC.saveAsWorkloadFile("/home/tao/software/station/Workplace/workload_" + j + ".txt");
+                            try {
+                                flag = false;
+                                System.out.println(time);
+                                FileUtil.writeFile(time + "\tEnd\t" + i + "\r\n", Settings.APC_PATH, true);
+                            } catch (IOException e) {
+                                e.printStackTrace();
+                            }
+                        }
+                        try {
+                            Thread.sleep(random.nextInt(500));
+                        } catch (InterruptedException e) {
+                            e.printStackTrace();
+                        }
+                    }
+                }
+            }
+            // update count
+//            count = jsonArray.size();
+            num++;
+            o = new Object();
+        }
+    }
+
     @Test
     public void HttpGetTest() {
         SqlParser parser = new SqlParser();