Fixing UI for CSV Compatibility (#200)

* hacks to make it not dump error for CSV plot+explore. Doesn't filter correctly yet. * UI fixs Got rid of sort panel and sort by risk ratio * console plot + explore filter based on predicates * Adding confidence interval corrections * plot colors * outline boxes instead * color fix * colors, thresh, commas * back to grey * Risk ratio edge case fix * addressing comments
stanford-futuredata · Jul 15, 2017 · f41fc5c · f41fc5c
1 parent e879c5b
commit f41fc5c
Show file tree

Hide file tree

Showing 20 changed files with 335 additions and 101 deletions.
diff --git a/conf/batch.yaml b/conf/batch.yaml
@@ -9,7 +9,7 @@ macrobase.analysis.transformType: MAD_OR_MCD
 
 macrobase.loader.db.baseQuery: SELECT * FROM sensor_data_demo;
 
-macrobase.analysis.minSupport: 0.001
+macrobase.analysis.minSupport: 0.01
 macrobase.analysis.minOIRatio: 3.0
 
 macrobase.analysis.usePercentile: true

diff --git a/conf/macrobase.yaml b/conf/macrobase.yaml
@@ -9,5 +9,5 @@ logging:
   loggers:
     "macrobase": TRACE
 
-macrobase.analysis.minSupport: 0.001
+macrobase.analysis.minSupport: 0.01
 macrobase.analysis.minOIRatio: 1
diff --git a/frontend/src/main/java/macrobase/runtime/resources/BaseResource.java b/frontend/src/main/java/macrobase/runtime/resources/BaseResource.java
@@ -3,29 +3,32 @@
 import macrobase.conf.ConfigurationException;
 import macrobase.conf.MacroBaseConf;
 import macrobase.conf.MacroBaseDefaults;
-import macrobase.ingest.SQLIngester;
+import macrobase.ingest.DataIngester;
 
 import java.io.IOException;
 import java.sql.SQLException;
 import java.util.ArrayList;
 
 abstract public class BaseResource {
     protected final MacroBaseConf conf;
-    protected final String configuredIngester;
+    protected String configuredIngester;
 
     public BaseResource(MacroBaseConf conf) {
+
         this.conf = conf;
         configuredIngester = conf.getString(MacroBaseConf.DATA_LOADER_TYPE,
                                             MacroBaseDefaults.DATA_LOADER_TYPE.toString());
     }
 
-    protected SQLIngester getLoader() throws ConfigurationException, SQLException, IOException {
+    protected DataIngester getLoader() throws ConfigurationException, SQLException, IOException {
         // constructs ingester of type specified in conf file initially,
+        // ^ used to be initially, now it's just whatever it currently is to work for CSV
         // or the default ingester
         // by default, REST calls may not have these defined.
         conf.set(MacroBaseConf.DATA_LOADER_TYPE, configuredIngester);
         conf.set(MacroBaseConf.ATTRIBUTES, new ArrayList<>());
         conf.set(MacroBaseConf.METRICS, new ArrayList<>());
-        return (SQLIngester) conf.constructIngester();
+        return conf.constructIngester();
     }
+
 }
diff --git a/frontend/src/main/java/macrobase/runtime/resources/FormattedRowSetResource.java b/frontend/src/main/java/macrobase/runtime/resources/FormattedRowSetResource.java
@@ -2,6 +2,7 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import macrobase.conf.MacroBaseConf;
+import macrobase.ingest.SQLIngester;
 import macrobase.ingest.result.RowSet;
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVPrinter;
@@ -61,7 +62,7 @@ public FormattedRowSetResponse getRowsFormatted(RowSetRequest request) {
             request.columnValues.stream().forEach(a -> preds.put(a.column, a.value));
 
             if(request.returnType == RETURNTYPE.SQL) {
-                response.response = getLoader().getRowsSql(request.baseQuery,
+                response.response = ((SQLIngester) getLoader()).getRowsSql(request.baseQuery,
                                                            preds,
                                                            request.limit,
                                                            request.offset)+";";

diff --git a/frontend/src/main/java/macrobase/runtime/resources/MultipleRowSetResource.java b/frontend/src/main/java/macrobase/runtime/resources/MultipleRowSetResource.java
@@ -1,6 +1,9 @@
 package macrobase.runtime.resources;
 
 import macrobase.conf.MacroBaseConf;
+import macrobase.conf.MacroBaseDefaults;
+import macrobase.ingest.CSVIngester;
+import macrobase.ingest.DataIngester;
 import macrobase.ingest.result.RowSet;
 import macrobase.ingest.SQLIngester;
 
@@ -45,12 +48,16 @@ public MultipleRowSetResponse getRows(MultipleRowSetRequest request) {
 
         try {
             conf.set(MacroBaseConf.DB_URL, request.pgUrl);
-            SQLIngester loader = getLoader();
+
+            // Need a CSV ingester. Changing so it's not fixed during construction (used to be final, but changed as
+            // this is set in AnalyzeResource after construction)
+            configuredIngester = conf.getString(MacroBaseConf.DATA_LOADER_TYPE, MacroBaseDefaults.DATA_LOADER_TYPE.toString());
+
+            DataIngester loader = getLoader();
             List<RowSet> lr = new ArrayList<RowSet>();
             for (List<RowSetResource.RowSetRequest.RowRequestPair> columnValue : request.columnValues) {
                 HashMap<String, String> preds = new HashMap<>();
                 columnValue.stream().forEach(a -> preds.put(a.column, a.value));
-
                 lr.add(loader.getRows(request.baseQuery,
                                       preds,
                                       request.limit,

diff --git a/frontend/src/main/java/macrobase/runtime/resources/RowSetResource.java b/frontend/src/main/java/macrobase/runtime/resources/RowSetResource.java
@@ -1,6 +1,7 @@
 package macrobase.runtime.resources;
 
 import macrobase.conf.MacroBaseConf;
+import macrobase.conf.MacroBaseDefaults;
 import macrobase.ingest.result.RowSet;
 
 import org.apache.commons.lang3.exception.ExceptionUtils;
@@ -49,6 +50,7 @@ public RowSetResponse getRows(RowSetRequest request) {
         try {
             conf.set(MacroBaseConf.DB_URL, request.pgUrl);
             conf.set(MacroBaseConf.BASE_QUERY, request.baseQuery);
+            configuredIngester = conf.getString(MacroBaseConf.DATA_LOADER_TYPE, MacroBaseDefaults.DATA_LOADER_TYPE.toString());
 
             HashMap<String, String> preds = new HashMap<>();
             request.columnValues.stream().forEach(a -> preds.put(a.column, a.value));

diff --git a/frontend/src/main/java/macrobase/runtime/resources/SchemaResource.java b/frontend/src/main/java/macrobase/runtime/resources/SchemaResource.java
@@ -1,6 +1,7 @@
 package macrobase.runtime.resources;
 
 import macrobase.conf.MacroBaseConf;
+import macrobase.ingest.SQLIngester;
 import macrobase.ingest.result.Schema;
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVParser;
@@ -57,7 +58,7 @@ public SchemaResponse getSchema(SchemaRequest request) {
             } else {
                 conf.set(MacroBaseConf.DB_URL, request.pgUrl);
                 conf.set(MacroBaseConf.BASE_QUERY, request.baseQuery);
-                response.schema = getLoader().getSchema(request.baseQuery);
+                response.schema = ((SQLIngester) getLoader()).getSchema(request.baseQuery);
             }
         } catch (Exception e) {
             log.error("An error occurred while processing a request:", e);

diff --git a/frontend/src/main/resources/frontend/console.html b/frontend/src/main/resources/frontend/console.html
@@ -7,13 +7,7 @@
 
 <link rel="stylesheet" href="css/bootstrap.min.css">
 <link rel="stylesheet" href="css/bootstrap-theme.min.css">
-
-<style>
-    .active-schema {
-    background-color: #97e17a !important;
-    background-image: none !important;
-    }
-</style>
+<link rel="stylesheet" href="css/mb-frontend.css">
 
 <head>
     <title>MacroBase Console</title>
@@ -55,7 +49,7 @@ <h4><strong>An error occurred while processing your request:</strong></h4>
                 <div class="col-md-6">
 
                     <div ng-controller="connectController" ng-init="get_schema(true)">
-                        <div class="panel panel-default">
+                        <div class="panel panel-gray">
                             <div class="panel-heading">Database Configuration</div>
                             <div class="panel-body">
                                 <table>
@@ -80,13 +74,12 @@ <h4><strong>An error occurred while processing your request:</strong></h4>
                                         </td>
                                     </tr>
                                 </table>
-                                <p style="margin-top: 5px; margin-bottom:-5px">Connected to {{pg_url}} database!</p>
 
                             </div>
                         </div>
                         <br>
 
-                        <div class="panel panel-default">
+                        <div class="panel panel-gray">
                             <div class="panel-heading">Schema Information and Selection
                                 <div style="float:right;">
                                     <button ng-click="clearSchema()">clear</button>
@@ -144,9 +137,9 @@ <h4><strong>An error occurred while processing your request:</strong></h4>
                         <span class="glyphicon glyphicon-flash" aria-hidden="true"></span> {{analyzeStr}}
                     </button>
 
-                    <div class="panel panel-info" ng-show="shouldShowAnalysis()" style="margin-top: 20px;">
+                    <div class="panel panel-gray" ng-show="shouldShowAnalysis()" style="margin-top: 20px;">
                         <div class="panel-heading">
-                            <h3 class="panel-title">Results:</h3>
+                            <h3 class="panel-title">Results</h3>
                         </div>
 
                         <div class="panel-body">
@@ -201,41 +194,66 @@ <h3 class="panel-title">Results:</h3>
                         <br>
 
                         <div class="panel panel-default" ng-repeat="i in itemsets">
-                            <div class="panel-body">
-                                <table>
-                                    <tr style="width:100%">
-                                        <td style="width:50%"><b>Support: </b></td>
-                                        <td style="width:50%"> {{i.support}}</td>
-                                    </tr>
-                                    <tr style="width:100%">
-                                        <td style="width:50%"><b>Ratio Out/In: </b></td>
-                                        <td style="width:50%"> {{i.ratioToInliers}}</td>
-                                    </tr>
-                                    <tr>
-                                        <td style="width:50%"><b>Records: </b></td>
-                                        <td style="width:50%"> {{i.numRecords}}</td>
-                                    </tr>
-                                </table>
-                                <table class="table" style="margin-top: 10px">
-                                    <tr>
-                                        <th style="width:50%">Column</th>
-                                        <th style="width:50%">Value</th>
-                                    </tr>
-                                    <tr ng-repeat="item in i.items">
-                                        <td>{{item.column}}</td>
-                                        <td>{{item.value}}</td>
-                                    </tr>
-                                </table>
-                                <div>
-                                    Plot: <div id="plotArea_{{$index}}" ng-hide="shouldHideItemsetPlot($index)"></div>
-										<span ng-repeat="h in getPlotColumns()">
-												<button ng-model="button" ng-click="prepareItemsetPlotData(h, $parent.$index)" ng-disabled="plotDataLoading">{{h}}</button>
-										</span>
+                                <div ng-class="setSev(i.ratioToInliers)" id="explanation_{{$index}}">
+                                    <div class="panel-body">
+                                        <div class="col-sm-9">
+                                        <table>
+                                            <tr>
+                                            <td>  <b>Attributes: </b> </td>
+                                            <td>
+                                                <span ng-repeat="item in i.items">
+                                                {{item.column}}{{$last ? '' : ', '}}
+                                                </span>
+                                            </td>
+                                            </tr>
+                                            <tr style="width:100%">
+                                                <td style="width:50%"><b>Ratio Out/In: </b></td>
+                                                <td style="width:50%">{{i.ratioToInliers}}</td>
+                                            </tr>
+                                            <tr style="width:100%">
+                                                <td style="width:50%"><b>Support: </b></td>
+                                                <td style="width:50%"> {{i.support}}</td>
+                                            </tr>
+                                            <tr>
+                                                <td style="width:50%"><b>Records: </b></td>
+                                                <td style="width:50%"> {{i.numRecords}}</td>
+                                            </tr>
+                                        </table>
+                                        </div>
+
+
+                                        <div class="col-sm-2">
+                                        <!--<button onclick="myFunction()">Click Me</button> -->
+                                            <button ng-click="revealItemset($index)" type="button"><span
+                                                    class="glyphicon glyphicon-chevron-down"></span></button>
+                                        </div>
+
+                                        <p></p>
+
+                                        <div id="itemset_{{$index}}" class="itemsetHidden" style="padding-top:4px">
+                                            <table class="table" style="margin-top: 100px">
+                                                <tr>
+                                                    <th style="width:50%">Column</th>
+                                                    <th style="width:50%">Value</th>
+                                                </tr>
+                                                <tr ng-repeat="item in i.items">
+                                                    <td>{{item.column}}</td>
+                                                    <td>{{item.value}}</td>
+                                                </tr>
+                                            </table>
+                                            <div>
+                                                Plot: <div id="plotArea_{{$index}}" ng-hide="shouldHideItemsetPlot($index)"></div>
+                                                <span ng-repeat="h in getPlotColumns()">
+                                                        <button ng-model="button" ng-click="prepareItemsetPlotData(h, $parent.$index)" ng-disabled="plotDataLoading">{{h}}</button>
+                                                </span>
+                                            </div>
+                                            <div style="padding: 15px; padding-top:30px;">
+                                                <a ng-click="exploreItems(i.items)">Explore</a>
+                                            </div>
+                                        </div>
+
                                 </div>
                             </div>
-                            <div style="padding: 15px; padding-top:0px;">
-                                <a ng-click="exploreItems(i.items)">Explore</a>
-                            </div>
                         </div>
                     </div>
                 </div>

diff --git a/frontend/src/main/resources/frontend/consoleController.js b/frontend/src/main/resources/frontend/consoleController.js
@@ -315,6 +315,26 @@ myApp.controller('analyzeController', ['$scope', '$http', '$window', 'configServ
 
     var analyzing = false
 
+    $scope.revealItemset = function(idx) {
+        var x = document.getElementById('itemset_'+idx);
+        if (x.classList.contains('itemsetHidden')) {
+            x.classList.remove('itemsetHidden');
+        }
+        else {
+            x.classList.add('itemsetHidden');
+        }
+    }
+
+    $scope.setSev = function(riskRatio){
+        if (riskRatio > 30){
+            return 'high-risk';
+        }
+        if (riskRatio > 10){
+            return 'med-risk';
+        }
+        return 'low-risk';
+    }
+
     $scope.shouldShowAnalysis = function() {
         return configService.hasAnalysis()
     }
@@ -356,7 +376,7 @@ myApp.controller('analyzeController', ['$scope', '$http', '$window', 'configServ
 			$scope.resetPlotDiv();
 			$scope.resetItemsetPlots();
 
-            $scope.sortAnalysis("support");
+            $scope.sortAnalysis("ratioToInliers");
 	    });
     }
     }
@@ -738,6 +758,8 @@ myApp.controller('analyzeController', ['$scope', '$http', '$window', 'configServ
 }]);
 
 
+
+
 myApp.controller('exploreController', ['$scope', '$http', 'configService', 'explorerService', function($scope, $http, configService, explorerService) {
 
     $scope.visibleRows = []

diff --git a/frontend/src/main/resources/frontend/css/mb-frontend.css b/frontend/src/main/resources/frontend/css/mb-frontend.css
@@ -0,0 +1,64 @@
+
+
+.itemsetHidden{
+    display:none;
+}
+
+.high-risk{
+    border-color: #D66F5E;
+    border-style: solid;
+    border-width: 2px;
+    border-radius: 5px;
+}
+
+.med-risk{
+    border-color: #E2AA85;
+    border-style: solid;
+    border-width: 2px;
+    border-radius: 5px;
+}
+
+.low-risk{
+    border-color: #ABC9AF;
+    border-style: solid;
+    border-width: 2px;
+    border-radius: 5px;
+}
+
+
+element {
+}
+.active-schema {
+    background-color: #ABC9AF !important;
+    background-image: none !important;
+}
+
+.panel-brown {
+    border-color: #DFCFB5;
+}
+.panel-brown > .panel-heading {
+    background-image: none;    
+    background: #E1E5D5; 
+    border-color: #E1E5D5;
+}
+
+.panel-green {
+    border-color: #DCEBD9;
+}
+.panel-green > .panel-heading {
+    background-image: none;    
+    background: #DCEBD9; 
+    border-color: #DCEBD9;
+}
+
+.panel-gray {
+    border-color: #dcddd7;
+}
+.panel-gray > .panel-heading {
+    background-image: none;    
+    background: #dcddd7; 
+    border-color: #dcddd7;
+}
+
+
+