better filtering for tree/road distance

docs/copa.graffle

@@ -952,8 +952,9 @@ Metadata}</string>
 \pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc
 
 \f0\b\fs20 \cf0 Scrub\
+\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc
 
-\b0 species}</string>
+\b0 \cf0 species}</string>
 			</dict>
 		</dict>
 		<dict>
@@ -3569,8 +3570,9 @@ Metadata}</string>
 \pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc
 
 \f0\b\fs20 \cf0 Checkpoint\
+\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc
 
-\b0 tsv}</string>
+\b0 \cf0 tsv}</string>
 			</dict>
 		</dict>
 		<dict>
@@ -4882,6 +4884,8 @@ Metadata}</string>
 		<integer>0</integer>
 		<key>ExpandedCanvases</key>
 		<array/>
+		<key>FitInWindow</key>
+		<true/>
 		<key>Frame</key>
 		<string>{{246, 4}, {1034, 771}}</string>
 		<key>ListView</key>
@@ -4897,15 +4901,15 @@ Metadata}</string>
 		<key>SidebarWidth</key>
 		<integer>120</integer>
 		<key>VisibleRegion</key>
-		<string>{{73, 107}, {1020, 632}}</string>
+		<string>{{0, -48}, {1942.0958745801363, 1203.337836014359}}</string>
 		<key>Zoom</key>
-		<real>1</real>
+		<real>0.52520579099655151</real>
 		<key>ZoomValues</key>
 		<array>
 			<array>
 				<string>6: tf-idf + tdd</string>
-				<real>1</real>
 				<real>0.0</real>
+				<real>1</real>
 			</array>
 		</array>
 	</dict>

src/main/java/copa/Main.java

@@ -32,6 +32,8 @@
 import cascading.pipe.CoGroup;
 import cascading.pipe.Checkpoint;
 import cascading.pipe.Each;
+import cascading.pipe.Every;
+import cascading.pipe.GroupBy;
 import cascading.pipe.HashJoin;
 import cascading.pipe.Pipe;
 import cascading.pipe.assembly.Rename;
@@ -152,30 +154,32 @@
 
     // generate road segments, with midpoint, y=mx+b, and road_geohash for each
     Fields segmentArguments = new Fields( "geo" );
-    Fields segmentResults = new Fields( "lat0", "lng0", "alt0", "lat1", "lng1", "alt1", "lat_mid", "lng_mid", "slope", "intercept" );
+    Fields segmentResults = new Fields( "lat0", "lng0", "alt0", "lat1", "lng1", "alt1", "lat_mid", "lng_mid" );
     roadPipe = new Each( roadPipe, segmentArguments, new RoadSegmentFunction( segmentResults ), Fields.ALL );
 
     geohashArguments = new Fields( "lat_mid", "lng_mid" );
     roadPipe = new Each( roadPipe, geohashArguments, new GeoHashFunction( new Fields( "road_geohash" ), 6 ), Fields.ALL );
 
-    fieldSelector = new Fields( "road_name", "year_construct", "traffic_count", "traffic_index", "traffic_class", "paving_length", "paving_width", "paving_area", "surface_type", "bike_lane", "bus_route", "truck_route", "albedo", "lat0", "lng0", "alt0", "lat1", "lng1", "alt1", "lat_mid", "lng_mid", "slope", "intercept", "road_geohash" );
+    fieldSelector = new Fields( "road_name", "year_construct", "traffic_count", "traffic_index", "traffic_class", "paving_length", "paving_width", "paving_area", "surface_type", "bike_lane", "bus_route", "truck_route", "albedo", "lat0", "lng0", "alt0", "lat1", "lng1", "alt1", "road_geohash" );
     roadPipe = new Retain( roadPipe, fieldSelector );
 
     // join the tree and road pipes to estimate shade
     Pipe shadePipe = new Pipe( "shade", roadPipe );
     shadePipe = new CoGroup( shadePipe, new Fields( "road_geohash" ), treePipe, new Fields( "tree_geohash" ), new InnerJoin() );
 
     // calculate a rough estimate for distance from tree to road, then filter for "< ~1 block"
-    Fields treeDistArguments = new Fields( "tree_lat", "tree_lng", "slope", "intercept" );
+    Fields treeDistArguments = new Fields( "tree_lat", "tree_lng", "lat0", "lng0", "lat1", "lng1" );
     Fields tree_dist = new Fields( "tree_dist" );
     shadePipe = new Each( shadePipe, treeDistArguments, new TreeDistanceFunction( tree_dist ), Fields.ALL );
 
-    ExpressionFilter distFilter = new ExpressionFilter( "tree_dist > 0.001", Double.class );
+    ExpressionFilter distFilter = new ExpressionFilter( "tree_dist > 25.0", Double.class );
     shadePipe = new Each( shadePipe, tree_dist, distFilter );
 
     // checkpoint this (big) calculation too
-    fieldSelector = new Fields( "road_name", "year_construct", "traffic_count", "traffic_index", "traffic_class", "paving_length", "paving_width", "paving_area", "surface_type", "bike_lane", "bus_route", "truck_route", "albedo", "lat0", "lng0", "alt0", "lat1", "lng1", "alt1", "slope", "intercept", "tree_name", "priv", "tree_id", "situs", "tree_site", "species", "wikipedia", "calflora", "min_height", "max_height", "tree_lat", "tree_lng", "tree_alt", "tree_dist", "tree_geohash" );
+    fieldSelector = new Fields( "road_name", "year_construct", "traffic_count", "traffic_index", "traffic_class", "paving_length", "paving_width", "paving_area", "surface_type", "bike_lane", "bus_route", "truck_route", "albedo", "lat0", "lng0", "lat1", "lng1", "tree_name", "priv", "tree_id", "situs", "tree_site", "species", "wikipedia", "calflora", "min_height", "max_height", "tree_lat", "tree_lng", "tree_alt", "tree_dist", "tree_geohash" );
     shadePipe = new Retain( shadePipe, fieldSelector );
+    shadePipe = new GroupBy( shadePipe, new Fields( "tree_name" ), new Fields( "tree_dist" ) );
+
     Checkpoint shadeCheck = new Checkpoint( "shade", shadePipe );
 
     // determine the geohash for GPS tracks log events

src/main/java/copa/RoadSegmentFunction.java

@@ -56,9 +56,6 @@ public void operate( FlowProcess flowProcess, FunctionCall functionCall )
       Double lat_mid = ( lat0 + lat1 ) / 2.0;
       Double lng_mid = ( lng0 + lng1 ) / 2.0;
 
-      Double slope = ( lat1 - lat0 ) / ( lng1 - lng0 );
-      Double intercept = lat0 - ( slope * lng0 );
-
       Tuple result = new Tuple();
 
       result.add( lat0 );
@@ -72,9 +69,6 @@ public void operate( FlowProcess flowProcess, FunctionCall functionCall )
       result.add( lat_mid );
       result.add( lng_mid );
 
-      result.add( slope );
-      result.add( intercept );
-
       functionCall.getOutputCollector().add( result );
       }
     }

src/main/java/copa/TreeDistanceFunction.java

@@ -39,17 +39,31 @@ public TreeDistanceFunction( Fields fieldDeclaration )
   public void operate( FlowProcess flowProcess, FunctionCall functionCall )
     {
     TupleEntry argument = functionCall.getArguments();
-    Double lat = argument.getDouble( 0 );
-    Double lng = argument.getDouble( 1 );
-    Double slope = argument.getDouble( 2 );
-    Double intercept = argument.getDouble( 3 );
+    double tree_lat = argument.getDouble( 0 );
+    double tree_lng = argument.getDouble( 1 );
+    double lat0 = argument.getDouble( 2 );
+    double lng0 = argument.getDouble( 3 );
+    double lat1 = argument.getDouble( 4 );
+    double lng1 = argument.getDouble( 5 );
 
-    // based on:
-    // http://math.ucsd.edu/~wgarner/math4c/derivations/distance/distptline.htm
-    Double tree_dist = Math.abs( lat - ( slope * lng ) - intercept ) / Math.sqrt( Math.pow( slope, 2.0 ) + 1.0 );
+    // approximation in meters, based on a euclidean approach
+    // for a better metric, try using a Haversine formula
+
+    //double tree_dist = pointToLineDistance( lat0, lng0, lat1, lng1, tree_lat, tree_lng );
+    //tree_dist = Math.min( tree_dist, Math.hypot( tree_lat - lat0, tree_lng - lng0 ) );
+    //tree_dist = Math.min( tree_dist, Math.hypot( tree_lat - lat1, tree_lng - lng1 ) );
+
+    double tree_dist = Math.hypot( tree_lat - ( ( lat0 + lat1 ) / 2.0 ), tree_lng - ( ( lng0 + lng1 ) / 2.0 ) );
 
     Tuple result = new Tuple();
-    result.add( tree_dist );
+    result.add( tree_dist * 61290.0 );
     functionCall.getOutputCollector().add( result );
     }
+
+  // http://en.wikipedia.org/wiki/Distance_from_a_point_to_a_line
+  public double pointToLineDistance( double x0, double y0, double x1, double y1, double x_, double y_ )
+    {
+    double norm_len = Math.hypot( x1 - x0, y1 - y0 );
+    return Math.abs( ( x_ - x0 ) * ( y1 - y0 ) - ( y_ - y0 ) * ( x1 - x0 ) ) / norm_len;
+    }
   }