Merge pull request #46 from unchartedsoftware/feature/45

Feature/45

Dockerfile

@@ -1,12 +1,12 @@
 #
 # Salt Example Container
-# Apache Spark 1.6.0
+# Apache Spark 1.6.1
 #
 # Build using:
 # $ docker build -t uncharted/salt-examples .
 #
 
-FROM uncharted/sparklet:1.6.0
+FROM uncharted/sparklet:1.6.1
 MAINTAINER Sean McIntyre <[email protected]>
 
 # Mount point for source directories

README.md

@@ -2,6 +2,8 @@
 
 > Examples for [Salt](https://github.com/unchartedsoftware/salt)
 
+**Please be sure to check out the release tag matching the version of Salt you intend to use**
+
  - [png-example](./png-example) - illustrates using Salt to build .png image datasets compatible with any TMS visualization library.
 
  - [bin-example](./bin-example) - illustrates using Salt to build byte file datasets which utilize client-side heatmap rendering using [Leaflet's TileLayer.Canvas](http://leafletjs.com/reference.html#tilelayer-canvas) library.

bin-example/generation/build.gradle

@@ -14,13 +14,13 @@ apply plugin: 'maven'
 apply plugin: 'idea'
 
 group = 'software.uncharted.salt'
-version = '2.0.1'
+version = '3.0.0'
 
 project.ext {
   scalaBinaryVersion = '2.10'
   scalaVersion = '2.10.4'
-  sparkVersion = '1.4.0'
-  saltversion = '2.0.1'
+  sparkVersion = '1.6.1'
+  saltVersion = '3.0.0'
 }
 
 jar {

bin-example/generation/src/main/scala/software/uncharted/salt/examples/bin/Main.scala

@@ -8,7 +8,7 @@ import org.apache.spark.sql.Row
 import software.uncharted.salt.core.projection.numeric._
 import software.uncharted.salt.core.generation.request._
 import software.uncharted.salt.core.generation.Series
-import software.uncharted.salt.core.generation.mapreduce.MapReduceTileGenerator
+import software.uncharted.salt.core.generation.TileGenerator
 import software.uncharted.salt.core.generation.output.SeriesData
 import software.uncharted.salt.core.analytic.numeric._
 
@@ -25,7 +25,7 @@ object Main {
   val layerName = "pickups"
 
   // Creates and returns an Array of Double values encoded as 64bit Integers
-  def createByteBuffer(tile: SeriesData[(Int, Int, Int), Double, (Double, Double)]): Array[Byte] = {
+  def createByteBuffer(tile: SeriesData[(Int, Int, Int), (Int, Int), Double, (Double, Double)]): Array[Byte] = {
     val byteArray = new Array[Byte](tileSize * tileSize * 8)
     var j = 0
     tile.bins.foreach(b => {
@@ -71,7 +71,7 @@ object Main {
     }
 
     // Tile Generator object, which houses the generation logic
-    val gen = new MapReduceTileGenerator(sc)
+    val gen = TileGenerator(sc)
 
     // Break levels into batches. Process several higher levels at once because the
     // number of tile outputs is quite low. Lower levels done individually due to high tile counts.
@@ -88,7 +88,7 @@ object Main {
       val pickups = new Series((tileSize - 1, tileSize - 1),
         pickupExtractor,
         new MercatorProjection(level),
-        None,
+        (r: Row) => Some(1),
         CountAggregator,
         Some(MinMaxAggregator))
 
@@ -98,7 +98,7 @@ object Main {
 
       // Translate RDD of Tiles to RDD of (coordinate,byte array), collect to master for serialization
       val output = rdd
-        .map(s => pickups(s))
+        .map(s => pickups(s).get)
         .map(tile => {
           // Return tuples of tile coordinate, byte array
           (tile.coords, createByteBuffer(tile))
@@ -120,7 +120,7 @@ object Main {
 
       // Create map from each level to min / max values.
       rdd
-        .map(s => pickups(s))
+        .map(s => pickups(s).get)
         .map(t => (t.coords._1.toString, t.tileMeta.get))
         .reduceByKey((l, r) => {
           (Math.min(l._1, r._1), Math.max(l._2, r._2))

bin-example/webapp/package.json

@@ -1,6 +1,6 @@
 {
   "name": "salt-bin-example",
-  "version": "2.0.1",
+  "version": "3.0.0",
   "description": "Example of using Salt to generate Bin tiles",
   "private": true,
   "main": "server.js",

hierarchical-pie-example/generation/build.gradle

@@ -14,13 +14,13 @@ apply plugin: 'maven'
 apply plugin: 'idea'
 
 group = 'software.uncharted.salt'
-version = '2.0.1'
+version = '3.0.0'
 
 project.ext {
   scalaBinaryVersion = '2.10'
   scalaVersion = '2.10.4'
-  sparkVersion = '1.5.1'
-  saltversion = '2.0.1'
+  sparkVersion = '1.6.1'
+  saltVersion = '3.0.0'
 }
 
 jar {

hierarchical-pie-example/generation/src/main/scala/software/uncharted/salt/examples/hpie/Main.scala

@@ -10,7 +10,7 @@ import java.sql.{Connection,DriverManager,Statement};
 
 import software.uncharted.salt.core.generation.request._
 import software.uncharted.salt.core.generation.Series
-import software.uncharted.salt.core.generation.mapreduce.MapReduceTileGenerator
+import software.uncharted.salt.core.generation.TileGenerator
 import software.uncharted.salt.core.generation.output.Tile
 import software.uncharted.salt.core.generation.output.SeriesData
 import software.uncharted.salt.core.analytic.numeric.{SumAggregator, CountAggregator}
@@ -22,7 +22,7 @@ import scala.collection.mutable.ArrayBuffer
 
 object Main {
 
-  val MAX_TILING_DEPTH = 10 //how many directories deep should we allow the visualization to go?
+  val MAX_TILING_DEPTH = 6 //how many directories deep should we allow the visualization to go?
   val INSERT_BATCH_SIZE = 1000 //batch insert size for SQLite. Probably don't need to change this.
 
   /*
@@ -92,9 +92,9 @@ object Main {
         Some(r.getLong(fieldIndex).toDouble)
       }
     }
-    val sBytes = new Series(0, pathExtractor, projection, Some(bytesExtractor), SumAggregator, None)
+    val sBytes = new Series(0, pathExtractor, projection, bytesExtractor, SumAggregator, None)
     // create Series for tracking total items
-    val ssItems = new Series(0, pathExtractor, projection, None, CountAggregator, None)
+    val ssItems = new Series(0, pathExtractor, projection, (r: Row) => Some(1), CountAggregator, None)
     // create Series for tracking total directory items
     val dirChildExtractor = (r: Row) => {
       val fieldIndex = r.fieldIndex("permissions_string")
@@ -106,7 +106,7 @@ object Main {
         Some(0D)
       }
     }
-    val sDirectories = new Series(0, pathExtractor, projection, Some(dirChildExtractor), SumAggregator, None)
+    val sDirectories = new Series(0, pathExtractor, projection, dirChildExtractor, SumAggregator, None)
     // create Series for tracking total executable items
     val exChildExtractor = (r: Row) => {
       val fieldIndex = r.fieldIndex("permissions_string")
@@ -118,7 +118,7 @@ object Main {
         Some(0D)
       }
     }
-    val sExecutables = new Series(0, pathExtractor, projection, Some(exChildExtractor), SumAggregator, None)
+    val sExecutables = new Series(0, pathExtractor, projection, exChildExtractor, SumAggregator, None)
 
     // using the Uncharted Spark Pipeline for ETL
     val inputDataPipe = Pipe(() => {
@@ -132,7 +132,7 @@ object Main {
     .to(ops.core.dataframe.toRDD)
     // pipe to salt
     .to(rdd => {
-      val gen = new MapReduceTileGenerator(sc)
+      val gen = TileGenerator(sc)
       gen.generate(rdd, Seq(sBytes, ssItems, sDirectories, sExecutables), new PathRequest())
     })
     // to simplify example, eliminate SQLite lock contention issue by collecting tiles to master
@@ -150,10 +150,10 @@ object Main {
           stmt.setString(2, tile.coords)
           stmt.setString(3, tile.coords.substring(tile.coords.lastIndexOf("/")+1))
           stmt.setInt(4, tile.coords.split("/").length - 1)
-          stmt.setInt(5, sBytes(tile).bins(0).toInt)
-          stmt.setInt(6, ssItems(tile).bins(0).toInt)
-          stmt.setInt(7, sDirectories(tile).bins(0).toInt)
-          stmt.setInt(8, sExecutables(tile).bins(0).toInt)
+          stmt.setInt(5, sBytes(tile).get.bins(0).toInt)
+          stmt.setInt(6, ssItems(tile).get.bins(0).toInt)
+          stmt.setInt(7, sDirectories(tile).get.bins(0).toInt)
+          stmt.setInt(8, sExecutables(tile).get.bins(0).toInt)
           stmt.addBatch()
           i+=1
           if (i % INSERT_BATCH_SIZE == 0 || i == tiles.length) {

hierarchical-pie-example/webapp/package.json

@@ -1,6 +1,6 @@
 {
   "name": "salt-hierarchical-pie-example",
-  "version": "2.0.1",
+  "version": "3.0.0",
   "description": "Example of using Salt to generate a visualization of heirarchical data",
   "private": true,
   "main": "server.js",

path-example/generation/build.gradle

@@ -14,13 +14,13 @@ apply plugin: 'maven'
 apply plugin: 'idea'
 
 group = 'software.uncharted.salt'
-version = '2.0.1'
+version = '3.0.0'
 
 project.ext {
   scalaBinaryVersion = '2.10'
   scalaVersion = '2.10.4'
-  sparkVersion = '1.4.0'
-  saltversion = '2.0.1'
+  sparkVersion = '1.6.1'
+  saltVersion = '3.0.0'
 }
 
 jar {

path-example/generation/src/main/scala/software/uncharted/salt/examples/path/Main.scala

@@ -23,7 +23,7 @@ import org.apache.spark.sql.Row
 
 import software.uncharted.salt.core.generation.request._
 import software.uncharted.salt.core.generation.Series
-import software.uncharted.salt.core.generation.mapreduce.MapReduceTileGenerator
+import software.uncharted.salt.core.generation.TileGenerator
 import software.uncharted.salt.core.generation.output.SeriesData
 import software.uncharted.salt.core.analytic.numeric._
 
@@ -40,7 +40,7 @@ object Main {
   val layerName = "paths"
 
   // Creates and returns an Array of Double values encoded as 64bit Integers
-  def createByteBuffer(tile: SeriesData[(Int, Int, Int), Double, (Double, Double)]): Array[Byte] = {
+  def createByteBuffer(tile: SeriesData[(Int, Int, Int), (Int, Int), Double, (Double, Double)]): Array[Byte] = {
     val byteArray = new Array[Byte](tileSize * tileSize * 8)
     var j = 0
     tile.bins.foreach(b => {
@@ -87,7 +87,7 @@ object Main {
     }
 
     // Tile Generator object, which houses the generation logic
-    val gen = new MapReduceTileGenerator(sc)
+    val gen = TileGenerator(sc)
 
     // Break levels into batches. Process several higher levels at once because the
     // number of tile outputs is quite low. Lower levels done individually due to high tile counts.
@@ -104,7 +104,7 @@ object Main {
       val paths = new Series((tileSize - 1, tileSize - 1),
         pathExtractor,
         new MercatorLineProjection(1024, level),
-        None,
+        (r: Row) => Some(1),
         CountAggregator,
         Some(MinMaxAggregator))
 
@@ -114,7 +114,7 @@ object Main {
 
       // Translate RDD of Tiles to RDD of (coordinate,byte array), collect to master for serialization
       val output = rdd
-        .map(s => paths(s))
+        .map(s => paths(s).get)
         .map(tile => {
           // Return tuples of tile coordinate, byte array
           (tile.coords, createByteBuffer(tile))
@@ -136,7 +136,7 @@ object Main {
 
       // Create map from each level to min / max values.
       rdd
-        .map(s => paths(s))
+        .map(s => paths(s).get)
         .map(t => (t.coords._1.toString, t.tileMeta.get))
         .reduceByKey((l, r) => {
           (Math.min(l._1, r._1), Math.max(l._2, r._2))

path-example/webapp/package.json

@@ -1,6 +1,6 @@
 {
   "name": "salt-path-example",
-  "version": "2.0.1",
+  "version": "3.0.0",
   "description": "Example of using Salt to generate binary tiles display paths",
   "private": true,
   "main": "server.js",

png-example/generation/build.gradle

@@ -14,13 +14,13 @@ apply plugin: 'maven'
 apply plugin: 'idea'
 
 group = 'software.uncharted.salt'
-version = '2.0.1'
+version = '3.0.0'
 
 project.ext {
   scalaBinaryVersion = '2.10'
   scalaVersion = '2.10.4'
-  sparkVersion = '1.4.0'
-  saltversion = '2.0.1'
+  sparkVersion = '1.6.1'
+  saltVersion = '3.0.0'
 }
 
 jar {

png-example/generation/src/main/scala/software/uncharted/salt/examples/png/Main.scala

@@ -8,7 +8,7 @@ import org.apache.spark.sql.Row
 import software.uncharted.salt.core.projection.numeric._
 import software.uncharted.salt.core.generation.request._
 import software.uncharted.salt.core.generation.Series
-import software.uncharted.salt.core.generation.mapreduce.MapReduceTileGenerator
+import software.uncharted.salt.core.generation.TileGenerator
 import software.uncharted.salt.core.generation.output.SeriesData
 import software.uncharted.salt.core.analytic.numeric._
 
@@ -54,7 +54,7 @@ object Main {
   }
 
   // Creates and returns an Array of RGBA values encoded as 32bit integers
-  def createRGBABuffer(tile: SeriesData[(Int, Int, Int), Double, (Double, Double)], min: Double, max: Double): Array[Int] = {
+  def createRGBABuffer(tile: SeriesData[(Int, Int, Int), (Int, Int), Double, (Double, Double)], min: Double, max: Double): Array[Int] = {
     val rgbArray = new Array[Int](tileSize * tileSize)
     tile.bins.zipWithIndex.foreach(b => {
       val count = b._1
@@ -98,7 +98,7 @@ object Main {
     }
 
     // Tile Generator object, which houses the generation logic
-    val gen = new MapReduceTileGenerator(sc)
+    val gen = TileGenerator(sc)
 
     // Iterate over sets of levels to generate. Process several higher levels at once because the
     // number of tile outputs is quite low. Lower levels done individually due to high tile counts.
@@ -111,7 +111,7 @@ object Main {
       val pickups = new Series((tileSize - 1, tileSize - 1),
         pickupExtractor,
         new MercatorProjection(level),
-        None,
+        (r: Row) => Some(1),
         CountAggregator,
         Some(MinMaxAggregator))
 
@@ -123,7 +123,7 @@ object Main {
       // counts for each zoom level must be known.
       // Create map from each level to min / max values.
       val levelInfo = rdd
-        .map(s => pickups(s))
+        .map(s => pickups(s).get)
         .map(t => (t.coords._1, t.tileMeta.get))
         .reduceByKey((l, r) => {
           (Math.min(l._1, r._1), Math.max(l._2, r._2))
@@ -139,7 +139,7 @@ object Main {
         .mapPartitions(partition => {
           val levelInfo = bLevelInfo.value
           partition
-            .map(s => pickups(s))
+            .map(s => pickups(s).get)
             .map(tile => {
               val level = tile.coords._1
               val minMax = levelInfo(level)

png-example/webapp/package.json

@@ -1,6 +1,6 @@
 {
   "name": "salt-png-example",
-  "version": "2.0.1",
+  "version": "3.0.0",
   "description": "Example of using Salt to generate PNG tiles",
   "private": true,
   "main": "server.js",

torque-example/generation/build.gradle

@@ -14,13 +14,13 @@ apply plugin: 'maven'
 apply plugin: 'idea'
 
 group = 'software.uncharted.salt'
-version = '2.0.1'
+version = '3.0.0'
 
 project.ext {
   scalaBinaryVersion = '2.10'
   scalaVersion = '2.10.4'
-  sparkVersion = '1.4.0'
-  saltversion = '2.0.1'
+  sparkVersion = '1.6.1'
+  saltVersion = '3.0.0'
 }
 
 jar {