Merge branch 'master' into kinetic

utexas-bwi · Dec 15, 2017 · f0254f5 · f0254f5
2 parents c768147 + 07613b3
commit f0254f5
Show file tree

Hide file tree

Showing 4 changed files with 105 additions and 2 deletions.
diff --git a/bwi_tools/CMakeLists.txt b/bwi_tools/CMakeLists.txt
@@ -18,7 +18,7 @@ catkin_package(
     roslib
   DEPENDS 
     OpenCV
-    BOOST
+    Boost
   INCLUDE_DIRS include
   LIBRARIES ${PROJECT_NAME} ${PROJECT_NAME}_json
 )

diff --git a/bwi_virtour/launch/virtour_passive.launch b/bwi_virtour/launch/virtour_passive.launch
@@ -1,5 +1,6 @@
 <launch>
   <include file="$(find rosbridge_server)/launch/rosbridge_websocket.launch"></include>
+  <include file="$(find audio_capture)/launch/capture.launch"></include>
   <node name="web_video_server" pkg="web_video_server" type="web_video_server">
     <param name="__image_transport" value="compressed" />
   </node>

diff --git a/bwi_virtour/package.xml b/bwi_virtour/package.xml
@@ -29,5 +29,6 @@
   <exec_depend>message_runtime</exec_depend>
   <exec_depend>rosbridge_server</exec_depend>
   <exec_depend>web_video_server</exec_depend>
-
+  <exec_depend>audio_capture</exec_depend>
+
 </package>
diff --git a/bwi_virtour/web/js/virtour.js b/bwi_virtour/web/js/virtour.js
@@ -40,6 +40,12 @@ var topics = null;
 var servosEnabled = false;
 var robot_v3 = false;
 var curr_color = 0;
+var audioSourceBuffer = null;
+var audioBufferQueue = [];
+var bufferLength = 0;
+var mediaSource = null;
+var audio = null;
+
 
 // Scavenger Hunt Statuses
 var FINISHED = "<span class=\"glyphicon glyphicon-ok\"></span> Done";
@@ -127,6 +133,14 @@ function error(errorMessage, errorTitle) {
   log("error: "+errorMessage);
 }
 
+function hasTextEncoder() {
+  return !!(window.TextEncoder); 
+}
+
+function hasMediaSource() {
+  return !!(window.MediaSource || window.WebKitMediaSource);
+}
+
 function createIdentity() {
   identity = getUUID();
 }
@@ -251,6 +265,20 @@ function subscribeScavengerHuntListener(ros) {
   });
 }
 
+function subscribeAudioListner(ros) {
+  var listener = new ROSLIB.Topic({
+    ros : ros,
+    name : '/audio',
+    messageType : 'audio_common_msgs/AudioData'
+  });
+  log("Added ping Audio listener");
+
+  listener.subscribe(function(audioChunck) {
+    //log('Received message on ' + listener.name + ': ' + audioChunck.data);
+    streamAudio(audioChunck);
+  });
+}
+
 function viewScavengerHunt() {
   $(".scavengerhunt-modal").modal();
 }
@@ -288,6 +316,56 @@ function updateScavengerHuntStatus(msg) {
   }
 }
 
+function streamAudio(audioChunckB64) {
+  var audioChunckAsStr = atob(audioChunckB64.data);
+  var chunckUint8Array = new Uint8Array(audioChunckAsStr.length);
+  for (var i = 0; i < audioChunckAsStr.length; i++) {
+      chunckUint8Array[i] = audioChunckAsStr.charCodeAt(i);
+  }
+
+  //log("audio started buffer num:" + chunckUint8Array);
+  bufferLength += chunckUint8Array.length;
+  audioBufferQueue.push(chunckUint8Array);
+  if (audioSourceBuffer && !audioSourceBuffer.updating) {
+     loadNextBuffer();
+  }
+}
+
+function onSourceOpen() {
+  // this.readyState === 'open'. Add a source buffer that expects webm chunks.
+  audioSourceBuffer = mediaSource.addSourceBuffer('audio/mpeg');
+  audioSourceBuffer.addEventListener('updateend', loadNextBuffer, false);
+  //....
+}
+
+function loadNextBuffer (audioChunck) {
+  //log("LoadNextBuffer....");
+  if ( audioBufferQueue.length > 0) {
+    var audioBufferSize = audioBufferQueue.length;
+    var mp3Buffer = new Uint8Array(bufferLength);
+    bufferLength = 0;
+    //log("mp3BufferEmpty" + mp3Buffer);
+    buffSize = 0;
+    for ( i = 0; i < audioBufferSize; i++) {
+      currBuff = audioBufferQueue.shift();
+      mp3Buffer.set(currBuff,buffSize);
+      buffSize += currBuff.length;
+    }
+    //log("Loading next buffer....");
+    //log("audio buffer" + mp3Buffer);
+    audioSourceBuffer.appendBuffer(mp3Buffer);
+
+    if (audio.paused) {
+      audio.play();
+    }
+  }
+}
+
+
+
+
+
+
 function updatePosition(x, y){
   xp = 100 * ((x - map_origin_x) / map_res - map_marker_offset) / map_width;
   yp = 100 * ((y - map_origin_y) / map_res - map_marker_offset) / map_height;
@@ -650,6 +728,7 @@ $(".robots").on("click", ".robot", function() {
   subscribeListener(segbot.ros);
   subscribePoseListener(segbot.ros);
   subscribeScavengerHuntListener(segbot.ros);
+  subscribeAudioListner(segbot.ros);
 
 
   // hide the intro stuff
@@ -789,6 +868,28 @@ $(".robots").on("click", ".robot", function() {
   log("Loading video from: " + videoSource);
   $(".controllingIframe").append("<img width=\"100%\" height=\"100%\" src=\"" + videoSource + "\">");
 
+  // set up video streaming (mp3 format)
+  if (hasMediaSource() && hasTextEncoder()) {
+    log("has media source, start streaming audio.");
+     //$(".controllingIframe").append("<audio id=\"audio\" controls=\"controls\">");
+     //$(".controllingIframe").append("<source id=\"mp3Source\" type=\"audio/mp3\"></source>");
+     //$(".controllingIframe").append("</audio>");
+     //audio = document.querySelector('audio');
+     $(".controllingIframe").append("<audio id=\"mp3Source\"></audio>");
+     audio = document.getElementById('mp3Source');
+
+     window.MediaSource = window.MediaSource || window.WebKitMediaSource;
+     mediaSource = new MediaSource();
+
+     audio.src= window.URL.createObjectURL(mediaSource);
+
+     mediaSource.addEventListener('sourceopen', onSourceOpen, false);
+     mediaSource.addEventListener('webkitsourceopen', onSourceOpen, false);
+
+  } else {
+    alert("Bummer. Your browser doesn't support the MediaSource API or the TextEncoder feature! Cannot stream audio.");
+  }
+
 });
 
 // add callback handlers for buttons