Create audified cube and use with spectrum at spaxel tool

jdaviz/configs/cubeviz/cubeviz.yaml

@@ -33,6 +33,7 @@ tray:
   - specviz-line-analysis
   - cubeviz-moment-maps
   - imviz-aper-phot-simple
+  - cubeviz-sonify-data
   - export
   - about
 viewer_area:

jdaviz/configs/cubeviz/plugins/__init__.py

@@ -4,3 +4,4 @@
 from .moment_maps.moment_maps import *  # noqa
 from .slice.slice import * # noqa
 from .spectral_extraction.spectral_extraction import * # noqa
+from .sonify_data.sonify_data import * # noqa

jdaviz/configs/cubeviz/plugins/cube_listener.py

@@ -0,0 +1,107 @@
+from strauss.sonification import Sonification
+from strauss.sources import Events, Objects
+from strauss import channels
+from strauss.score import Score
+from strauss.generator import Spectralizer
+import numpy as np
+from tqdm import tqdm
+from contextlib import contextmanager
+import sys, os
+
+# some beginner utility functions for STRAUSS + CubeViz
+
+@contextmanager
+def suppress_stderr():
+    with open(os.devnull, "w") as devnull:
+        old_stderr = sys.stderr
+        sys.stderr = devnull
+        try:  
+            yield
+        finally:
+            sys.stderr = old_stderr
+
+def audify_spectrum(spec, duration, overlap=0.05, system='mono', srate=44100, fmin=40, fmax=1300):
+    notes = [["A2"]]
+    score =  Score(notes, duration)
+
+    #set up spectralizer generator
+    generator = Spectralizer(samprate=srate)
+
+    # Lets pick the mapping frequency range for the spectrum...
+    generator.modify_preset({'min_freq':fmin, 'max_freq':fmax})
+
+    data = {'spectrum':[spec], 'pitch':[1]}
+
+    # again, use maximal range for the mapped parameters
+    lims = {'spectrum': ('0','100')}
+
+    # set up source
+    sources = Events(data.keys())
+    sources.fromdict(data)
+    sources.apply_mapping_functions(map_lims=lims)
+
+    # render and play sonification!
+    soni = Sonification(score, sources, generator, system, samprate=srate)
+    soni.render()
+    soni._make_seamless(overlap)
+    # print(soni.loop_channels)
+    # sd.play(soni.loop_channels['0'].values * 0.5,loop=True)
+    return soni.loop_channels['0'].values
+
+class CubeListenerData:
+    def __init__(self, cube, wlens, samplerate=44100, duration=1, overlap=0.05, buffsize=1024, bdepth=16):
+        self.siglen = int(samplerate*(duration-overlap))
+        self.cube = cube
+        self.dur = duration
+        self.bdepth = bdepth
+        self.srate = samplerate
+        self.maxval = pow(2,bdepth-1) - 1
+        self.fadedx = 0
+
+        self.wlens = wlens
+
+        # control fades
+        fade = np.linspace(0,1, buffsize+1)
+        self.ifade = fade[:-1]
+        self.ofade = fade[::-1][:-1]
+
+        self.idx1 = 0
+        self.idx2 = 0
+        self.cbuff = False
+        self.cursig = np.zeros(self.siglen, dtype='int16')
+        self.newsig = np.zeros(self.siglen, dtype='int16')
+
+        if self.cursig.nbytes * pow(1024,-3) > 2:
+            raise Exception("Cube projected to be > 2Gb!")
+
+        self.sigcube = np.zeros((*self.cube.shape[:2], self.siglen), dtype='int16')
+
+    def audify_cube(self, fmin=50, fmax=1500):
+        """
+        Iterate through the cube, convert each spectrum to a signal, and store
+        in class attributes
+        """
+        lo2hi = self.wlens.argsort()[::-1]
+        for i in tqdm(range(self.cube.shape[0])):
+            for j in range(self.cube.shape[1]):
+                with suppress_stderr():
+                    sig = audify_spectrum(self.cube[i,j,lo2hi], self.dur, 
+                                          srate=self.srate,  
+                                          fmin=fmin, fmax=fmax)
+                    sig = (sig*self.maxval).astype('int16')
+                    self.sigcube[i,j,:] = sig
+        self.cursig[:] = self.sigcube[self.idx1,self.idx2, :]
+        self.newsig[:] = self.cursig[:]
+
+    def player_callback(self, outdata, frames, time, status):
+        cur = self.cursig
+        new = self.newsig
+        sdx = int(time.outputBufferDacTime*self.srate)
+        dxs = np.arange(sdx, sdx+frames).astype(int) % self.sigcube.shape[-1]  
+        if self.cbuff:
+            outdata[:,0] = (cur[dxs] * self.ofade).astype('int16')
+            outdata[:,0] += (new[dxs] * self.ifade).astype('int16')
+            self.cursig[:] = self.newsig[:]
+            self.cbuff = False
+        else:
+            outdata[:,0] = self.cursig[dxs] 

jdaviz/configs/cubeviz/plugins/sonify_data/sonify_data.py

@@ -0,0 +1,26 @@
+from jdaviz.core.custom_traitlets import IntHandleEmpty, FloatHandleEmpty
+from jdaviz.core.registries import tray_registry
+from jdaviz.core.template_mixin import PluginTemplateMixin, DatasetSelectMixin
+
+__all__ = ['SonifyData']
+
+
+@tray_registry('cubeviz-sonify-data', label="Sonify Data",
+               viewer_requirements=['spectrum', 'image'])
+class SonifyData(PluginTemplateMixin, DatasetSelectMixin):
+    template_file = __file__, "sonify_data.vue"
+
+    sample_rate = IntHandleEmpty(44100).tag(sync=True)
+    buffer_size = IntHandleEmpty(2048).tag(sync=True)
+    assidx = FloatHandleEmpty(2.5).tag(sync=True)
+    ssvidx = FloatHandleEmpty(0.65).tag(sync=True)
+    wavemin = FloatHandleEmpty(15800).tag(sync=True)
+    wavemax = FloatHandleEmpty(16000).tag(sync=True)
+    pccut = IntHandleEmpty(20).tag(sync=True)
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def vue_sonify_cube(self, *args):
+        viewer = self.app.get_viewer('flux-viewer')
+        viewer.get_sonified_cube(self.sample_rate, self.buffer_size, self.assidx, self.ssvidx, self.pccut)

jdaviz/configs/cubeviz/plugins/sonify_data/sonify_data.vue

@@ -0,0 +1,103 @@
+<template>
+  <j-tray-plugin
+    :description="docs_description || 'Create a 2D image from a data cube.'"
+    :link="docs_link || 'https://jdaviz.readthedocs.io/en/'+vdocs+'/'+config+'/plugins.html#moment-maps'"
+    :uses_active_status="uses_active_status"
+    @plugin-ping="plugin_ping($event)"
+    :keep_active.sync="keep_active"
+    :popout_button="popout_button"
+    :scroll_to.sync="scroll_to">
+
+    <j-plugin-section-header>Cube</j-plugin-section-header>
+    <v-row>
+      <j-docs-link>Choose the input cube and spectral subset.</j-docs-link>
+    </v-row>
+
+    <plugin-dataset-select
+      :items="dataset_items"
+      :selected.sync="dataset_selected"
+      :show_if_single_entry="false"
+      label="Data"
+      hint="Select the data set."
+    />
+
+    <v-row>
+      <v-text-field
+        ref="sample_rate"
+        type="number"
+        label="Sample Rate"
+        v-model.number="sample_rate"
+        hint="The desired sample rate."
+        persistent-hint
+      ></v-text-field>
+    </v-row>
+    <v-row>
+      <v-text-field
+        ref="buffer_size"
+        type="number"
+        label="Buffer Size"
+        v-model.number="buffer_size"
+        hint="The desired buffer size."
+        persistent-hint
+      ></v-text-field>
+    </v-row>
+    <v-row>
+      <v-text-field
+        ref="assidx"
+        type="number"
+        label="Audio Spectrum Scaling Index"
+        v-model.number="assidx"
+        hint="The desired audio spectrum scaling index, typically > 1."
+        persistent-hint
+      ></v-text-field>
+    </v-row>
+    <v-row>
+      <v-text-field
+        ref="ssvidx"
+        type="number"
+        label="Spectrum-Spectrum Volume Index"
+        v-model.number="ssvidx"
+        hint="The desired spectrum-spectrum volume index, typically [0,1]."
+        persistent-hint
+      ></v-text-field>
+    </v-row>
+    <v-row>
+      <v-text-field
+        ref="wavemin"
+        type="number"
+        label="Minimum Wavelength"
+        v-model.number="wavemin"
+        hint="The desired minimum wavelength."
+        persistent-hint
+      ></v-text-field>
+    </v-row>
+    <v-row>
+      <v-text-field
+        ref="wavemax"
+        type="number"
+        label="Maximum Wavelength"
+        v-model.number="wavemax"
+        hint="The desired maximum wavelength."
+        persistent-hint
+      ></v-text-field>
+    </v-row>
+    <v-row>
+      <v-text-field
+        ref="pccut"
+        type="number"
+        label="Flux Percentile Cut"
+        v-model.number="pccut"
+        hint="The minimum flux percentile to be heard."
+        persistent-hint
+      ></v-text-field>
+    </v-row>
+
+    <v-row>
+        <plugin-action-button
+        @click="sonify_cube"
+      >
+        Sonify data
+      </plugin-action-button>
+    </v-row>
+ </j-tray-plugin>
+</template>

jdaviz/configs/cubeviz/plugins/tools.py

@@ -13,6 +13,7 @@
 from jdaviz.core.tools import PanZoom, BoxZoom, SinglePixelRegion
 from jdaviz.core.marks import PluginLine
 
+
 __all__ = []
 
 ICON_DIR = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'data', 'icons')
@@ -117,22 +118,26 @@ def activate(self):
         # Store these so we can revert to previous user-set zoom after preview view
         sv_state = self._spectrum_viewer.state
         self._previous_bounds = [sv_state.x_min, sv_state.x_max, sv_state.y_min, sv_state.y_max]
+
         super().activate()
 
     def deactivate(self):
         self.viewer.remove_event_callback(self.on_mouse_move)
         self._reset_spectrum_viewer_bounds()
+        self.viewer.stop_stream()
         super().deactivate()
 
     def on_mouse_move(self, data):
         if data['event'] == 'mouseleave':
             self._mark.visible = False
             self._reset_spectrum_viewer_bounds()
+            self.viewer.stop_stream()
             return
 
         x = int(np.round(data['domain']['x']))
         y = int(np.round(data['domain']['y']))
 
+        # some alternative scaling
         # Use the selected layer from coords_info as long as it's 3D
         coords_dataset = self.viewer.session.application._tools['g-coords-info'].dataset.selected
         if coords_dataset == 'auto':
@@ -166,6 +171,7 @@ def on_mouse_move(self, data):
         if x >= spectrum.flux.shape[0] or x < 0 or y >= spectrum.flux.shape[1] or y < 0:
             self._reset_spectrum_viewer_bounds()
             self._mark.visible = False
+            self.viewer.stop_stream()
         else:
             y_values = spectrum.flux[x, y, :]
             if np.all(np.isnan(y_values)):
@@ -175,3 +181,6 @@ def on_mouse_move(self, data):
             self._mark.visible = True
             self._spectrum_viewer.state.y_max = np.nanmax(y_values.value) * 1.2
             self._spectrum_viewer.state.y_min = np.nanmin(y_values.value) * 0.8
+
+            self.viewer.start_stream()
+            self.viewer.update_cube(x, y)

jdaviz/configs/cubeviz/plugins/viewers.py

@@ -10,6 +10,9 @@
 from jdaviz.configs.specviz.plugins.viewers import SpecvizProfileView
 from jdaviz.core.freezable_state import FreezableBqplotImageViewerState
 
+from jdaviz.configs.cubeviz.plugins.cube_listener import CubeListenerData
+import sounddevice as sd
+
 __all__ = ['CubevizImageView', 'CubevizProfileView',
            'WithSliceIndicator', 'WithSliceSelection']
 
@@ -184,6 +187,9 @@ def __init__(self, *args, **kwargs):
         # Hide axes by default
         self.state.show_axes = False
 
+        self.audified_cube = None
+        self.stream = None
+
     @property
     def _default_spectrum_viewer_reference_name(self):
         return self.jdaviz_helper._default_spectrum_viewer_reference_name
@@ -225,6 +231,44 @@ def data(self, cls=None):
                 if hasattr(layer_state, 'layer') and
                 isinstance(layer_state.layer, BaseData)]
 
+    def start_stream(self):
+        if self.stream:
+            self.stream.start()
+
+    def stop_stream(self):
+        if self.stream:
+            self.stream.stop()
+
+    def update_cube(self, x, y):
+        if not self.audified_cube or not hasattr(self.audified_cube, 'newsig') or not hasattr(self.audified_cube, 'sigcube'):
+            return
+        self.audified_cube.newsig = self.audified_cube.sigcube[x, y, :]
+        self.audified_cube.cbuff = True
+
+    def get_sonified_cube(self, sample_rate, buffer_size, assidx, ssvidx, pccut):
+        spectrum = self.active_image_layer.layer.get_object(statistic=None)
+        pc_cube = np.percentile(np.nan_to_num(spectrum.flux.value), np.clip(pccut,0,99), axis=-1)
+
+        # clip zeros and remove NaNs
+        clipped_arr = np.nan_to_num(np.clip(spectrum.flux.value, 0, np.inf), copy=False)
+
+        # make a rough white-light image from the clipped array
+        whitelight = np.expand_dims(clipped_arr.sum(-1), axis=2)
+
+        # subtract any percentile cut
+        clipped_arr -= np.expand_dims(pc_cube, axis=2)
+
+        # and re-clip
+        clipped_arr = np.clip(clipped_arr, 0, np.inf)
+
+        self.audified_cube = CubeListenerData(clipped_arr ** assidx, spectrum.wavelength.value, duration=0.8,
+                                  samplerate=sample_rate, buffsize=buffer_size)
+        self.audified_cube.audify_cube()
+        self.audified_cube.sigcube = (self.audified_cube.sigcube * pow(whitelight / whitelight.max(), ssvidx)).astype('int16')
+        self.stream = sd.OutputStream(samplerate=sample_rate, blocksize=buffer_size, channels=1, dtype='int16', latency='low',
+                                      callback=self.audified_cube.player_callback)
+        self.audified_cube.cbuff = True
+
 
 @viewer_registry("cubeviz-profile-viewer", label="Profile 1D (Cubeviz)")
 class CubevizProfileView(SpecvizProfileView, WithSliceIndicator):