Implement functionality to open spatialdata parquet spots

pom.xml

@@ -247,6 +247,18 @@
             <groupId>net.imagej</groupId>
             <artifactId>imagej</artifactId>
         </dependency>
+        <!-- Logback Classic -->
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+<!--            <version>1.2.11</version>-->
+        </dependency>
+        <!-- Logback Core -->
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-core</artifactId>
+<!--            <version>1.2.11</version>-->
+        </dependency>
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter-api</artifactId>

src/main/java/org/embl/mobie/lib/SpotImageCreator.java

@@ -54,6 +54,7 @@ public SpotImageCreator( SpotDataSource dataSource, MoBIE moBIE )
 
 		Table table = TableOpener.open( tableLocation, tableFormat );
 
+		// TODO: maybe make the spot column names mapping configurable?
 		final TableSawAnnotationCreator< TableSawAnnotatedSpot > annotationCreator = new TableSawAnnotatedSpotCreator( table );
 
 		final TableSawAnnotationTableModel< AnnotatedSpot > tableModel = new TableSawAnnotationTableModel( dataSource.getName(), annotationCreator, tableLocation, tableFormat, table );

src/main/java/org/embl/mobie/lib/data/CollectionTableDataSetter.java

@@ -2,13 +2,16 @@
 
 import ij.IJ;
 import net.imglib2.type.numeric.ARGBType;
-import org.apache.commons.io.FilenameUtils;
 import org.embl.mobie.DataStore;
+import org.embl.mobie.MoBIE;
 import org.embl.mobie.io.ImageDataFormat;
 import org.embl.mobie.io.util.IOHelper;
+import org.embl.mobie.lib.SpotImageCreator;
 import org.embl.mobie.lib.annotation.AnnotatedRegion;
+import org.embl.mobie.lib.annotation.AnnotatedSpot;
 import org.embl.mobie.lib.bdv.blend.BlendingMode;
 import org.embl.mobie.lib.color.ColorHelper;
+import org.embl.mobie.lib.image.SpotAnnotationImage;
 import org.embl.mobie.lib.io.StorageLocation;
 import org.embl.mobie.lib.serialize.*;
 import org.embl.mobie.lib.serialize.display.*;
@@ -62,19 +65,20 @@ public void addToDataset( Dataset dataset )
             if ( rootPath != null )
                 storageLocation.absolutePath = IOHelper.combinePath( rootPath, storageLocation.absolutePath );
 
-            ImageDataFormat imageDataFormat = ImageDataFormat.fromPath( storageLocation.absolutePath );
-            storageLocation.setChannel( getChannel( row ) ); // TODO: Fetch from table or URI? https://forum.image.sc/t/loading-only-one-channel-from-an-ome-zarr/97798
-            String imageName = getName( row );
-            String pixelType = getPixelType( row );
+            String name = getName( row );
+            String dataType = getDataType( row );
 
             Display< ? > display = null;
-            if ( pixelType.equals( CollectionTableConstants.LABELS )  )
+            if ( dataType.equals( CollectionTableConstants.LABELS )  )
             {
+                ImageDataFormat imageDataFormat = getImageDataFormat( row, storageLocation );
+                storageLocation.setChannel( getChannel( row ) ); // TODO: Fetch from table or URI? https://forum.image.sc/t/loading-only-one-channel-from-an-ome-zarr/97798
+
                 TableSource tableSource = getTable( row, rootPath );
 
                 SegmentationDataSource segmentationDataSource =
                         SegmentationDataSource.create(
-                                imageName,
+                                name,
                                 imageDataFormat,
                                 storageLocation,
                                 tableSource
@@ -84,21 +88,45 @@ public void addToDataset( Dataset dataset )
                 dataset.putDataSource( segmentationDataSource );
 
                 display = createSegmentationDisplay(
-                            imageName,
+                            name,
                             row,
                             tableSource != null );
             }
-            else // intensities
+            else if ( dataType.equals( CollectionTableConstants.SPOTS )  )
+            {
+                SpotDataSource spotDataSource = new SpotDataSource(
+                        name,
+                        TableDataFormat.fromPath( storageLocation.absolutePath ),
+                        storageLocation
+                        );
+                dataset.putDataSource( spotDataSource );
+
+                // build spots image from spots table
+                final SpotImageCreator spotImageCreator = new SpotImageCreator(
+                        spotDataSource,
+                        MoBIE.getInstance() );
+
+                SpotAnnotationImage< AnnotatedSpot > spotImage = spotImageCreator.create();
+                DataStore.addImage( spotImage );
+
+                SpotDisplay< AnnotatedRegion > spotDisplay = new SpotDisplay<>( name );
+                spotDisplay.sources = Collections.singletonList( spotImage.getName() );
+                display = spotDisplay;
+            }
+            else // default: intensities
             {
+                ImageDataFormat imageDataFormat = getImageDataFormat( row, storageLocation );
+                storageLocation.setChannel( getChannel( row ) ); // TODO: Fetch from table or URI? https://forum.image.sc/t/loading-only-one-channel-from-an-ome-zarr/97798
+
                 final ImageDataSource imageDataSource = new ImageDataSource(
-                        imageName,
+                        name,
                         imageDataFormat,
                         storageLocation );
                 imageDataSource.preInit( false );
                 dataset.putDataSource( imageDataSource );
 
                 display = createImageDisplay(
-                    imageName,
+                    name,
                     row );
             }
 
@@ -136,11 +164,11 @@ public void addToDataset( Dataset dataset )
 
                     if ( existingDisplay instanceof SegmentationDisplay )
                     {
-                        existingDisplay.getSources().add( imageName );
+                        existingDisplay.getSources().add( name );
                     }
                     else if ( existingDisplay instanceof ImageDisplay )
                     {
-                        ( ( ImageDisplay ) existingDisplay ).addSource( imageName, getContrastLimits( row ) );
+                        ( ( ImageDisplay ) existingDisplay ).addSource( name, getContrastLimits( row ) );
                     }
                 }
                 else
@@ -151,14 +179,12 @@ else if ( existingDisplay instanceof ImageDisplay )
 
                 gridToTransformations
                         .computeIfAbsent( gridId, k -> new ArrayList<>() )
-                        .addAll( getAffineTransformationAsList( Collections.singletonList( imageName ), row ) );
+                        .addAll( getAffineTransformationAsList( Collections.singletonList( name ), row ) );
             }
 
-            IJ.log(" " );
-            IJ.log("Name: " + imageName );
+            IJ.log("\nName: " + name );
             IJ.log("URI: " + storageLocation.absolutePath );
-            IJ.log("Opener: " + imageDataFormat );
-            IJ.log("Type: " + pixelType );
+            IJ.log("Type: " + dataType );
         }
 
         // Create grid views
@@ -215,6 +241,18 @@ else if ( existingDisplay instanceof ImageDisplay )
 
     }
 
+    private static ImageDataFormat getImageDataFormat( Row row, StorageLocation storageLocation )
+    {
+        try {
+            String string = row.getString( CollectionTableConstants.FORMAT );
+            return ImageDataFormat.valueOf( string );
+        }
+        catch ( Exception e )
+        {
+            return ImageDataFormat.fromPath( storageLocation.absolutePath );
+        }
+    }
+
     private static boolean getExclusive( Row row )
     {
         try {
@@ -279,7 +317,7 @@ private static String getNameFromURI( Row row )
         return MoBIEHelper.removeExtension( IOHelper.getFileName( uri ) );
     }
 
-    private static String getPixelType( Row row )
+    private static String getDataType( Row row )
     {
         try
         {

src/main/java/org/embl/mobie/lib/image/SpotAnnotationImage.java

@@ -93,8 +93,6 @@ private void createImage()
 	{
 		final ArrayList< AS > annotations = annData.getTable().annotations();
 		kdTree = new KDTree( annotations, annotations );
-		// TODO: there is a KDTreeFloat implementation in this repo that we could use
-		//   to save memory
 
 		if ( boundingBoxMin == null )
 			boundingBoxMin = kdTree.minAsDoubleArray();

src/main/java/org/embl/mobie/lib/serialize/SpotDataSource.java

@@ -30,7 +30,9 @@
 
 import org.embl.mobie.lib.io.StorageLocation;
 import org.embl.mobie.lib.table.TableDataFormat;
+import org.embl.mobie.lib.table.TableSource;
 
+import java.util.HashMap;
 import java.util.Map;
 
 public class SpotDataSource extends AbstractDataSource
@@ -45,10 +47,16 @@ public class SpotDataSource extends AbstractDataSource
 
 	public String unit; // spatial
 
-
 	public SpotDataSource( String name )
 	{
 		super( name );
 	}
+
+	public SpotDataSource( String name, TableDataFormat tableDataFormat, StorageLocation storageLocation )
+	{
+		super( name );
+		tableData = new HashMap<>();
+		tableData.put( tableDataFormat, storageLocation );
+	}
 }
 

src/main/java/org/embl/mobie/lib/serialize/display/SpotDisplay.java

@@ -36,6 +36,7 @@
 import org.embl.mobie.lib.source.AnnotationType;
 import sc.fiji.bdvpg.services.SourceAndConverterServices;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
 
@@ -75,6 +76,11 @@ public SpotDisplay()
 		super();
 	}
 
+	public SpotDisplay( String name )
+	{
+		super( name );
+	}
+
 	// Gson
 //	public SpotDisplay( String name, double opacity, Map< String, List< String > > sources, String lut, String colorByColumn, Double[] valueLimits, Set< String > selectedSpotIds, boolean showScatterPlot, String[] scatterPlotAxes, List< String > tables, boolean showAsBoundaries, float boundaryThickness  )
 //	{

src/main/java/org/embl/mobie/lib/table/TableDataFormat.java

@@ -48,9 +48,11 @@
 public enum TableDataFormat
 {
 	@SerializedName( TableDataFormatNames.TSV )
-	TSV,  // TSV file
+	TSV,
 	@SerializedName( TableDataFormatNames.CSV )
-	CSV,  // CSV file
+	CSV,
+	@SerializedName( TableDataFormatNames.PARQUET )
+	PARQUET,
 	@SerializedName( TableDataFormatNames.RESULTS_TABLE )
 	ResultsTable,  // ResultsTable in memory
 	@SerializedName( TableDataFormatNames.TABLE )
@@ -77,9 +79,17 @@ public String toString()
 
 	public static TableDataFormat fromPath( String path )
 	{
-		Character delimiter = TableOpener.determineDelimiter( path );
-		if ( delimiter.equals( ',' ) ) return CSV;
-		if ( delimiter.equals( '\t' ) ) return TSV;
+		if ( path.endsWith( ".parquet" ) )
+		{
+			return TableDataFormat.PARQUET;
+		}
+		else
+		{
+			Character delimiter = TableOpener.determineDelimiter( path );
+			if ( delimiter.equals( ',' ) ) return CSV;
+			if ( delimiter.equals( '\t' ) ) return TSV;
+		}
+
 		throw new RuntimeException("Could not determine table format of " + path );
 	}
 

src/main/java/org/embl/mobie/lib/table/TableDataFormatNames.java

@@ -32,6 +32,7 @@ public class TableDataFormatNames
 {
 	public static final String TSV = "tsv";
 	public static final String CSV = "csv";
+	public static final String PARQUET = "parquet";
 	public static final String RESULTS_TABLE = "ResultsTable";
 	public static final String TABLE = "Table";
 }

src/main/java/org/embl/mobie/lib/table/columns/CollectionTableConstants.java

@@ -16,7 +16,7 @@
 public class CollectionTableConstants
 {
     /**
-     * The "uri" column MUST be present, and it MUST point to a valid image dataset.
+     * The "uri" column MUST be present, and it MUST point to a valid image or spots dataset.
      *
      * The same URI MAY be present several times in the same table;
      * this can be useful to display the same data within various views,
@@ -26,11 +26,15 @@ public class CollectionTableConstants
      * - Local files system paths
      * - AWS S3 URLs
      *
-     * Supported image file formats include:
-     * - OME-Zarr
+     * Supported image formats include:
+     * - OME-Zarr (local and on S3)
      * - Everything that Bio-Formats can open
      * - BDV XML (incl. HDF5 and N5)
      * - ilastik hdf5
+     *
+     * Support spots file formats include:
+     * - Parquet
+     * - Tab or comma separated value text files
      */
     public static final String URI = "uri";
 
@@ -65,6 +69,7 @@ public class CollectionTableConstants
      * Supported values:
      * - "intensities"
      * - "labels"
+     * - "spots"
      *
      * Default value: "intensities"
      * The default value will be assigned if
@@ -74,6 +79,8 @@ public class CollectionTableConstants
     public static final String TYPE = "type";
     public static final String INTENSITIES = "intensities";
     public static final String LABELS = "labels";
+    public static final String SPOTS = "spots";
+
 
     /**
      * The "channel" column MAY be present.
@@ -289,4 +296,19 @@ public class CollectionTableConstants
      * - Plate/well based high-throughput microscopy data is a typical use-case
      */
     public static final String GRID = "grid";
+
+    /**
+     * The "format" column MAY be present.
+     *
+     * Supported values:
+     * - OmeZarr
+     *
+     * Default:
+     * If the column is absent or contains an empty or unsupported string the
+     * data format will be determined from the file ending
+     *
+     * Use cases:
+     * - OME-Zarr data that does not contain .ome.zarr in the path
+     */
+    public static final String FORMAT = "format";
 }