Prototype for storing single-cell data

gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/ExpressionDataMatrixBuilder.java

@@ -30,6 +30,7 @@
 import ubic.gemma.model.expression.bioAssayData.BioAssayDimension;
 import ubic.gemma.model.expression.bioAssayData.DesignElementDataVector;
 import ubic.gemma.model.expression.bioAssayData.ProcessedExpressionDataVector;
+import ubic.gemma.model.expression.bioAssayData.BulkExpressionDataVector;
 import ubic.gemma.model.expression.designElement.CompositeSequence;
 import ubic.gemma.model.expression.experiment.ExpressionExperiment;
 import ubic.gemma.persistence.util.ChannelUtils;
@@ -50,7 +51,7 @@ public class ExpressionDataMatrixBuilder {
     private static final Log log = LogFactory.getLog( ExpressionDataMatrixBuilder.class.getName() );
     private final Map<ArrayDesign, BioAssayDimension> dimMap = new HashMap<>();
     private final Map<QuantitationType, Integer> numMissingValues = new HashMap<>();
-    private Collection<DesignElementDataVector> vectors;
+    private Collection<BulkExpressionDataVector> vectors;
     private ExpressionExperiment expressionExperiment;
     private Collection<ProcessedExpressionDataVector> processedDataVectors = new HashSet<>();
     private QuantitationTypeData dat = null;
@@ -59,7 +60,7 @@ public class ExpressionDataMatrixBuilder {
     /**
      * @param vectors collection of vectors. They should be thawed first.
      */
-    public ExpressionDataMatrixBuilder( Collection<? extends DesignElementDataVector> vectors ) {
+    public ExpressionDataMatrixBuilder( Collection<? extends BulkExpressionDataVector> vectors ) {
         if ( vectors == null || vectors.size() == 0 )
             throw new IllegalArgumentException( "No vectors" );
         this.vectors = new HashSet<>();
@@ -75,7 +76,7 @@ public ExpressionDataMatrixBuilder( Collection<? extends DesignElementDataVector
     }
 
     public ExpressionDataMatrixBuilder( Collection<ProcessedExpressionDataVector> processedVectors,
-            Collection<? extends DesignElementDataVector> otherVectors ) {
+            Collection<? extends BulkExpressionDataVector> otherVectors ) {
         this.vectors = new HashSet<>();
         this.vectors.addAll( otherVectors );
         this.processedDataVectors = processedVectors;
@@ -87,7 +88,7 @@ public ExpressionDataMatrixBuilder( Collection<ProcessedExpressionDataVector> pr
      * @param vectors raw vectors
      * @return matrix of appropriate type.
      */
-    public static ExpressionDataMatrix<?> getMatrix( Collection<? extends DesignElementDataVector> vectors ) {
+    public static ExpressionDataMatrix<?> getMatrix( Collection<? extends BulkExpressionDataVector> vectors ) {
         if ( vectors == null || vectors.isEmpty() )
             throw new IllegalArgumentException( "No vectors" );
         PrimitiveType representation = vectors.iterator().next().getQuantitationType().getRepresentation();
@@ -100,7 +101,7 @@ public static ExpressionDataMatrix<?> getMatrix( Collection<? extends DesignElem
      * @return matrix of appropriate type.
      */
     private static ExpressionDataMatrix<?> getMatrix( PrimitiveType representation,
-            Collection<? extends DesignElementDataVector> vectors ) {
+            Collection<? extends BulkExpressionDataVector> vectors ) {
         ExpressionDataMatrix<?> expressionDataMatrix;
         if ( representation.equals( PrimitiveType.DOUBLE ) ) {
             expressionDataMatrix = new ExpressionDataDoubleMatrix( vectors );
@@ -291,7 +292,7 @@ public List<BioAssayDimension> getBioAssayDimensions() {
 
         ExpressionDataMatrixBuilder.log.debug( "Checking all vectors to get bioAssayDimensions" );
         Collection<BioAssayDimension> dimensions = new HashSet<>();
-        for ( DesignElementDataVector vector : vectors ) {
+        for ( BulkExpressionDataVector vector : vectors ) {
             ArrayDesign adUsed = this.arrayDesignForVector( vector );
             if ( !dimMap.containsKey( adUsed ) ) {
                 dimMap.put( adUsed, vector.getBioAssayDimension() );
@@ -421,7 +422,7 @@ public List<QuantitationType> getPreferredQTypes() {
         }
 
         for ( BioAssayDimension dimension : dimensions ) {
-            for ( DesignElementDataVector vector : vectors ) {
+            for ( BulkExpressionDataVector vector : vectors ) {
                 if ( !vector.getBioAssayDimension().equals( dimension ) )
                     continue;
 
@@ -566,7 +567,7 @@ private List<QuantitationType> getMissingValueQTypes() {
         List<BioAssayDimension> dimensions = this.getBioAssayDimensions();
 
         for ( BioAssayDimension dim : dimensions ) {
-            for ( DesignElementDataVector vector : vectors ) {
+            for ( BulkExpressionDataVector vector : vectors ) {
 
                 if ( !vector.getBioAssayDimension().equals( dim ) )
                     continue;
@@ -591,13 +592,13 @@ private List<QuantitationType> getMissingValueQTypes() {
     /**
      * @return The 'preferred' data vectors - NOT the processed data vectors!
      */
-    private Collection<DesignElementDataVector> getPreferredDataVectors() {
-        Collection<DesignElementDataVector> result = new HashSet<>();
+    private Collection<BulkExpressionDataVector> getPreferredDataVectors() {
+        Collection<BulkExpressionDataVector> result = new HashSet<>();
 
         List<BioAssayDimension> dimensions = this.getBioAssayDimensions();
         List<QuantitationType> qtypes = this.getPreferredQTypes();
 
-        for ( DesignElementDataVector vector : vectors ) {
+        for ( BulkExpressionDataVector vector : vectors ) {
             if ( !( vector instanceof ProcessedExpressionDataVector ) && dimensions
                     .contains( vector.getBioAssayDimension() ) && qtypes.contains( vector.getQuantitationType() ) )
                 result.add( vector );
@@ -620,7 +621,7 @@ private Collection<ProcessedExpressionDataVector> getProcessedDataVectors() {
         List<BioAssayDimension> dimensions = this.getBioAssayDimensions();
         List<QuantitationType> qtypes = this.getPreferredQTypes();
 
-        for ( DesignElementDataVector vector : vectors ) {
+        for ( BulkExpressionDataVector vector : vectors ) {
             if ( vector instanceof ProcessedExpressionDataVector && dimensions.contains( vector.getBioAssayDimension() )
                     && qtypes.contains( vector.getQuantitationType() ) )
                 result.add( ( ProcessedExpressionDataVector ) vector );
@@ -644,7 +645,7 @@ private QuantitationTypeData getQuantitationTypesNeeded() {
 
             Collection<QuantitationType> checkedQts = new HashSet<>();
 
-            for ( DesignElementDataVector vector : vectors ) {
+            for ( BulkExpressionDataVector vector : vectors ) {
 
                 BioAssayDimension dim = vector.getBioAssayDimension();
 

gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/TwoChannelMissingValuesImpl.java

@@ -31,10 +31,7 @@
 import ubic.gemma.core.datastructure.matrix.ExpressionDataMatrixRowElement;
 import ubic.gemma.model.common.auditAndSecurity.eventType.MissingValueAnalysisEvent;
 import ubic.gemma.model.common.quantitationtype.*;
-import ubic.gemma.model.expression.bioAssayData.BioAssayDimension;
-import ubic.gemma.model.expression.bioAssayData.DesignElementDataVector;
-import ubic.gemma.model.expression.bioAssayData.ProcessedExpressionDataVector;
-import ubic.gemma.model.expression.bioAssayData.RawExpressionDataVector;
+import ubic.gemma.model.expression.bioAssayData.*;
 import ubic.gemma.model.expression.designElement.CompositeSequence;
 import ubic.gemma.model.expression.experiment.ExpressionExperiment;
 import ubic.gemma.persistence.service.common.auditAndSecurity.AuditTrailService;
@@ -122,7 +119,7 @@ public Collection<RawExpressionDataVector> computeMissingValues( ExpressionExper
         timer.stop();
         this.logTimeInfo( timer, procVectors.size() + rawVectors.size() );
 
-        Collection<? extends DesignElementDataVector> builderVectors = new HashSet<>(
+        Collection<? extends BulkExpressionDataVector> builderVectors = new HashSet<>(
                 rawVectors.isEmpty() ? procVectors : rawVectors );
 
         ExpressionDataMatrixBuilder builder = new ExpressionDataMatrixBuilder( builderVectors );

gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataFileServiceImpl.java

@@ -45,6 +45,7 @@
 import ubic.gemma.model.common.quantitationtype.QuantitationType;
 import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
 import ubic.gemma.model.expression.bioAssayData.DesignElementDataVector;
+import ubic.gemma.model.expression.bioAssayData.BulkExpressionDataVector;
 import ubic.gemma.model.expression.designElement.CompositeSequence;
 import ubic.gemma.model.expression.experiment.*;
 import ubic.gemma.model.genome.Taxon;
@@ -481,7 +482,7 @@ public File writeOrLocateDataFile( QuantitationType type, boolean forceWrite ) {
             ExpressionDataFileServiceImpl.log
                     .info( "Creating new quantitation type expression data file: " + f.getName() );
 
-            Collection<DesignElementDataVector> vectors = rawAndProcessedExpressionDataVectorService.findAndThaw( type );
+            Collection<BulkExpressionDataVector> vectors = rawAndProcessedExpressionDataVectorService.findAndThaw( type );
             Collection<ArrayDesign> arrayDesigns = this.getArrayDesigns( vectors );
             Map<CompositeSequence, String[]> geneAnnotations = this.getGeneAnnotationsAsStringsByProbe( arrayDesigns );
 
@@ -564,7 +565,7 @@ public File writeOrLocateJSONDataFile( QuantitationType type, boolean forceWrite
 
             ExpressionDataFileServiceImpl.log.info( "Creating new quantitation type  JSON data file: " + f.getName() );
 
-            Collection<DesignElementDataVector> vectors = rawAndProcessedExpressionDataVectorService.findAndThaw( type );
+            Collection<BulkExpressionDataVector> vectors = rawAndProcessedExpressionDataVectorService.findAndThaw( type );
 
             if ( vectors.size() == 0 ) {
                 ExpressionDataFileServiceImpl.log.warn( "No vectors for " + type );
@@ -1173,7 +1174,7 @@ private File writeDesignMatrix( File file, ExpressionExperiment expressionExperi
         return file;
     }
 
-    private void writeJson( File file, Collection<DesignElementDataVector> vectors ) throws IOException {
+    private void writeJson( File file, Collection<BulkExpressionDataVector> vectors ) throws IOException {
         ExpressionDataMatrix<?> expressionDataMatrix = ExpressionDataMatrixBuilder.getMatrix( vectors );
         try ( Writer writer = new OutputStreamWriter( new GZIPOutputStream( new FileOutputStream( file ) ) ) ) {
             MatrixWriter matrixWriter = new MatrixWriter();
@@ -1211,7 +1212,7 @@ private void writeMatrix( File file, Map<CompositeSequence, String[]> geneAnnota
 
     }
 
-    private void writeVectors( File file, Collection<DesignElementDataVector> vectors,
+    private void writeVectors( File file, Collection<BulkExpressionDataVector> vectors,
             Map<CompositeSequence, String[]> geneAnnotations ) throws IOException {
         ExpressionDataMatrix<?> expressionDataMatrix = ExpressionDataMatrixBuilder.getMatrix( vectors );
         this.writeMatrix( file, geneAnnotations, expressionDataMatrix );

gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BaseExpressionDataMatrix.java

@@ -25,8 +25,8 @@
 import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
 import ubic.gemma.model.expression.bioAssay.BioAssay;
 import ubic.gemma.model.expression.bioAssayData.BioAssayDimension;
-import ubic.gemma.model.expression.bioAssayData.DesignElementDataVector;
 import ubic.gemma.model.expression.bioAssayData.RawExpressionDataVector;
+import ubic.gemma.model.expression.bioAssayData.BulkExpressionDataVector;
 import ubic.gemma.model.expression.biomaterial.BioMaterial;
 import ubic.gemma.model.expression.designElement.CompositeSequence;
 import ubic.gemma.model.expression.experiment.ExpressionExperiment;
@@ -204,7 +204,7 @@ public ExpressionDataMatrixRowElement getRowElement( int index ) {
     }
 
     @SuppressWarnings("unused") // useful interface
-    protected abstract void vectorsToMatrix( Collection<? extends DesignElementDataVector> vectors );
+    protected abstract void vectorsToMatrix( Collection<? extends BulkExpressionDataVector> vectors );
 
     int getColumnIndex( BioAssay bioAssay ) {
         return columnAssayMap.get( bioAssay );
@@ -368,11 +368,11 @@ int setUpColumnElements() {
     /**
      * Selects all the vectors passed in (uses them to initialize the data)
      */
-    void selectVectors( Collection<? extends DesignElementDataVector> vectors ) {
+    void selectVectors( Collection<? extends BulkExpressionDataVector> vectors ) {
         QuantitationType quantitationType = null;
         int i = 0;
-        List<DesignElementDataVector> sorted = this.sortVectorsByDesignElement( vectors );
-        for ( DesignElementDataVector vector : sorted ) {
+        List<BulkExpressionDataVector> sorted = this.sortVectorsByDesignElement( vectors );
+        for ( BulkExpressionDataVector vector : sorted ) {
             if ( this.expressionExperiment == null )
                 this.expressionExperiment = vector.getExpressionExperiment();
             QuantitationType vectorQuantitationType = vector.getQuantitationType();
@@ -397,14 +397,14 @@ void selectVectors( Collection<? extends DesignElementDataVector> vectors ) {
 
     }
 
-    Collection<DesignElementDataVector> selectVectors( Collection<? extends DesignElementDataVector> vectors,
+    Collection<BulkExpressionDataVector> selectVectors( Collection<? extends BulkExpressionDataVector> vectors,
             Collection<QuantitationType> qTypes ) {
         this.quantitationTypes.addAll( qTypes );
 
-        Collection<DesignElementDataVector> vectorsOfInterest = new LinkedHashSet<>();
+        Collection<BulkExpressionDataVector> vectorsOfInterest = new LinkedHashSet<>();
         int i = 0;
 
-        for ( DesignElementDataVector vector : vectors ) {
+        for ( BulkExpressionDataVector vector : vectors ) {
             QuantitationType vectorQuantitationType = vector.getQuantitationType();
             if ( qTypes.contains( vectorQuantitationType ) ) {
                 if ( this.expressionExperiment == null )
@@ -421,14 +421,14 @@ Collection<DesignElementDataVector> selectVectors( Collection<? extends DesignEl
         return vectorsOfInterest;
     }
 
-    Collection<DesignElementDataVector> selectVectors( Collection<? extends DesignElementDataVector> vectors,
+    Collection<BulkExpressionDataVector> selectVectors( Collection<? extends BulkExpressionDataVector> vectors,
             List<QuantitationType> qTypes ) {
         this.quantitationTypes.addAll( qTypes );
-        List<DesignElementDataVector> sorted = this.sortVectorsByDesignElement( vectors );
-        Collection<DesignElementDataVector> vectorsOfInterest = new LinkedHashSet<>();
+        List<BulkExpressionDataVector> sorted = this.sortVectorsByDesignElement( vectors );
+        Collection<BulkExpressionDataVector> vectorsOfInterest = new LinkedHashSet<>();
         int rowIndex = 0;
         for ( QuantitationType soughtType : qTypes ) {
-            for ( DesignElementDataVector vector : sorted ) {
+            for ( BulkExpressionDataVector vector : sorted ) {
                 QuantitationType vectorQuantitationType = vector.getQuantitationType();
                 if ( vectorQuantitationType.equals( soughtType ) ) {
                     if ( this.expressionExperiment == null )
@@ -446,14 +446,14 @@ Collection<DesignElementDataVector> selectVectors( Collection<? extends DesignEl
         return vectorsOfInterest;
     }
 
-    Collection<DesignElementDataVector> selectVectors( Collection<? extends DesignElementDataVector> vectors,
+    Collection<BulkExpressionDataVector> selectVectors( Collection<? extends BulkExpressionDataVector> vectors,
             QuantitationType quantitationType ) {
         this.quantitationTypes.add( quantitationType );
 
-        Collection<DesignElementDataVector> vectorsOfInterest = new LinkedHashSet<>();
+        Collection<BulkExpressionDataVector> vectorsOfInterest = new LinkedHashSet<>();
         int i = 0;
 
-        for ( DesignElementDataVector vector : vectors ) {
+        for ( BulkExpressionDataVector vector : vectors ) {
             QuantitationType vectorQuantitationType = vector.getQuantitationType();
             if ( vectorQuantitationType.equals( quantitationType ) ) {
                 if ( this.expressionExperiment == null )
@@ -470,18 +470,18 @@ Collection<DesignElementDataVector> selectVectors( Collection<? extends DesignEl
         return vectorsOfInterest;
     }
 
-    Collection<DesignElementDataVector> selectVectors( ExpressionExperiment ee, QuantitationType quantitationType ) {
+    Collection<BulkExpressionDataVector> selectVectors( ExpressionExperiment ee, QuantitationType quantitationType ) {
         Collection<RawExpressionDataVector> vectors = ee.getRawExpressionDataVectors();
         return this.selectVectors( quantitationType, vectors );
     }
 
-    private Collection<DesignElementDataVector> selectVectors( QuantitationType quantitationType,
-            Collection<? extends DesignElementDataVector> vectors ) {
-        Collection<DesignElementDataVector> vectorsOfInterest = new LinkedHashSet<>();
+    private Collection<BulkExpressionDataVector> selectVectors( QuantitationType quantitationType,
+            Collection<? extends BulkExpressionDataVector> vectors ) {
+        Collection<BulkExpressionDataVector> vectorsOfInterest = new LinkedHashSet<>();
         this.quantitationTypes.add( quantitationType );
-        List<DesignElementDataVector> sorted = this.sortVectorsByDesignElement( vectors );
+        List<BulkExpressionDataVector> sorted = this.sortVectorsByDesignElement( vectors );
         int i = 0;
-        for ( DesignElementDataVector vector : sorted ) {
+        for ( BulkExpressionDataVector vector : sorted ) {
             QuantitationType vectorQuantitationType = vector.getQuantitationType();
             if ( this.expressionExperiment == null )
                 this.expressionExperiment = vector.getExpressionExperiment();
@@ -512,12 +512,12 @@ private void getBioMaterialGroupsForAssays( Map<BioMaterial, Collection<BioAssay
         }
     }
 
-    private List<DesignElementDataVector> sortVectorsByDesignElement(
-            Collection<? extends DesignElementDataVector> vectors ) {
-        List<DesignElementDataVector> vectorSort = new ArrayList<>( vectors );
-        Comparator<DesignElementDataVector> cmp = Comparator
-                .comparing( ( DesignElementDataVector vector ) -> vector.getDesignElement().getName(), Comparator.nullsLast( Comparator.naturalOrder() ) )
-                .thenComparing( ( DesignElementDataVector vector ) -> vector.getDesignElement().getId() );
+    private List<BulkExpressionDataVector> sortVectorsByDesignElement(
+            Collection<? extends BulkExpressionDataVector> vectors ) {
+        List<BulkExpressionDataVector> vectorSort = new ArrayList<>( vectors );
+        Comparator<BulkExpressionDataVector> cmp = Comparator
+                .comparing( ( BulkExpressionDataVector vector ) -> vector.getDesignElement().getName(), Comparator.nullsLast( Comparator.naturalOrder() ) )
+                .thenComparing( ( BulkExpressionDataVector vector ) -> vector.getDesignElement().getId() );
         vectorSort.sort( cmp );
         return vectorSort;
     }

gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/EmptyExpressionMatrix.java

@@ -20,7 +20,7 @@
 
 import ubic.gemma.model.expression.bioAssay.BioAssay;
 import ubic.gemma.model.expression.bioAssayData.BioAssayDimension;
-import ubic.gemma.model.expression.bioAssayData.DesignElementDataVector;
+import ubic.gemma.model.expression.bioAssayData.BulkExpressionDataVector;
 import ubic.gemma.model.expression.designElement.CompositeSequence;
 
 import java.util.Collection;
@@ -130,7 +130,7 @@ public void set( int row, int column, Object value ) {
     }
 
     @Override
-    protected void vectorsToMatrix( Collection<? extends DesignElementDataVector> vectors ) {
+    protected void vectorsToMatrix( Collection<? extends BulkExpressionDataVector> vectors ) {
         throw new UnsupportedOperationException();
     }