diff --git a/docker/swarm/README_zeke_stack_on_linux.md b/docker/swarm/README_zeke_stack_on_linux.md index f18809c8c5..97b7d0bf65 100644 --- a/docker/swarm/README_zeke_stack_on_linux.md +++ b/docker/swarm/README_zeke_stack_on_linux.md @@ -11,7 +11,7 @@ 2. ```bash pushd ../build - ./build.sh --skip-maven --skip-singularity --skip-sudo all localhost:5000/virtualcell dev_zeke + ./build.sh --skip-maven --skip-sudo all localhost:5000/virtualcell dev_zeke popd ``` diff --git a/vcell-core/src/main/java/cbit/vcell/export/server/ExportServiceImpl.java b/vcell-core/src/main/java/cbit/vcell/export/server/ExportServiceImpl.java index 6d1efd8fdd..8e15f1676d 100644 --- a/vcell-core/src/main/java/cbit/vcell/export/server/ExportServiceImpl.java +++ b/vcell-core/src/main/java/cbit/vcell/export/server/ExportServiceImpl.java @@ -53,12 +53,6 @@ public class ExportServiceImpl implements ExportConstants, ExportService { private Hashtable jobRequestIDs = new Hashtable(); private Hashtable completedExportRequests = new Hashtable(); - - private ASCIIExporter asciiExporter = new ASCIIExporter(this); - private IMGExporter imgExporter = new IMGExporter(this); - private RasterExporter rrExporter = new RasterExporter(this); - - private N5Exporter n5Exporter = new N5Exporter(this); /** @@ -271,6 +265,7 @@ public ExportEvent makeRemoteFile(OutputContext outputContext,User user, DataSer switch (exportSpecs.getFormat()) { case CSV: case HDF5: + ASCIIExporter asciiExporter = new ASCIIExporter(this); Collection asciiOut = asciiExporter.makeASCIIData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager); exportOutputs = asciiOut.toArray(new ExportOutput[asciiOut.size()]); if(((ASCIISpecs)exportSpecs.getFormatSpecificSpecs()).isHDF5()) { @@ -289,6 +284,7 @@ public ExportEvent makeRemoteFile(OutputContext outputContext,User user, DataSer case GIF: case FORMAT_JPEG: case ANIMATED_GIF: + IMGExporter imgExporter = new IMGExporter(this); exportOutputs = imgExporter.makeMediaData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,clientTaskStatusSupport,fileDataContainerManager); boolean bOverrideZip = exportOutputs.length == 1; if(bSaveAsZip && !bOverrideZip){ @@ -298,24 +294,29 @@ public ExportEvent makeRemoteFile(OutputContext outputContext,User user, DataSer } case NRRD: // case IMAGEJ: + RasterExporter rrExporter = new RasterExporter(this); NrrdInfo[] nrrdInfos = rrExporter.makeRasterData(outputContext,newExportJob, user, dataServerImpl, exportSpecs, fileDataContainerManager); return makeRemoteFile(fileFormat, exportBaseDir, exportBaseURL, nrrdInfos, exportSpecs, newExportJob, fileDataContainerManager); case UCD: - exportOutputs = rrExporter.makeUCDData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager); + RasterExporter rrExporterUCD = new RasterExporter(this); + exportOutputs = rrExporterUCD.makeUCDData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager); return saveResultsToRemoteFile(fileFormat, exportBaseDir, exportBaseURL, exportOutputs, exportSpecs, newExportJob,fileDataContainerManager); case PLY: - exportOutputs = rrExporter.makePLYWithTexData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager); + RasterExporter rrExporterPLY = new RasterExporter(this); + exportOutputs = rrExporterPLY.makePLYWithTexData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager); return saveResultsToRemoteFile(fileFormat, exportBaseDir, exportBaseURL, exportOutputs, exportSpecs, newExportJob,fileDataContainerManager); case VTK_IMAGE: - exportOutputs = rrExporter.makeVTKImageData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager); + RasterExporter rrExporterVTK = new RasterExporter(this); + exportOutputs = rrExporterVTK.makeVTKImageData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager); return saveResultsToRemoteFile(fileFormat, exportBaseDir, exportBaseURL, exportOutputs, exportSpecs, newExportJob,fileDataContainerManager); case VTK_UNSTRUCT: - exportOutputs = rrExporter.makeVTKUnstructuredData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager); + RasterExporter rrExporterVTKU = new RasterExporter(this); + exportOutputs = rrExporterVTKU.makeVTKUnstructuredData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager); return saveResultsToRemoteFile(fileFormat, exportBaseDir, exportBaseURL, exportOutputs, exportSpecs, newExportJob,fileDataContainerManager); case N5: - n5Exporter.initalizeDataControllers(user, dataServerImpl, (VCSimulationDataIdentifier) exportSpecs.getVCDataIdentifier()); + N5Exporter n5Exporter = new N5Exporter(this, user, dataServerImpl, (VCSimulationDataIdentifier) exportSpecs.getVCDataIdentifier()); ExportOutput exportOutput = n5Exporter.makeN5Data(outputContext, newExportJob, exportSpecs, fileDataContainerManager); - return makeRemoteN5File(fileFormat, n5Exporter.getN5FileNameHash(), exportOutput, exportSpecs, newExportJob); + return makeRemoteN5File(fileFormat, n5Exporter.getN5FileNameHash(), exportOutput, exportSpecs, newExportJob, n5Exporter.getN5FilePathSuffix()); default: throw new DataAccessException("Unknown export format requested"); } @@ -402,7 +403,6 @@ private ExportEvent saveResultsToRemoteFile(String fileFormat, String exportBase } ZipEntry zipEntry = new ZipEntry(filename); zipOut.putNextEntry(zipEntry); - System.out.println("writing entry "+i); exportOutputs[i].writeDataToOutputStream(zipOut,fileDataContainerManager); //zipOut.write(exportOutputs[i].getData()); } else { @@ -475,11 +475,11 @@ private ExportEvent makeRemoteFile_Unzipped(String fileFormat, String exportBase } -private ExportEvent makeRemoteN5File(String fileFormat, String fileName, ExportOutput exportOutput, ExportSpecs exportSpecs, JobRequest newExportJob) throws DataFormatException, IOException{ +private ExportEvent makeRemoteN5File(String fileFormat, String fileName, ExportOutput exportOutput, ExportSpecs exportSpecs, JobRequest newExportJob, String pathSuffix) throws DataFormatException, IOException{ if (exportOutput.isValid()) { completedExportRequests.put(exportSpecs, newExportJob); String url = PropertyLoader.getRequiredProperty(PropertyLoader.s3ExportBaseURLProperty); - url += "/" + n5Exporter.getN5FilePathSuffix(); + url += "/" + pathSuffix; N5Specs n5Specs = (N5Specs) exportSpecs.getFormatSpecificSpecs(); url += "?dataSetName=" + newExportJob.getExportJobID(); if (lg.isTraceEnabled()) lg.trace("ExportServiceImpl.makeRemoteFile(): Successfully exported to file: " + fileName); diff --git a/vcell-core/src/main/java/cbit/vcell/export/server/N5Exporter.java b/vcell-core/src/main/java/cbit/vcell/export/server/N5Exporter.java index 57f6253de0..ff1e56f0c5 100644 --- a/vcell-core/src/main/java/cbit/vcell/export/server/N5Exporter.java +++ b/vcell-core/src/main/java/cbit/vcell/export/server/N5Exporter.java @@ -64,9 +64,12 @@ public class N5Exporter implements ExportConstants { )); - public N5Exporter(ExportServiceImpl exportServiceImpl) { - this.exportServiceImpl = exportServiceImpl; -} + public N5Exporter(ExportServiceImpl exportServiceImpl, User user, DataServerImpl dataServer, VCSimulationDataIdentifier vcSimulationDataIdentifier) { + this.exportServiceImpl = exportServiceImpl; + this.user = user; + this.dataServer = dataServer; + this.vcDataID = vcSimulationDataIdentifier; + } private ExportOutput exportToN5(OutputContext outputContext, long jobID, N5Specs n5Specs, ExportSpecs exportSpecs, FileDataContainerManager fileDataContainerManager) throws Exception { VCUnitDefinition lengthUnit = ModelUnitSystem.createDefaultVCModelUnitSystem().getLengthUnit(); @@ -139,6 +142,7 @@ private ExportOutput exportToN5(OutputContext outputContext, long jobID, N5Specs int timeLoops = 1; double progress = 0; + for (int variableIndex=0; variableIndex < (numVariables -1); variableIndex++){ for (int timeIndex=timeSpecs.getBeginTimeIndex(); timeIndex <= timeSpecs.getEndTimeIndex(); timeIndex++){ int normalizedTimeIndex = timeIndex - timeSpecs.getBeginTimeIndex(); @@ -195,12 +199,6 @@ private boolean containsPostProcessedVariable(String[] variableNames, OutputCont return false; } - public void initalizeDataControllers(User user, DataServerImpl dataServer, VCSimulationDataIdentifier vcSimulationDataIdentifier) throws IOException, DataAccessException { - this.user = user; - this.vcDataID = vcSimulationDataIdentifier; - this.dataServer = dataServer; - } - public VCSimulationDataIdentifier getVcDataID(){return vcDataID;} diff --git a/vcell-core/src/test/java/cbit/vcell/export/N5ExporterTest.java b/vcell-core/src/test/java/cbit/vcell/export/N5ExporterTest.java index ae26485fe4..4a6fee1aff 100644 --- a/vcell-core/src/test/java/cbit/vcell/export/N5ExporterTest.java +++ b/vcell-core/src/test/java/cbit/vcell/export/N5ExporterTest.java @@ -60,6 +60,7 @@ enum TestModels{ private User testUser; private CartesianMesh modelMesh; private double[] times; + private ExportServiceImpl exportService = new ExportServiceImpl(); private ArrayList dataIdentifiers; @@ -129,8 +130,6 @@ public void setUp() throws IOException { previousSimCacheSize = PropertyLoader.getProperty(PropertyLoader.simdataCacheSizeProperty, null); PropertyLoader.setProperty(PropertyLoader.simdataCacheSizeProperty, "100000"); - ExportServiceImpl exportService = new ExportServiceImpl(); - Cachetable cachetable = new Cachetable(10 * Cachetable.minute, Long.parseLong(PropertyLoader.getRequiredProperty(PropertyLoader.simdataCacheSizeProperty))); File primaryDir = new File(PropertyLoader.getRequiredProperty(PropertyLoader.primarySimDataDirInternalProperty)); File secodaryDir = new File(PropertyLoader.getRequiredProperty(PropertyLoader.secondarySimDataDirInternalProperty)); @@ -138,7 +137,6 @@ public void setUp() throws IOException { DataServerImpl dataServer = new DataServerImpl(dataSetController, exportService); testUser = new User("ezequiel23", new KeyValue("258925427")); - n5Exporter = new N5Exporter(exportService); this.dataServer = dataServer; } @@ -202,7 +200,7 @@ private void setExportTestState(TestModels simModel) throws IOException, DataAcc VCSimulationIdentifier vcSimulationIdentifier = new VCSimulationIdentifier(new KeyValue(simModel.simID), testUser); vcDataID = new VCSimulationDataIdentifier(vcSimulationIdentifier, 0); - n5Exporter.initalizeDataControllers(testUser, dataServer, vcDataID); + n5Exporter = new N5Exporter(exportService, testUser, dataServer, vcDataID); dataIdentifiers = new ArrayList<>(Arrays.asList(dataServer.getDataIdentifiers(new OutputContext(new AnnotatedFunction[0]), testUser, vcDataID))); modelMesh = dataServer.getMesh(testUser, vcDataID);