diff --git a/pom.xml b/pom.xml index 003e4ab6f..af4fd9cff 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ 2.11.0 provided - 3.0.0 + 3.1.0-SNAPSHOT 5.4.0 3.1.6 2.2.20 @@ -494,6 +494,16 @@ dbptk-bindings-siard2-1 2.0.0 + + com.databasepreservation + dbptk-bindings-siarddk-128 + 2.0.0 + + + com.databasepreservation + dbptk-bindings-siarddk-1007 + 2.0.0 + com.databasepreservation dbptk-model diff --git a/src/main/java/com/databasepreservation/DBVTK.java b/src/main/java/com/databasepreservation/DBVTK.java index 07d1a9660..540e2f405 100644 --- a/src/main/java/com/databasepreservation/DBVTK.java +++ b/src/main/java/com/databasepreservation/DBVTK.java @@ -132,7 +132,7 @@ public FilterRegistrationBean casSingleSignOutFilter() { FilterRegistrationBean registrationBean = new FilterRegistrationBean<>(); registrationBean.setFilter(new OnOffFilter()); registrationBean.setName("CasSingleSignOutFilter"); - registrationBean.addInitParameter("inner-filter-class", "org.jasig.cas.client.session.SingleSignOutFilter"); + registrationBean.addInitParameter("inner-filter-class", "org.apereo.cas.client.session.SingleSignOutFilter"); registrationBean.addInitParameter("config-prefix", "ui.filter.cas"); registrationBean.addInitParameter("casServerUrlPrefix", "http://localhost:8888/cas"); registrationBean.addUrlPatterns("/*"); @@ -151,7 +151,7 @@ public FilterRegistrationBean casValidationFilter() { registrationBean.setFilter(new OnOffFilter()); registrationBean.setName("CasValidationFilter"); registrationBean.addInitParameter("inner-filter-class", - "org.jasig.cas.client.validation.Cas30ProxyReceivingTicketValidationFilter"); + "org.apereo.cas.client.validation.Cas30ProxyReceivingTicketValidationFilter"); registrationBean.addInitParameter("config-prefix", "ui.filter.cas"); registrationBean.addInitParameter("casServerUrlPrefix", "https://localhost:8443/cas"); registrationBean.addInitParameter("serverName", "https://localhost:8888"); @@ -170,7 +170,7 @@ public FilterRegistrationBean casAuthenticationFilter() { FilterRegistrationBean registrationBean = new FilterRegistrationBean<>(); registrationBean.setFilter(new OnOffFilter()); registrationBean.setName("CasAuthenticationFilter"); - registrationBean.addInitParameter("inner-filter-class", "org.jasig.cas.client.authentication.AuthenticationFilter"); + registrationBean.addInitParameter("inner-filter-class", "org.apereo.cas.client.authentication.AuthenticationFilter"); registrationBean.addInitParameter("config-prefix", "ui.filter.cas"); registrationBean.addInitParameter("casServerLoginUrl", "https://localhost:8443/cas/login"); registrationBean.addUrlPatterns("/login"); @@ -184,7 +184,7 @@ public FilterRegistrationBean casRequestWrapperFilter() { registrationBean.setFilter(new OnOffFilter()); registrationBean.setName("CasRequestWrapperFilter"); registrationBean.addInitParameter("inner-filter-class", - "org.jasig.cas.client.util.HttpServletRequestWrapperFilter"); + "org.apereo.cas.client.util.HttpServletRequestWrapperFilter"); registrationBean.addInitParameter("config-prefix", "ui.filter.cas"); registrationBean.addUrlPatterns("/*"); diff --git a/src/main/java/com/databasepreservation/common/api/utils/ExtraMediaType.java b/src/main/java/com/databasepreservation/common/api/utils/ExtraMediaType.java index 40e6a9433..09d47c594 100644 --- a/src/main/java/com/databasepreservation/common/api/utils/ExtraMediaType.java +++ b/src/main/java/com/databasepreservation/common/api/utils/ExtraMediaType.java @@ -28,6 +28,7 @@ private ExtraMediaType() { */ public static final String TEXT_CSV = "text/csv"; public static final String APPLICATION_ZIP = "application/zip"; + public static final String ZIP_FILE_EXTENSION = ".zip"; public static final String APPLICATION_JAVASCRIPT = "application/javascript"; } diff --git a/src/main/java/com/databasepreservation/common/api/v1/CollectionResource.java b/src/main/java/com/databasepreservation/common/api/v1/CollectionResource.java index b7ca5d06c..ef9c336aa 100644 --- a/src/main/java/com/databasepreservation/common/api/v1/CollectionResource.java +++ b/src/main/java/com/databasepreservation/common/api/v1/CollectionResource.java @@ -13,6 +13,7 @@ import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -24,9 +25,14 @@ import java.util.List; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; +import java.util.zip.ZipOutputStream; import com.databasepreservation.common.api.exceptions.RESTException; import com.databasepreservation.common.exceptions.AuthorizationException; +import com.databasepreservation.common.client.models.structure.ViewerCell; +import com.databasepreservation.model.data.Cell; +import com.databasepreservation.common.api.utils.ExtraMediaType; +import com.databasepreservation.model.exception.ModuleException; import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang.StringUtils; import org.roda.core.data.exceptions.*; @@ -173,7 +179,7 @@ public ResponseEntity getReport(@PathVariable(name = "databaseUUID") S } @Override - public StringResponse createCollection(String databaseUUID) { + public StringResponse createCollection(String databaseUUID, String siardVersion) { ControllerAssistant controllerAssistant = new ControllerAssistant() {}; LogEntryState state = LogEntryState.SUCCESS; @@ -197,9 +203,9 @@ public StringResponse createCollection(String databaseUUID) { } final ViewerDatabase database = ViewerFactory.getSolrManager().retrieve(ViewerDatabase.class, databaseUUID); - StringResponse collection = new StringResponse(SIARDController.loadFromLocal(database.getPath(), databaseUUID)); - - return collection; + + return new StringResponse(SIARDController.loadFromLocal(database.getPath(), databaseUUID, siardVersion)); + } catch (GenericException | AuthorizationDeniedException | NotFoundException | AuthorizationException e) { state = LogEntryState.FAILURE; throw new RESTException(e); @@ -529,7 +535,8 @@ public ResponseEntity exportLOB( .equals(row.getCells().get(configTable.getColumnByIndex(columnIndex).getId()).getStoreType())) { return handleExternalLobDownload(configTable, row, columnIndex); } else { - return handleInternalLobDownload(database.getPath(), configTable, row, columnIndex); + String version = ViewerFactory.getSolrManager().retrieve(ViewerDatabase.class, databaseUUID).getVersion(); + return handleInternalLobDownload(database.getPath(), configTable, row, columnIndex, version); } } } catch (NotFoundException | GenericException | IOException | AuthorizationException e) { @@ -598,37 +605,64 @@ private ResponseEntity handleExternalLobDownload(TableSta } private ResponseEntity handleInternalLobDownload(String databasePath, - TableStatus tableConfiguration, ViewerRow row, int columnIndex) throws IOException, GenericException { + TableStatus tableConfiguration, ViewerRow row, int columnIndex, String version) + throws IOException, GenericException { String handlebarsFilename = HandlebarsUtils.applyExportTemplate(row, tableConfiguration, columnIndex); if (ViewerStringUtils.isBlank(handlebarsFilename)) { handlebarsFilename = ViewerConstants.SIARD_RECORD_PREFIX + row.getUuid() + ViewerConstants.SIARD_LOB_FILE_EXTENSION; } - String handlebarsMimeType = HandlebarsUtils.applyMimeTypeTemplate(row, tableConfiguration, columnIndex); + String handlebarsMimeType = HandlebarsUtils.applyMimeTypeTemplate(row, tableConfiguration, columnIndex); if (ViewerStringUtils.isBlank(handlebarsMimeType)) { handlebarsMimeType = tableConfiguration.getColumnByIndex(columnIndex).getApplicationType(); } - if (LobManagerUtils.isLobEmbedded(tableConfiguration, row, columnIndex)) { - // handle lob as embedded - String lobCellValue = LobManagerUtils.getLobCellValue(tableConfiguration, row, columnIndex); - lobCellValue = lobCellValue.replace(ViewerConstants.SIARD_EMBEDDED_LOB_PREFIX, ""); - String decodedString = new String(Base64.decodeBase64(lobCellValue.getBytes())); + if (version.equals(ViewerConstants.SIARD_DK_1007) || version.equals(ViewerConstants.SIARD_DK_128)) { + String filePath = row.getCells().get(row.getCells().keySet().toArray()[row.getCells().size() - 1]).getValue(); + Path path = Paths.get(filePath); + if (path.toFile().isDirectory()) { + Path tempZipFile = Files.createTempFile("temp-", ExtraMediaType.ZIP_FILE_EXTENSION); + try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(tempZipFile.toFile()))) { + Files.walk(path).filter(p -> !Files.isDirectory(p)).forEach(p -> { + ZipEntry zipEntry = new ZipEntry(path.relativize(p).toString()); + try { + zos.putNextEntry(zipEntry); + Files.copy(p, zos); + zos.closeEntry(); + } catch (IOException e) { + throw new RESTException(e); + } + }); + } - return ApiUtils.okResponse(new StreamResponse(handlebarsFilename, handlebarsMimeType, - DownloadUtils.stream(new BufferedInputStream(new ByteArrayInputStream(decodedString.getBytes()))))); - } else { - // handle lob as internal on separated folder - ZipFile zipFile = new ZipFile(databasePath); - final ZipEntry entry = zipFile.getEntry(LobManagerUtils.getZipFilePath(tableConfiguration, columnIndex, row)); - if (entry == null) { - throw new GenericException("Zip archive entry is missing"); + return ApiUtils.okResponse(new StreamResponse(handlebarsFilename, handlebarsMimeType, + DownloadUtils.stream(new BufferedInputStream(new FileInputStream(tempZipFile.toFile()))))); + } else { + return ApiUtils.okResponse(new StreamResponse(handlebarsFilename, handlebarsMimeType, + DownloadUtils.stream(new BufferedInputStream(new FileInputStream(filePath))))); } + } else { + if (LobManagerUtils.isLobEmbedded(tableConfiguration, row, columnIndex)) { + // handle lob as embedded + String lobCellValue = LobManagerUtils.getLobCellValue(tableConfiguration, row, columnIndex); + lobCellValue = lobCellValue.replace(ViewerConstants.SIARD_EMBEDDED_LOB_PREFIX, ""); + String decodedString = new String(Base64.decodeBase64(lobCellValue.getBytes())); + + return ApiUtils.okResponse(new StreamResponse(handlebarsFilename, handlebarsMimeType, + DownloadUtils.stream(new BufferedInputStream(new ByteArrayInputStream(decodedString.getBytes()))))); + } else { + // handle lob as internal on separated folder + ZipFile zipFile = new ZipFile(databasePath); + final ZipEntry entry = zipFile.getEntry(LobManagerUtils.getZipFilePath(tableConfiguration, columnIndex, row)); + if (entry == null) { + throw new GenericException("Zip archive entry is missing"); + } - return ApiUtils.okResponse(new StreamResponse(handlebarsFilename, handlebarsMimeType, - DownloadUtils.stream(new BufferedInputStream(zipFile.getInputStream(entry))))); + return ApiUtils.okResponse(new StreamResponse(handlebarsFilename, handlebarsMimeType, + DownloadUtils.stream(new BufferedInputStream(zipFile.getInputStream(entry))))); + } } } diff --git a/src/main/java/com/databasepreservation/common/api/v1/DatabaseResource.java b/src/main/java/com/databasepreservation/common/api/v1/DatabaseResource.java index 0293ae426..c6dfa9153 100644 --- a/src/main/java/com/databasepreservation/common/api/v1/DatabaseResource.java +++ b/src/main/java/com/databasepreservation/common/api/v1/DatabaseResource.java @@ -125,7 +125,7 @@ public IndexResult findAll(FindRequest findRequest, String local } @Override - public StringResponse create(String path) { + public StringResponse create(String path, String siardVersion) { final ControllerAssistant controllerAssistant = new ControllerAssistant() {}; LogEntryState state = LogEntryState.SUCCESS; @@ -133,7 +133,7 @@ public StringResponse create(String path) { try { user = controllerAssistant.checkRoles(request); - return new StringResponse(SIARDController.loadMetadataFromLocal(path)); + return new StringResponse(SIARDController.loadMetadataFromLocal(path, siardVersion)); } catch (GenericException | AuthorizationException e) { state = LogEntryState.FAILURE; throw new RESTException(e); diff --git a/src/main/java/com/databasepreservation/common/api/v1/FileResource.java b/src/main/java/com/databasepreservation/common/api/v1/FileResource.java index 033a38c75..bb88391c5 100644 --- a/src/main/java/com/databasepreservation/common/api/v1/FileResource.java +++ b/src/main/java/com/databasepreservation/common/api/v1/FileResource.java @@ -10,11 +10,14 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Paths; import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import org.roda.core.data.exceptions.AlreadyExistsException; import org.roda.core.data.exceptions.GenericException; @@ -90,8 +93,12 @@ public ResponseEntity getSIARDFile(String filename) { java.nio.file.Path basePath = Paths.get(ViewerConfiguration.getInstance().getViewerConfigurationAsString("/", ViewerConfiguration.PROPERTY_BASE_UPLOAD_PATH)); java.nio.file.Path siardPath = siardFilesPath.resolve(filename); - if (java.nio.file.Files.exists(siardPath) && !java.nio.file.Files.isDirectory(siardPath) - && (ViewerConfiguration.checkPathIsWithin(siardPath, siardFilesPath) + + if (java.nio.file.Files.isDirectory(siardPath)) { + siardPath = zipDirectory(siardPath); + } + + if (java.nio.file.Files.exists(siardPath) && (ViewerConfiguration.checkPathIsWithin(siardPath, siardFilesPath) || ViewerConfiguration.checkPathIsWithin(siardPath, basePath))) { InputStreamResource resource = new InputStreamResource(new FileInputStream(siardPath.toFile())); @@ -104,12 +111,32 @@ public ResponseEntity getSIARDFile(String filename) { } catch (NotFoundException | FileNotFoundException | AuthorizationException e) { state = LogEntryState.FAILURE; throw new RESTException(e); + } catch (IOException e) { + throw new RESTException(e); } finally { // register action controllerAssistant.registerAction(user, state, ViewerConstants.CONTROLLER_FILENAME_PARAM, filename); } } + private java.nio.file.Path zipDirectory(java.nio.file.Path dirPath) throws IOException { + java.nio.file.Path zipFilePath = dirPath.resolveSibling(dirPath.getFileName().toString() + ".zip"); + try (FileOutputStream fos = new FileOutputStream(zipFilePath.toFile()); + ZipOutputStream zos = new ZipOutputStream(fos)) { + java.nio.file.Files.walk(dirPath).filter(path -> !java.nio.file.Files.isDirectory(path)).forEach(path -> { + ZipEntry zipEntry = new ZipEntry(dirPath.relativize(path).toString()); + try { + zos.putNextEntry(zipEntry); + java.nio.file.Files.copy(path, zos); + zos.closeEntry(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + return zipFilePath; + } + @Override public void deleteSiardFile(String filename) { final ControllerAssistant controllerAssistant = new ControllerAssistant() {}; diff --git a/src/main/java/com/databasepreservation/common/client/ViewerConstants.java b/src/main/java/com/databasepreservation/common/client/ViewerConstants.java index d909e3085..a11c7b32b 100644 --- a/src/main/java/com/databasepreservation/common/client/ViewerConstants.java +++ b/src/main/java/com/databasepreservation/common/client/ViewerConstants.java @@ -91,7 +91,7 @@ public class ViewerConstants { public static final String DENORMALIZATION_STATUS_PREFIX = "denormalization-"; public static final String INTERNAL_ZIP_LOB_FOLDER = "lobs/"; - + public static final String SIARDDK_RESEARCH_INDEX_PATH = "/Schemas/standard/researchIndex.xsd"; /* * SOLR CONFIGSETS */ @@ -441,7 +441,9 @@ public class ViewerConstants { public static final String SIARD_V10 = "1.0"; public static final String SIARD_V20 = "2.0"; public static final String SIARD_V21 = "2.1"; - + public static final String SIARD_DK = "dk"; + public static final String SIARD_DK_1007 = "dk-1007"; + public static final String SIARD_DK_128 = "dk-128"; /* * DBPTK Metadata */ diff --git a/src/main/java/com/databasepreservation/common/client/common/helpers/HelperUploadSIARDFile.java b/src/main/java/com/databasepreservation/common/client/common/helpers/HelperUploadSIARDFile.java index 557726ea1..0c7f73e14 100644 --- a/src/main/java/com/databasepreservation/common/client/common/helpers/HelperUploadSIARDFile.java +++ b/src/main/java/com/databasepreservation/common/client/common/helpers/HelperUploadSIARDFile.java @@ -108,7 +108,7 @@ private void uploadMetadataSIARD(String path, FlowPanel panel) { Dialogs.showErrors(messages.errorMessagesOpenFile(path), errorMessage, messages.basicActionClose()); // Toast.showError(messages.errorMessagesOpenFile(), errorMessage); panel.remove(loading); - }).create(path); + }).create(path, ViewerConstants.SIARD_V21); } private void successHandler(Boolean confirm, FlowPanel panel, String databaseUUID, String path) { diff --git a/src/main/java/com/databasepreservation/common/client/common/lists/DatabaseList.java b/src/main/java/com/databasepreservation/common/client/common/lists/DatabaseList.java index 7ef5d800a..e73030fda 100644 --- a/src/main/java/com/databasepreservation/common/client/common/lists/DatabaseList.java +++ b/src/main/java/com/databasepreservation/common/client/common/lists/DatabaseList.java @@ -86,7 +86,7 @@ public SafeHtml getValue(ViewerDatabase database) { Column description = new TooltipColumn() { @Override public SafeHtml getValue(ViewerDatabase database) { - return database != null && database.getMetadata() != null + return database != null && database.getMetadata() != null && database.getMetadata().getDescription() != null ? SafeHtmlUtils.fromString(database.getMetadata().getDescription()) : SafeHtmlUtils.fromString("unknown"); } @@ -104,7 +104,7 @@ public SafeHtml getValue(ViewerDatabase database) { Column dataOwnerColumn = new TooltipColumn() { @Override public SafeHtml getValue(ViewerDatabase database) { - return database != null && database.getMetadata() != null + return database != null && database.getMetadata() != null && database.getMetadata().getDataOwner() != null ? SafeHtmlUtils.fromString(database.getMetadata().getDataOwner()) : SafeHtmlUtils.fromString("unknown"); } diff --git a/src/main/java/com/databasepreservation/common/client/common/visualization/browse/RowPanel.java b/src/main/java/com/databasepreservation/common/client/common/visualization/browse/RowPanel.java index e5581ba1d..d84758e24 100644 --- a/src/main/java/com/databasepreservation/common/client/common/visualization/browse/RowPanel.java +++ b/src/main/java/com/databasepreservation/common/client/common/visualization/browse/RowPanel.java @@ -421,11 +421,29 @@ private void getNestedHTML(ColumnStatus columnStatus) { null, false, new ArrayList<>()); CollectionService.Util.call((IndexResult result) -> { if (result.getTotalCount() >= 1) { + RowField rowField; String json = JSOUtils.cellsToJson(result.getResults().get(0).getCells(), nestedTable); String s = JavascriptUtils.compileTemplate(template, json); - RowField rowField = RowField.createInstance(columnStatus.getCustomName(), new Label(s)); - rowField.addColumnDescription(columnStatus.getCustomDescription()); + if (columnStatus.getTypeName().contains("BINARY LARGE OBJECT")) { + String templateLob = "{{download_label}}"; + if ((database.getPath() == null || database.getPath().isEmpty()) + && !status.getConsolidateProperty().equals(LargeObjectConsolidateProperty.CONSOLIDATED)) { + rowField = RowField.createInstance(new Label(s).getText(), new HTML(messages.rowPanelTextForLobUnavailable())); + } else { + SafeHtml safeHtml = SafeHtmlUtils.EMPTY_SAFE_HTML; + json = JSOUtils.cellsToJson(ViewerConstants.TEMPLATE_LOB_DOWNLOAD_LABEL, messages.row_downloadLOB(), + ViewerConstants.TEMPLATE_LOB_DOWNLOAD_LINK, RestUtils.createExportLobUri(database.getUuid(), + nestedTable.getSchemaName(), nestedTable.getName(), "3309", columnStatus.getColumnIndex())); + safeHtml = SafeHtmlUtils.fromSafeConstant(JavascriptUtils.compileTemplate(templateLob, json)); + + + rowField = RowField.createInstance(columnStatus.getCustomName(), new HTML(safeHtml)); + } + } else { + rowField = RowField.createInstance(columnStatus.getCustomName(), new Label(s)); + } + rowField.addColumnDescription(columnStatus.getCustomDescription()); panel.add(rowField); } }).findRows(database.getUuid(), database.getUuid(), nestedTable.getSchemaName(), nestedTable.getName(), diff --git a/src/main/java/com/databasepreservation/common/client/common/visualization/browse/information/DatabaseInformationPanel.java b/src/main/java/com/databasepreservation/common/client/common/visualization/browse/information/DatabaseInformationPanel.java index db913bb08..9b2a026ee 100644 --- a/src/main/java/com/databasepreservation/common/client/common/visualization/browse/information/DatabaseInformationPanel.java +++ b/src/main/java/com/databasepreservation/common/client/common/visualization/browse/information/DatabaseInformationPanel.java @@ -154,7 +154,7 @@ private void loadOnAccess() { instances.remove(database.getUuid()); HistoryManager.gotoSIARDInfo(database.getUuid()); Dialogs.showErrors(messages.SIARDHomePageDialogTitleForBrowsing(), errorMessage, messages.basicActionClose()); - }).createCollection(database.getUuid()); + }).createCollection(database.getUuid(), database.getVersion()); } } diff --git a/src/main/java/com/databasepreservation/common/client/common/visualization/manager/SIARDPanel/SIARDManagerPage.java b/src/main/java/com/databasepreservation/common/client/common/visualization/manager/SIARDPanel/SIARDManagerPage.java index ac58e1c0b..031e82e4b 100644 --- a/src/main/java/com/databasepreservation/common/client/common/visualization/manager/SIARDPanel/SIARDManagerPage.java +++ b/src/main/java/com/databasepreservation/common/client/common/visualization/manager/SIARDPanel/SIARDManagerPage.java @@ -207,16 +207,21 @@ private void setupFooterButtons() { if (ApplicationType.getType().equals(ViewerConstants.APPLICATION_ENV_DESKTOP)) { message = messages.SIARDHomePageTextForDeleteAllFromDesktop(); } - CommonDialogs.showConfirmDialog(messages.SIARDHomePageDialogTitleForDelete(), message, - messages.basicActionCancel(), messages.basicActionConfirm(), CommonDialogs.Level.DANGER, "500px", - new DefaultAsyncCallback() { - @Override - public void onSuccess(Boolean result) { - if (result) { - deleteAll(); + if (database.getVersion().equals(ViewerConstants.SIARD_DK_1007) || database.getVersion().equals(ViewerConstants.SIARD_DK_128)) { + Dialogs.showInformationDialog(messages.SIARDHomePageDialogTitleForDelete(), + "SIARD deletion only supports SIARD version 2.1.", messages.basicActionUnderstood(), "btn btn-link"); + } else { + CommonDialogs.showConfirmDialog(messages.SIARDHomePageDialogTitleForDelete(), message, + messages.basicActionCancel(), messages.basicActionConfirm(), CommonDialogs.Level.DANGER, "500px", + new DefaultAsyncCallback() { + @Override + public void onSuccess(Boolean result) { + if (result) { + deleteAll(); + } } - } - }); + }); + } } else if (ViewerDatabaseStatus.INGESTING.equals(database.getStatus())) { Dialogs.showInformationDialog(messages.SIARDManagerPageInformationDialogTitle(), messages.SIARDManagerPageTextForWaitForFinishing(), messages.basicActionClose(), "btn btn-link"); diff --git a/src/main/java/com/databasepreservation/common/client/common/visualization/manager/SIARDPanel/navigation/BrowseNavigationPanel.java b/src/main/java/com/databasepreservation/common/client/common/visualization/manager/SIARDPanel/navigation/BrowseNavigationPanel.java index 79433cf4a..b1dfb30fd 100644 --- a/src/main/java/com/databasepreservation/common/client/common/visualization/manager/SIARDPanel/navigation/BrowseNavigationPanel.java +++ b/src/main/java/com/databasepreservation/common/client/common/visualization/manager/SIARDPanel/navigation/BrowseNavigationPanel.java @@ -68,8 +68,11 @@ private void browseButton() { } private void handleBrowseAction() { - if (database.getStatus().equals(ViewerDatabaseStatus.METADATA_ONLY)) { // Initial state - if (database.getVersion().equals(ViewerConstants.SIARD_V21)) { + if (database.getStatus().equals(ViewerDatabaseStatus.METADATA_ONLY)) { + // Initial state + if (database.getVersion().equals(ViewerConstants.SIARD_V21) + || database.getVersion().equals(ViewerConstants.SIARD_DK_1007) + || database.getVersion().equals(ViewerConstants.SIARD_DK_128)) { if (!btnIngestClicked) { btnIngestClicked = true; @@ -86,7 +89,7 @@ private void handleBrowseAction() { HistoryManager.gotoSIARDInfo(database.getUuid()); Dialogs.showErrors(messages.SIARDHomePageDialogTitleForBrowsing(), errorMessage, messages.basicActionClose()); - }).createCollection(database.getUuid()); + }).createCollection(database.getUuid(), database.getVersion()); } } else { Dialogs.showInformationDialog(messages.SIARDHomePageDialogTitleForBrowsing(), diff --git a/src/main/java/com/databasepreservation/common/client/common/visualization/manager/SIARDPanel/navigation/SIARDNavigationPanel.java b/src/main/java/com/databasepreservation/common/client/common/visualization/manager/SIARDPanel/navigation/SIARDNavigationPanel.java index 18204e459..c29273c2c 100644 --- a/src/main/java/com/databasepreservation/common/client/common/visualization/manager/SIARDPanel/navigation/SIARDNavigationPanel.java +++ b/src/main/java/com/databasepreservation/common/client/common/visualization/manager/SIARDPanel/navigation/SIARDNavigationPanel.java @@ -15,6 +15,7 @@ import com.databasepreservation.common.client.common.DefaultAsyncCallback; import com.databasepreservation.common.client.common.NavigationPanel; import com.databasepreservation.common.client.common.dialogs.CommonDialogs; +import com.databasepreservation.common.client.common.dialogs.Dialogs; import com.databasepreservation.common.client.common.fields.GenericField; import com.databasepreservation.common.client.common.fields.MetadataField; import com.databasepreservation.common.client.common.utils.ApplicationType; @@ -66,7 +67,13 @@ public NavigationPanel build() { btnEditMetadata.setText(messages.SIARDHomePageButtonTextEditMetadata()); btnEditMetadata.addStyleName("btn btn-outline-primary btn-edit"); btnEditMetadata.addClickHandler(clickEvent -> { - HistoryManager.gotoSIARDEditMetadata(database.getUuid()); + if (database.getVersion().equals(ViewerConstants.SIARD_DK_1007) + || database.getVersion().equals(ViewerConstants.SIARD_DK_128)) { + Dialogs.showInformationDialog("Edit metadata information", "Metadata edition only supports SIARD version 2.1.", + messages.basicActionUnderstood(), "btn btn-link"); + } else { + HistoryManager.gotoSIARDEditMetadata(database.getUuid()); + } }); // Migration button @@ -109,18 +116,24 @@ public NavigationPanel build() { if (database.getPath() != null && !database.getPath().isEmpty()) { btnDelete.setText(messages.SIARDHomePageButtonTextForDeleteIngested()); btnDelete.addClickHandler(event -> { - if (!database.getStatus().equals(ViewerDatabaseStatus.REMOVING) - && !database.getStatus().equals(ViewerDatabaseStatus.INGESTING)) { - CommonDialogs.showConfirmDialog(messages.SIARDHomePageDialogTitleForDelete(), - messages.SIARDHomePageTextForDeleteSIARD(), messages.basicActionCancel(), messages.basicActionConfirm(), - CommonDialogs.Level.DANGER, "500px", new DefaultAsyncCallback() { - @Override - public void onSuccess(Boolean result) { - if (result) { - delete(); + if (database.getVersion().equals(ViewerConstants.SIARD_DK_1007) + || database.getVersion().equals(ViewerConstants.SIARD_DK_128)) { + Dialogs.showInformationDialog("Delete metadata information", + "Metadata deletion only supports SIARD version 2.1.", messages.basicActionUnderstood(), "btn btn-link"); + } else { + if (!database.getStatus().equals(ViewerDatabaseStatus.REMOVING) + && !database.getStatus().equals(ViewerDatabaseStatus.INGESTING)) { + CommonDialogs.showConfirmDialog(messages.SIARDHomePageDialogTitleForDelete(), + messages.SIARDHomePageTextForDeleteSIARD(), messages.basicActionCancel(), messages.basicActionConfirm(), + CommonDialogs.Level.DANGER, "500px", new DefaultAsyncCallback() { + @Override + public void onSuccess(Boolean result) { + if (result) { + delete(); + } } - } - }); + }); + } } }); } diff --git a/src/main/java/com/databasepreservation/common/client/services/CollectionService.java b/src/main/java/com/databasepreservation/common/client/services/CollectionService.java index 7c7db3773..0fffbb9a1 100644 --- a/src/main/java/com/databasepreservation/common/client/services/CollectionService.java +++ b/src/main/java/com/databasepreservation/common/client/services/CollectionService.java @@ -71,7 +71,8 @@ public static CollectionService call(Consumer callback, Consumer *******************************************************************************/ @RequestMapping(path = "/{databaseUUID}/collection", method = RequestMethod.POST, produces = MediaType.APPLICATION_JSON_VALUE) @Operation(summary = "Creates a collection for a database") - StringResponse createCollection(@PathVariable(name = "databaseUUID") String databaseUUID); + StringResponse createCollection(@PathVariable(name = "databaseUUID") String databaseUUID, + @Parameter(name = "version") @RequestParam(name = "version") String version); @RequestMapping(path = "/{databaseUUID}/collection/{collectionUUID}/status", method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) @Operation(summary = "Retrieves the progress data associated with an action done in the database") diff --git a/src/main/java/com/databasepreservation/common/client/services/DatabaseService.java b/src/main/java/com/databasepreservation/common/client/services/DatabaseService.java index 864efe3c6..09a94e343 100644 --- a/src/main/java/com/databasepreservation/common/client/services/DatabaseService.java +++ b/src/main/java/com/databasepreservation/common/client/services/DatabaseService.java @@ -76,7 +76,8 @@ IndexResult findAll(@RequestBody FindRequest findRequest, @RequestMapping(path = "/create", method = RequestMethod.POST, produces = MediaType.APPLICATION_JSON_VALUE) @Operation(summary = "Creates a database") - StringResponse create(@Parameter(name = "path") @RequestParam(name = "path") String path); + StringResponse create(@Parameter(name = "path") @RequestParam(name = "path") String path, + @Parameter(name = "version") @RequestParam(defaultValue = "2.1", name = "version") String version); @RequestMapping(path = "/{databaseUUID}", method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE) @Operation(summary = "Retrieves a specific database") diff --git a/src/main/java/com/databasepreservation/common/server/controller/SIARDController.java b/src/main/java/com/databasepreservation/common/server/controller/SIARDController.java index 068dd3dd8..18d82e34f 100644 --- a/src/main/java/com/databasepreservation/common/server/controller/SIARDController.java +++ b/src/main/java/com/databasepreservation/common/server/controller/SIARDController.java @@ -20,6 +20,8 @@ import java.util.Map; import java.util.Set; +import com.databasepreservation.modules.siard.SIARDDKEditFactory; +import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.joda.time.DateTime; @@ -79,6 +81,8 @@ import com.databasepreservation.modules.config.ImportConfigurationModuleFactory; import com.databasepreservation.modules.jdbc.in.JDBCImportModule; import com.databasepreservation.modules.siard.SIARD2ModuleFactory; +import com.databasepreservation.modules.siard.SIARDDK1007ModuleFactory; +import com.databasepreservation.modules.siard.SIARDDK128ModuleFactory; import com.databasepreservation.modules.siard.SIARDEditFactory; import com.databasepreservation.modules.siard.SIARDValidateFactory; import com.databasepreservation.modules.viewer.DbvtkModuleFactory; @@ -578,20 +582,22 @@ private static Module getFilterModuleParameters(DatabaseFilterFactory factory) { return module; } - public static String loadMetadataFromLocal(String localPath) throws GenericException { + public static String loadMetadataFromLocal(String localPath, String siardVersion) throws GenericException { String databaseUUID = SolrUtils.randomUUID(); - return loadMetadataFromLocal(databaseUUID, localPath); + return loadMetadataFromLocal(databaseUUID, localPath, siardVersion); } - private static String loadMetadataFromLocal(String databaseUUID, String localPath) throws GenericException { + private static String loadMetadataFromLocal(String databaseUUID, String localPath, String siardVersion) + throws GenericException { Path basePath = Paths.get(ViewerConfiguration.getInstance().getViewerConfigurationAsString("/", ViewerConfiguration.PROPERTY_BASE_UPLOAD_PATH)); Path siardPath = basePath.resolve(localPath); - convertSIARDMetadataToSolr(siardPath, databaseUUID); + convertSIARDMetadataToSolr(siardPath, databaseUUID, siardVersion); return databaseUUID; } - private static void convertSIARDMetadataToSolr(Path siardPath, String databaseUUID) throws GenericException { + private static void convertSIARDMetadataToSolr(Path siardPath, String databaseUUID, String siardVersion) + throws GenericException { validateSIARDLocation(siardPath); LOGGER.info("starting to import metadata database {}", siardPath.toAbsolutePath()); @@ -599,9 +605,15 @@ private static void convertSIARDMetadataToSolr(Path siardPath, String databaseUU try { Reporter reporter = new NoOpReporter(); SIARDEdition siardEdition = SIARDEdition.newInstance(); - - siardEdition.editModule(new SIARDEditFactory()).editModuleParameter(SIARDEditFactory.PARAMETER_FILE, - Collections.singletonList(siardPath.toAbsolutePath().toString())); + if (siardVersion.equals(ViewerConstants.SIARD_DK)) { + siardEdition.editModule(new SIARDDKEditFactory()).editModuleParameter( + SIARDDKEditFactory.PARAMETER_FOLDER, Collections.singletonList(siardPath.toAbsolutePath().toString())); + } else if (siardVersion.equals(ViewerConstants.SIARD_V21)) { + siardEdition.editModule(new SIARDEditFactory()).editModuleParameter(SIARDEditFactory.PARAMETER_FILE, + Collections.singletonList(siardPath.toAbsolutePath().toString())); + } else { + throw new SIARDVersionNotSupportedException(); + } siardEdition.reporter(reporter); @@ -615,7 +627,13 @@ private static void convertSIARDMetadataToSolr(Path siardPath, String databaseUU viewerDatabase.setUuid(databaseUUID); viewerDatabase.setPath(siardPath.toAbsolutePath().toString()); - viewerDatabase.setSize(siardPath.toFile().length()); + + if (siardVersion.equals(ViewerConstants.SIARD_DK)) { + viewerDatabase.setSize(FileUtils.sizeOfDirectory(siardPath.toFile())); + } else { + viewerDatabase.setSize(siardPath.toFile().length()); + } + viewerDatabase.setVersion(siardEdition.getSIARDVersion()); viewerDatabase.setAvailableToSearchAll(ViewerConfiguration.getInstance().getViewerConfigurationAsBoolean(true, ViewerConfiguration.SIARD_AVAILABLE_TO_SEARCH_ALL)); @@ -666,13 +684,14 @@ private static void validateSIARDLocation(Path siardPath) throws GenericExceptio } } - public static String loadFromLocal(String localPath, String databaseUUID) throws GenericException { + public static String loadFromLocal(String localPath, String databaseUUID, String siardVersion) + throws GenericException { LOGGER.info("Preparing the SIARD to be browsable ({})", databaseUUID); Path basePath = Paths.get(ViewerConfiguration.getInstance().getViewerConfigurationAsString("/", ViewerConfiguration.PROPERTY_BASE_UPLOAD_PATH)); try { Path siardPath = basePath.resolve(localPath); - convertSIARDtoSolr(siardPath, databaseUUID); + convertSIARDtoSolr(siardPath, databaseUUID, siardVersion); LOGGER.info("Conversion to SIARD successful, database: {}", databaseUUID); } catch (GenericException e) { LOGGER.error("Conversion to SIARD failed for database {}", databaseUUID, e); @@ -681,7 +700,8 @@ public static String loadFromLocal(String localPath, String databaseUUID) throws return databaseUUID; } - private static void convertSIARDtoSolr(Path siardPath, String databaseUUID) throws GenericException { + private static void convertSIARDtoSolr(Path siardPath, String databaseUUID, String siardVersion) + throws GenericException { validateSIARDLocation(siardPath); LOGGER.info("starting to convert database {}", siardPath.toAbsolutePath()); @@ -696,9 +716,21 @@ private static void convertSIARDtoSolr(Path siardPath, String databaseUUID) thro // XXX remove this workaround after fix of NPE databaseMigration.filterFactories(new ArrayList<>()); - databaseMigration.importModule(new SIARD2ModuleFactory()) - .importModuleParameter(SIARD2ModuleFactory.PARAMETER_FILE, siardPath.toAbsolutePath().toString()) - .importModuleParameter(SIARD2ModuleFactory.PARAMETER_IGNORE_LOBS, "true"); + if (siardVersion.equals(ViewerConstants.SIARD_DK_128)) { + databaseMigration.importModule(new SIARDDK128ModuleFactory()) + .importModuleParameter(SIARDDK128ModuleFactory.PARAMETER_FOLDER, siardPath.toAbsolutePath().toString()) + .importModuleParameter(SIARDDK128ModuleFactory.PARAMETER_AS_SCHEMA, "public"); + } else if (siardVersion.equals(ViewerConstants.SIARD_DK_1007)) { + databaseMigration.importModule(new SIARDDK1007ModuleFactory()) + .importModuleParameter(SIARDDK1007ModuleFactory.PARAMETER_FOLDER, siardPath.toAbsolutePath().toString()) + .importModuleParameter(SIARDDK1007ModuleFactory.PARAMETER_AS_SCHEMA, "public"); + } else if (siardVersion.equals(ViewerConstants.SIARD_V21)) { + databaseMigration.importModule(new SIARD2ModuleFactory()) + .importModuleParameter(SIARD2ModuleFactory.PARAMETER_FILE, siardPath.toAbsolutePath().toString()) + .importModuleParameter(SIARD2ModuleFactory.PARAMETER_IGNORE_LOBS, "true"); + } else { + throw new GenericException("SIARD version not supported"); + } databaseMigration.exportModule(new DbvtkModuleFactory()) .exportModuleParameter(DbvtkModuleFactory.PARAMETER_DATABASE_UUID, databaseUUID); diff --git a/src/main/java/com/databasepreservation/common/transformers/DenormalizeTransformer.java b/src/main/java/com/databasepreservation/common/transformers/DenormalizeTransformer.java index f94f96c35..da64df206 100644 --- a/src/main/java/com/databasepreservation/common/transformers/DenormalizeTransformer.java +++ b/src/main/java/com/databasepreservation/common/transformers/DenormalizeTransformer.java @@ -243,7 +243,7 @@ private void queryOverRelatedTables(ViewerRow row, RelatedTablesConfiguration re fieldsToReturn.addAll(auxColumns); fieldsToReturn.addAll(columnsToDisplay); - + //TODO USE NESTEDORIGINALUUID FOR LINK IterableIndexResult nestedRows = solrManager.findAllRows(databaseUUID, resultingFilter, null, fieldsToReturn); for (ViewerRow nestedRow : nestedRows) { for (RelatedTablesConfiguration innerRelatedTable : relatedTable.getRelatedTables()) { diff --git a/src/main/java/com/databasepreservation/common/transformers/ToolkitStructure2ViewerStructure.java b/src/main/java/com/databasepreservation/common/transformers/ToolkitStructure2ViewerStructure.java index 85cdb5f59..385056ee4 100644 --- a/src/main/java/com/databasepreservation/common/transformers/ToolkitStructure2ViewerStructure.java +++ b/src/main/java/com/databasepreservation/common/transformers/ToolkitStructure2ViewerStructure.java @@ -7,10 +7,13 @@ */ package com.databasepreservation.common.transformers; +import com.databasepreservation.common.api.utils.ExtraMediaType; import com.databasepreservation.common.client.models.structure.ViewerLobStoreType; import java.io.ByteArrayInputStream; +import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -365,7 +368,10 @@ private static List getViews(ViewerDatabaseFromToolkit vdb, List result = new ArrayList<>(); if (views != null) { for (ViewStructure view : views) { - result.add(getView(vdb, view)); + ViewerView viewerView = getView(vdb, view); + if (viewerView != null) { + result.add(viewerView); + } } } return result; @@ -376,7 +382,11 @@ private static ViewerView getView(ViewerDatabaseFromToolkit vdb, ViewStructure v result.setName(view.getName()); result.setUuid(SolrUtils.randomUUID()); try { - result.setColumns(getColumns(view.getColumns())); + if (view.getColumns() != null) { + result.setColumns(getColumns(view.getColumns())); + } else { + return null; + } } catch (ViewerException e) { LOGGER.error("Could not convert the columns for view {}", view, e); result.setColumns(new ArrayList<>()); @@ -749,7 +759,7 @@ private static ViewerType getType(Type type) throws ViewerException { } public static ViewerRow getRow(CollectionStatus collectionConfiguration, ViewerTable table, Row row, long rowIndex, - String databasePath) { + String databasePath, String siardVersion) { setCurrentTable(table); ViewerRow result = new ViewerRow(); @@ -759,7 +769,7 @@ public static ViewerRow getRow(CollectionStatus collectionConfiguration, ViewerT result.setTableId(table.getId()); result.setTableUUID(table.getUuid()); result.setUuid(rowUUID); - result.setCells(getCells(collectionConfiguration, table, row, databasePath, result)); + result.setCells(getCells(collectionConfiguration, table, row, databasePath, result, siardVersion)); return result; } @@ -770,7 +780,7 @@ private static void setCurrentTable(ViewerTable table) { } private static Map getCells(CollectionStatus collectionConfiguration, ViewerTable table, Row row, - String databasePath, ViewerRow actualViewerRow) { + String databasePath, ViewerRow actualViewerRow, String siardVersion) { Map result = new LinkedHashMap<>(); int colIndex = 0; @@ -779,7 +789,7 @@ private static Map getCells(CollectionStatus collectionConfi String solrColumnName = viewerColumn.getSolrName(); try { result.put(solrColumnName, getCell(collectionConfiguration, table, toolkitCells.get(colIndex), colIndex++, - databasePath, actualViewerRow)); + databasePath, actualViewerRow, siardVersion)); } catch (ViewerException e) { LOGGER.error("Problem converting cell, omitted it (as if it were NULL)", e); } @@ -789,10 +799,11 @@ private static Map getCells(CollectionStatus collectionConfi } private static ViewerCell getCell(CollectionStatus collectionConfiguration, ViewerTable table, Cell cell, - int colIndex, String databasePath, ViewerRow actualViewerRow) throws ViewerException { + int colIndex, String databasePath, ViewerRow actualViewerRow, String siardVersion) throws ViewerException { ViewerCell result = new ViewerCell(); ViewerType columnType = table.getColumns().get(colIndex).getType(); + boolean isSiardDK = false; if (cell instanceof BinaryCell) { BinaryCell binaryCell = (BinaryCell) cell; @@ -810,11 +821,11 @@ private static ViewerCell getCell(CollectionStatus collectionConfiguration, View String index = getRowIndex(cell.getId()); String lobName = ViewerConstants.SIARD_RECORD_PREFIX + index + ViewerConstants.SIARD_LOB_FILE_EXTENSION; actualViewerRow.addLobType( - collectionConfiguration.getTableStatusByTableId(table.getId()).getColumnByIndex(colIndex).getId(), - ViewerLobStoreType.EXTERNALLY); + collectionConfiguration.getTableStatusByTableId(table.getId()).getColumnByIndex(colIndex).getId(), + ViewerLobStoreType.EXTERNALLY); - detectMimeType(actualViewerRow, result, databasePath, collectionConfiguration, table, colIndex, lobName, - true); + detectMimeType(actualViewerRow, result, databasePath, collectionConfiguration, table, colIndex, lobName, true, + isSiardDK); } catch (ModuleException e) { throw new ViewerException(e.getMessage(), e); @@ -832,22 +843,30 @@ private static ViewerCell getCell(CollectionStatus collectionConfiguration, View result.setValue(siardFilesPath.relativize(lobPath).normalize().toString()); collectionConfiguration.getTableStatusByTableId(table.getId()).getColumnByIndex(colIndex).setExternalLob(true); actualViewerRow.addLobType( - collectionConfiguration.getTableStatusByTableId(table.getId()).getColumnByIndex(colIndex).getId(), - ViewerLobStoreType.EXTERNALLY); + collectionConfiguration.getTableStatusByTableId(table.getId()).getColumnByIndex(colIndex).getId(), + ViewerLobStoreType.EXTERNALLY); - detectMimeType(actualViewerRow, result, databasePath, collectionConfiguration, table, colIndex, lobName, false); + detectMimeType(actualViewerRow, result, databasePath, collectionConfiguration, table, colIndex, lobName, false, + isSiardDK); } else { // BLOB is internal to the SIARD but is stored outside the table.xml (Normal) - - String lobName = Paths.get(binaryCell.getFile()).getFileName().toString(); - result.setValue(lobName); + String lobName; + collectionConfiguration.getTableStatusByTableId(table.getId()).getColumnByIndex(colIndex).setExternalLob(false); actualViewerRow.addLobType( collectionConfiguration.getTableStatusByTableId(table.getId()).getColumnByIndex(colIndex).getId(), ViewerLobStoreType.INTERNALLY); - detectMimeType(actualViewerRow, result, databasePath, collectionConfiguration, table, colIndex, lobName, true); + if (siardVersion.equals(ViewerConstants.SIARD_DK_128) || siardVersion.equals(ViewerConstants.SIARD_DK_1007)) { + lobName = binaryCell.getFile(); + isSiardDK = true; + } else { + lobName = Paths.get(binaryCell.getFile()).getFileName().toString(); + } + result.setValue(lobName); + detectMimeType(actualViewerRow, result, databasePath, collectionConfiguration, table, colIndex, lobName, true, + isSiardDK); } } else if (cell instanceof ComposedCell) { ComposedCell composedCell = (ComposedCell) cell; @@ -880,71 +899,88 @@ private static ViewerCell getCell(CollectionStatus collectionConfiguration, View private static void detectMimeType(ViewerRow row, ViewerCell cell, String databasePath, CollectionStatus collectionConfiguration, ViewerTable table, int colIndex, String lobName, - boolean blobIsInsideSiard) { + boolean blobIsInsideSiard, boolean isSiardDK) { try { String mimeType; String fileExtension; InputStream inputStream; TableStatus tableStatus = collectionConfiguration.getTableStatusByTableId(table.getId()); - String siardLobPath = LobManagerUtils.getZipFilePath(tableStatus, colIndex, lobName); - - ZipFile zipFile = new ZipFile(databasePath); - ZipEntry entry = zipFile.getEntry(siardLobPath); - String lobCellValue = cell.getValue(); + ZipFile zipFile = null; + ZipEntry entry = null; + String siardLobPath; - if (entry != null && blobIsInsideSiard) { - inputStream = zipFile.getInputStream(entry); - } else if (blobIsInsideSiard) { - lobCellValue = lobCellValue.replace(ViewerConstants.SIARD_EMBEDDED_LOB_PREFIX, ""); - inputStream = new ByteArrayInputStream(Base64.decodeBase64(lobCellValue.getBytes())); + File lobFile = new File(lobName); + if (lobFile.isDirectory()) { + mimeType = ExtraMediaType.APPLICATION_ZIP; + fileExtension = ExtraMediaType.ZIP_FILE_EXTENSION; } else { - lobCellValue = cell.getValue(); - final Path lobPath = Paths.get(lobCellValue); - final Path completeLobPath = ViewerFactory.getViewerConfiguration().getSIARDFilesPath().resolve(lobPath); - inputStream = Files.newInputStream(completeLobPath); - } - - mimeType = tika.detect(inputStream); - fileExtension = MimeTypes.getDefaultMimeTypes().forName(mimeType).getExtension(); - - if (StringUtils.isAllBlank(fileExtension)) { - try { - if (blobIsInsideSiard) { - if (entry != null) { - inputStream = zipFile.getInputStream(entry); - } else { - inputStream = new ByteArrayInputStream(lobCellValue.getBytes()); - } + String lobCellValue = cell.getValue(); + + if (!isSiardDK) { + siardLobPath = LobManagerUtils.getZipFilePath(tableStatus, colIndex, lobName); + zipFile = new ZipFile(databasePath); + entry = zipFile.getEntry(siardLobPath); + + if (entry != null && blobIsInsideSiard) { + inputStream = zipFile.getInputStream(entry); + } else if (blobIsInsideSiard) { + lobCellValue = lobCellValue.replace(ViewerConstants.SIARD_EMBEDDED_LOB_PREFIX, ""); + inputStream = new ByteArrayInputStream(Base64.decodeBase64(lobCellValue.getBytes())); } else { + lobCellValue = cell.getValue(); final Path lobPath = Paths.get(lobCellValue); final Path completeLobPath = ViewerFactory.getViewerConfiguration().getSIARDFilesPath().resolve(lobPath); inputStream = Files.newInputStream(completeLobPath); } + } else { + inputStream = Files.newInputStream(Paths.get(lobName)); + } + + mimeType = tika.detect(inputStream); + fileExtension = MimeTypes.getDefaultMimeTypes().forName(mimeType).getExtension(); + + if (StringUtils.isAllBlank(fileExtension)) { + try { + if (blobIsInsideSiard) { + if (entry != null) { + inputStream = zipFile.getInputStream(entry); + } else { + inputStream = new ByteArrayInputStream(lobCellValue.getBytes()); + } + } else { + final Path lobPath = Paths.get(lobCellValue); + final Path completeLobPath = ViewerFactory.getViewerConfiguration().getSIARDFilesPath().resolve(lobPath); + inputStream = Files.newInputStream(completeLobPath); + } - AutoDetectParser parser = new AutoDetectParser(); - Metadata metadata = new Metadata(); + AutoDetectParser parser = new AutoDetectParser(); + Metadata metadata = new Metadata(); - Boolean autoDetectParserNoLimit = ViewerFactory.getEnvBoolean("AUTO_DETECT_PARSER_NO_LIMIT", false); + Boolean autoDetectParserNoLimit = ViewerFactory.getEnvBoolean("AUTO_DETECT_PARSER_NO_LIMIT", false); - if (autoDetectParserNoLimit) { - parser.parse(inputStream, new BodyContentHandler(-1), metadata, new ParseContext()); - } else { - parser.parse(inputStream, new BodyContentHandler(), metadata, new ParseContext()); + if (autoDetectParserNoLimit) { + parser.parse(inputStream, new BodyContentHandler(-1), metadata, new ParseContext()); + } else { + parser.parse(inputStream, new BodyContentHandler(), metadata, new ParseContext()); + } + + mimeType = metadata.get("Content-Type"); + fileExtension = MimeTypes.getDefaultMimeTypes().forName(mimeType).getExtension(); + + } catch (SAXException | TikaException e) { + LOGGER.error("Could not calculate mimeType for special extensions in the cell: [{}]", cell.getValue(), e); } + } - mimeType = metadata.get("Content-Type"); - fileExtension = MimeTypes.getDefaultMimeTypes().forName(mimeType).getExtension(); + inputStream.close(); - } catch (SAXException | TikaException e) { - LOGGER.error("Could not calculate mimeType for special extensions in the cell: [{}]", cell.getValue(), e); + if (zipFile != null) { + zipFile.close(); } } - inputStream.close(); - zipFile.close(); - cell.setMimeType(mimeType); cell.setFileExtension(fileExtension); diff --git a/src/main/java/com/databasepreservation/common/utils/LobManagerUtils.java b/src/main/java/com/databasepreservation/common/utils/LobManagerUtils.java index 670f8e312..4e5048822 100644 --- a/src/main/java/com/databasepreservation/common/utils/LobManagerUtils.java +++ b/src/main/java/com/databasepreservation/common/utils/LobManagerUtils.java @@ -7,7 +7,12 @@ */ package com.databasepreservation.common.utils; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; import java.nio.file.Path; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import com.databasepreservation.common.client.ViewerConstants; import com.databasepreservation.common.client.models.status.collection.TableStatus; @@ -48,6 +53,24 @@ public static String getZipFilePath(TableStatus configTable, int columnIndex, St String siardLobFolder = ViewerConstants.SIARD_LOB_FOLDER_PREFIX + (columnIndex + 1); return "content" + "/" + siardSchemaFolder + "/" + siardTableFolder + "/" + siardLobFolder + "/" + recordValue; + + } + + public static Path createZipFromDirectory(Path directoryPath) throws IOException { + Path zipFilePath = directoryPath.resolveSibling(directoryPath.getFileName().toString() + ".zip"); + try (ZipOutputStream zos = new ZipOutputStream(Files.newOutputStream(zipFilePath))) { + Files.walk(directoryPath).filter(path -> !Files.isDirectory(path)).forEach(path -> { + ZipEntry zipEntry = new ZipEntry(directoryPath.relativize(path).toString()); + try { + zos.putNextEntry(zipEntry); + Files.copy(path, zos); + zos.closeEntry(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } + return zipFilePath; } public static Path getConsolidatedPath(ViewerAbstractConfiguration configuration, String databaseUUID, diff --git a/src/main/java/com/databasepreservation/desktop/client/dbptk/wizard/upload/CreateWizardManager.java b/src/main/java/com/databasepreservation/desktop/client/dbptk/wizard/upload/CreateWizardManager.java index 4cbd9ab3c..717e9ef06 100644 --- a/src/main/java/com/databasepreservation/desktop/client/dbptk/wizard/upload/CreateWizardManager.java +++ b/src/main/java/com/databasepreservation/desktop/client/dbptk/wizard/upload/CreateWizardManager.java @@ -502,6 +502,6 @@ private void importSIARDMetadata(String path) { HistoryManager.gotoHome(); Dialogs.showErrors(messages.createSIARDWizardManagerInformationMessagesTitle(), errorMessage, messages.basicActionClose()); - }).create(path); + }).create(path, ViewerConstants.SIARD_V21); } } diff --git a/src/main/java/com/databasepreservation/modules/viewer/DbvtkExportModule.java b/src/main/java/com/databasepreservation/modules/viewer/DbvtkExportModule.java index f3327e2af..410d88102 100644 --- a/src/main/java/com/databasepreservation/modules/viewer/DbvtkExportModule.java +++ b/src/main/java/com/databasepreservation/modules/viewer/DbvtkExportModule.java @@ -141,8 +141,8 @@ public void handleDataOpenTable(String tableId) throws ModuleException { */ @Override public void handleDataRow(Row row) throws ModuleException { - solrManager.addRow(retrieved.getUuid(), - ToolkitStructure2ViewerStructure.getRow(collectionConfiguration, currentTable, row, rowIndex++,retrieved.getPath())); + solrManager.addRow(retrieved.getUuid(), ToolkitStructure2ViewerStructure.getRow(collectionConfiguration, + currentTable, row, rowIndex++, retrieved.getPath(), retrieved.getVersion())); rowsProcessedByTableCounter++; rowCounter++; @@ -153,15 +153,14 @@ public void handleDataRow(Row row) throws ModuleException { } } - /** * Checks if a row process log should be done * */ private boolean shouldLogRowProgress() { - return rowCounter == rowThreshold || - (currentTable.getCountRows() <= rowThreshold && rowsProcessedByTableCounter == currentTable.getCountRows()) || - rowsProcessedByTableCounter == currentTable.getCountRows(); + return rowCounter == rowThreshold + || (currentTable.getCountRows() <= rowThreshold && rowsProcessedByTableCounter == currentTable.getCountRows()) + || rowsProcessedByTableCounter == currentTable.getCountRows(); } /** diff --git a/src/main/java/com/databasepreservation/server/client/browse/upload/SIARDUpload.java b/src/main/java/com/databasepreservation/server/client/browse/upload/SIARDUpload.java index 7b1c091c5..450a486e6 100644 --- a/src/main/java/com/databasepreservation/server/client/browse/upload/SIARDUpload.java +++ b/src/main/java/com/databasepreservation/server/client/browse/upload/SIARDUpload.java @@ -161,7 +161,7 @@ public void onSuccess(String id) { // Toast.showError("Cannot create SIARD", PathUtils.getFileName(path)); item.addClassName("error"); doneItemLoadHandler(item, errorMessage, null); - }).create(path); + }).create(path, ViewerConstants.SIARD_V21); } }); } else { diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml index 7cd2d4ac6..753234774 100644 --- a/src/main/webapp/WEB-INF/web.xml +++ b/src/main/webapp/WEB-INF/web.xml @@ -83,7 +83,7 @@ com.databasepreservation.common.filter.OnOffFilter inner-filter-class - org.jasig.cas.client.session.SingleSignOutFilter + org.apereo.cas.client.session.SingleSignOutFilter config-prefix @@ -100,7 +100,7 @@ - org.jasig.cas.client.session.SingleSignOutHttpSessionListener + org.apereo.cas.client.session.SingleSignOutHttpSessionListener @@ -109,7 +109,7 @@ com.databasepreservation.common.filter.OnOffFilter inner-filter-class - org.jasig.cas.client.validation.Cas30ProxyReceivingTicketValidationFilter + org.apereo.cas.client.validation.Cas30ProxyReceivingTicketValidationFilter config-prefix @@ -150,7 +150,7 @@ com.databasepreservation.common.filter.OnOffFilter inner-filter-class - org.jasig.cas.client.authentication.AuthenticationFilter + org.apereo.cas.client.authentication.AuthenticationFilter config-prefix @@ -175,7 +175,7 @@ com.databasepreservation.common.filter.OnOffFilter inner-filter-class - org.jasig.cas.client.util.HttpServletRequestWrapperFilter + org.apereo.cas.client.util.HttpServletRequestWrapperFilter config-prefix