Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Bxc 4416 cleanup temp files #1667

Merged
merged 3 commits into from
Feb 6, 2024
Merged

Bxc 4416 cleanup temp files #1667

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
cefeabf
BXC-4416 clean up temporary files
Feb 1, 2024
2fbc42f
Capture the email attachment when sendEmail is called for testing pur…
bbpennel Feb 5, 2024
afb6458
BXC-4416 fix up tests
Feb 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion ...camel-app/src/main/java/edu/unc/lib/boxc/services/camel/exportxml/ExportXMLProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
Expand Down Expand Up @@ -150,7 +151,10 @@ private void performExport(ExportXMLRequest request, long startTime) throws IOEx
xfop.write(("</" + BULK_MD_TAG + ">").getBytes(UTF_8));
}

sendEmail(zipit(mdExportFile, filename), request, filename, pageStart, pageEnd, totalPids);
File zipFile = zipit(mdExportFile, filename);
sendEmail(zipFile, request, filename, pageStart, pageEnd, totalPids);
cleanupTempFiles(zipFile, mdExportFile);

log.info("Completed exported objects {} through {} for user {} to {}",
pageStart, pageEnd, username, filename);
}
Expand Down Expand Up @@ -346,6 +350,11 @@ private File zipit(File mdExportFile, String filename) throws IOException {
return mdExportZip;
}

private void cleanupTempFiles(File zipFile, File xmlFile) throws IOException {
Files.delete(zipFile.toPath());
Files.delete(xmlFile.toPath());
}

private void sendEmail(File mdExportFile, ExportXMLRequest request, String filename,
int pageStart, int pageEnd, int totalPids) {
String emailBody;
Expand Down
55 changes: 50 additions & 5 deletions ...s-camel-app/src/test/java/edu/unc/lib/boxc/services/camel/exportxml/ExportXMLRouteIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,14 @@
import static edu.unc.lib.boxc.model.fcrepo.ids.DatastreamPids.getTechnicalMetadataPid;
import static edu.unc.lib.boxc.model.fcrepo.ids.RepositoryPaths.getContentRootPid;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.commons.io.FilenameUtils.wildcardMatch;
import static org.apache.jena.rdf.model.ResourceFactory.createResource;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
Expand All @@ -23,7 +26,9 @@
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.List;
Expand All @@ -42,8 +47,10 @@
import org.jdom2.output.Format;
import org.jdom2.output.XMLOutputter;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.jupiter.api.AfterEach;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
Expand Down Expand Up @@ -137,8 +144,9 @@ public class ExportXMLRouteIT {
private ArgumentCaptor<String> bodyCaptor;
@Captor
private ArgumentCaptor<String> filenameCaptor;
@Captor
private ArgumentCaptor<File> attachmentCaptor;
private List<Path> attachmentPaths;
@Rule
public final TemporaryFolder tmpFolder = new TemporaryFolder();

private ContentRootObject rootObj;
private AdminUnit unitObj;
Expand All @@ -158,6 +166,18 @@ public void setup() throws Exception {
agent = new AgentPrincipalsImpl("user", new AccessGroupSetImpl("adminGroup"));
generateBaseStructure();
exportXmlProcessor.setObjectsPerExport(500);

attachmentPaths = new ArrayList<>();
doAnswer(invocation -> {
var attachment = invocation.getArgument(4, File.class);
if (attachment == null) {
return null;
}
var copiedFile = new File(tmpFolder.newFolder(), attachment.getName());
FileUtils.copyFile(attachment, copiedFile);
attachmentPaths.add(copiedFile.toPath());
return null;
}).when(emailHandler).sendEmail(any(), any(), any(), any(), any());
}

@AfterEach
Expand All @@ -179,6 +199,7 @@ public void exportWorksExcludeChildrenTest() throws Exception {
assertTrue("Processing message did not match expectations", result);

assertEmailSent();
assertTempFilesDeleted();

Element rootEl = getExportedDocumentRootEl();

Expand All @@ -202,6 +223,7 @@ public void exportCollectionExcludeChildrenTest() throws Exception {
assertTrue("Processing message did not match expectations", result);

assertEmailSent();
assertTempFilesDeleted();

Element rootEl = getExportedDocumentRootEl();

Expand All @@ -224,6 +246,7 @@ public void exportCollectionIncludeChildrenTest() throws Exception {
assertTrue("Processing message did not match expectations", result);

assertEmailSent();
assertTempFilesDeleted();

Element rootEl = getExportedDocumentRootEl();

Expand Down Expand Up @@ -292,6 +315,7 @@ public void exportUnitIncludeChildrenPagedTest() throws Exception {
assertTrue("Processing message did not match expectations", result);

assertEmailSent(3);
assertTempFilesDeleted();

Element rootEl1 = getExportedDocumentRootEl(1);
assertHasObjectWithoutMods(rootEl1, ResourceType.AdminUnit, unitObj.getPid());
Expand Down Expand Up @@ -325,6 +349,7 @@ public void exportUnitIncludeChildrenPagedExcludeNoDatastreamsTest() throws Exce
assertTrue("Processing message did not match expectations", result);

assertEmailSent(2);
assertTempFilesDeleted();

Element rootEl1 = getExportedDocumentRootEl(1);
assertHasObjectWithMods(rootEl1, ResourceType.Collection, collObj1.getPid());
Expand All @@ -349,6 +374,7 @@ public void exportCollectionExcludeChildrenExcludeNoDatastreamsTest() throws Exc
assertTrue("Processing message did not match expectations", result);

assertEmailSent(1);
assertTempFilesDeleted();

Element rootEl1 = getExportedDocumentRootEl(1);
assertHasObjectWithMods(rootEl1, ResourceType.Collection, collObj1.getPid());
Expand All @@ -371,7 +397,7 @@ public void exportWorkNoModsExcludeNoDatastreamsTest() throws Exception {
assertEmailSent();

assertNull(filenameCaptor.getValue());
assertNull(attachmentCaptor.getValue());
assertTrue(attachmentPaths.isEmpty());
assertEquals("DCR Metadata Export returned no results", subjectCaptor.getValue());
}

Expand All @@ -389,6 +415,7 @@ public void exportWorkWithModsExcludeNoDatastreamsIncChildrenTest() throws Excep
assertTrue("Processing message did not match expectations", result);

assertEmailSent();
assertTempFilesDeleted();

Element rootEl = getExportedDocumentRootEl();

Expand All @@ -411,6 +438,7 @@ public void exportWorkWithModsExcludeNoDatastreamsExcChildrenTest() throws Excep
assertTrue("Processing message did not match expectations", result);

assertEmailSent();
assertTempFilesDeleted();

Element rootEl = getExportedDocumentRootEl();

Expand Down Expand Up @@ -441,6 +469,7 @@ public void exportWorkModsAndFitsTest() throws Exception {
assertTrue("Processing message did not match expectations", result);

assertEmailSent();
assertTempFilesDeleted();

Element rootEl = getExportedDocumentRootEl();

Expand Down Expand Up @@ -476,6 +505,7 @@ public void exportWorkModsAndPremisTest() throws Exception {
assertTrue("Processing message did not match expectations", result);

assertEmailSent();
assertTempFilesDeleted();

Element rootEl = getExportedDocumentRootEl();

Expand Down Expand Up @@ -510,6 +540,7 @@ public void exportWorkModsAndPremisNoModsTest() throws Exception {
assertTrue("Processing message did not match expectations", result);

assertEmailSent();
assertTempFilesDeleted();

Element rootEl = getExportedDocumentRootEl();

Expand Down Expand Up @@ -543,6 +574,7 @@ public void exportCollectionFitsExcludeNoDatastreamTest() throws Exception {
assertTrue("Processing message did not match expectations", result);

assertEmailSent();
assertTempFilesDeleted();

Element rootEl = getExportedDocumentRootEl();

Expand All @@ -565,7 +597,7 @@ private void assertEmailSent() {

private void assertEmailSent(int numberEmails) {
verify(emailHandler, times(numberEmails)).sendEmail(toCaptor.capture(), subjectCaptor.capture(),
bodyCaptor.capture(), filenameCaptor.capture(), attachmentCaptor.capture());
bodyCaptor.capture(), filenameCaptor.capture(), any());
}

private ExportXMLRequest createRequest(boolean exportChildren, boolean excludeNoDs, PID... pids) {
Expand Down Expand Up @@ -602,7 +634,7 @@ private Document getExportedDocument(int page) throws Exception {
String exportFile = filenameCaptor.getAllValues().get(page - 1);
assertTrue("Unexpected export filename: " + exportFile,
exportFile.matches("xml\\_export\\_.*\\_0+" + page + "\\.zip"));
return getExportedDocument(attachmentCaptor.getAllValues().get(page - 1));
return getExportedDocument(attachmentPaths.get(page - 1).toFile());
}

private Document getExportedDocument(File reportZip) throws Exception {
Expand Down Expand Up @@ -658,6 +690,19 @@ private void assertHasObjectWithDatastream(Element rootEl, ResourceType expected
assertEquals(expectedContent.trim(), content);
}

private void assertTempFilesDeleted() {
var tempDir = System.getProperty("java.io.tmpdir");
String wildCardValue = "xml_export*";
try (var stream = Files.list(Paths.get(tempDir))) {
var filters = stream.filter(file ->
wildcardMatch(file.getFileName().toString(), wildCardValue));
var results = filters.collect(Collectors.toList());
assertTrue(results.isEmpty());
} catch (IOException e) {
throw new RuntimeException(e);
}
}

private Element getDatastreamElByType(Element objEl, DatastreamType dsType) {
return objEl.getChildren("datastream").stream()
.filter(e -> e.getAttributeValue("type").equals(dsType.getId()))
Expand Down
Loading