Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Bxc 4691 chompb cropping #1840

Merged
merged 8 commits into from
Dec 4, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
package edu.unc.lib.boxc.web.admin.controllers;

import edu.unc.lib.boxc.auth.fcrepo.models.AgentPrincipalsImpl;
import edu.unc.lib.boxc.auth.fcrepo.services.GroupsThreadStore;
import edu.unc.lib.boxc.web.admin.controllers.processing.ChompbPreIngestService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.InputStreamResource;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Controller;
Expand All @@ -16,6 +20,8 @@

import java.io.IOException;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;

import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;

Expand All @@ -24,6 +30,7 @@
*/
@Controller
public class ChompbController {
private static final Logger log = LoggerFactory.getLogger(ChompbController.class);
@Autowired
ChompbPreIngestService chompbPreIngestService;

Expand Down Expand Up @@ -72,6 +79,23 @@ public ResponseEntity<InputStreamResource> getProcessingResults(@PathVariable("p
.body(resource);
}

@RequestMapping(value = "chompb/project/{projectName}/action/velocicroptor", method = RequestMethod.POST)
public @ResponseBody ResponseEntity<Object> startCropping(@PathVariable("projectName") String projectName) {
String userEmail = GroupsThreadStore.getEmail();
Map<String, Object> result = new HashMap<>();
result.put("action", "Start cropping for project " + projectName);
try {
var agentPrincipals = AgentPrincipalsImpl.createFromThread();
chompbPreIngestService.startCropping(agentPrincipals, projectName, userEmail);
} catch (Exception e){
log.error("Failed to start cropping for project {}", projectName, e);
result.put("status", "Chompb command failed");
return new ResponseEntity<>(result, HttpStatus.INTERNAL_SERVER_ERROR);
}
result.put("timestamp", System.currentTimeMillis());
return new ResponseEntity<>(result, HttpStatus.OK);
}

private MediaType getMediaType(String filename) {
if (filename.endsWith(".json")) {
return MediaType.APPLICATION_JSON;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,12 @@
public class ChompbPreIngestService {
private GlobalPermissionEvaluator globalPermissionEvaluator;
private Path baseProjectsPath;
private String serviceKeyPath;
private String serviceUser;
private static final Set<String> VALID_FILENAMES = Set.of("data.json", "data.csv");

/**
* List all of the chompb projects in the base projects path
* List all the chompb projects in the base projects path
*
* @param agent
* @return output of the list projects command, which is a json string
Expand All @@ -35,6 +37,17 @@ public String getProjectLists(AgentPrincipals agent) {
return executeChompbCommand("chompb", "-w", baseProjectsPath.toAbsolutePath().toString(), "list_projects");
}

public String startCropping(AgentPrincipals agent, String projectName, String email) {
assertHasPermission(agent);

return executeChompbCommand("chompb", "process_source_files",
"--action", "velocicroptor",
"-w", baseProjectsPath.resolve(projectName).toAbsolutePath().toString(),
"-k", serviceKeyPath,
"--user", serviceUser,
"--email", email);
}

protected String executeChompbCommand(String... command) {
StringBuilder output = new StringBuilder();
String outputString;
Expand Down Expand Up @@ -114,4 +127,12 @@ public void setGlobalPermissionEvaluator(GlobalPermissionEvaluator globalPermiss
public void setBaseProjectsPath(Path baseProjectsPath) {
this.baseProjectsPath = baseProjectsPath;
}

public void setServiceKeyPath(String serviceKeyPath) {
this.serviceKeyPath = serviceKeyPath;
}

public void setServiceUser(String serviceUser) {
this.serviceUser = serviceUser;
}
}
2 changes: 2 additions & 0 deletions web-admin-app/src/main/webapp/WEB-INF/service-context.xml
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@
<bean id="chompbPreIngestService" class="edu.unc.lib.boxc.web.admin.controllers.processing.ChompbPreIngestService">
<property name="globalPermissionEvaluator" ref="globalPermissionEvaluator" />
<property name="baseProjectsPath" value="${chompb.projects.basePath}" />
<property name="serviceKeyPath" value="${boxctron.service.key}" />
<property name="serviceUser" value="${boxctron.service.user}" />
</bean>

<bean class="edu.unc.lib.boxc.web.common.view.CDRViewResolver" p:suffix=".jsp">
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
package edu.unc.lib.boxc.web.admin.controllers;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.type.MapType;
import edu.unc.lib.boxc.model.api.exceptions.RepositoryException;
import edu.unc.lib.boxc.web.admin.controllers.processing.ChompbPreIngestService;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
Expand All @@ -11,13 +14,18 @@
import org.springframework.test.web.servlet.setup.MockMvcBuilders;

import java.io.ByteArrayInputStream;
import java.util.HashMap;
import java.util.Map;

import static com.fasterxml.jackson.databind.type.TypeFactory.defaultInstance;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.when;
import static org.mockito.MockitoAnnotations.openMocks;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;

/**
Expand Down Expand Up @@ -110,4 +118,32 @@ public void testGetProcessingResultsImage() throws Exception {
var resp = result.getResponse().getContentAsString();
assertEquals(resultContent, resp);
}

@Test
public void testStartCropping() throws Exception {
sharonluong marked this conversation as resolved.
Show resolved Hide resolved
when(chompbPreIngestService.startCropping(any(), any(), any())).thenReturn("test");
MvcResult result = mvc.perform(post("/chompb/project/test_proj/action/velocicroptor"))
.andExpect(status().isOk())
.andReturn();
Map<String, Object> respMap = getMapFromResponse(result);
assertEquals("Start cropping for project test_proj", respMap.get("action"));
}

@Test
public void testStartCroppingReturnsError() throws Exception {
doThrow(new RepositoryException("Command exited with status code 500")).when(chompbPreIngestService)
.startCropping(any(), any(), any());
MvcResult result = mvc.perform(post("/chompb/project/test_proj/action/velocicroptor"))
.andExpect(status().is5xxServerError())
.andReturn();
Map<String, Object> respMap = getMapFromResponse(result);
assertEquals("Start cropping for project test_proj", respMap.get("action"));
assertEquals("Chompb command failed", respMap.get("status"));
}

private static Map<String, Object> getMapFromResponse(MvcResult result) throws Exception {
MapType type = defaultInstance().constructMapType(HashMap.class, String.class, Object.class);
ObjectMapper mapper = new ObjectMapper();
return mapper.readValue(result.getResponse().getContentAsString(), type);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,8 @@

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.when;
import static org.mockito.MockitoAnnotations.openMocks;

Expand Down Expand Up @@ -161,6 +159,31 @@ public void getProcessingResultsInvalidFilenameTest() throws Exception {
() -> service.getProcessingResults(agentPrincipals, PROJ_NAME, VELO_JOB_NAME, "rando.json"));
}

@Test
public void startCroppingSuccessTest() throws InterruptedException {
String expectedOutput = "[ \"success\"]";
InputStream mockInputStream = new ByteArrayInputStream(expectedOutput.getBytes());
when(mockProcess.getInputStream()).thenReturn(mockInputStream);
when(mockProcess.waitFor()).thenReturn(0);
try (MockedConstruction<ProcessBuilder> mocked = Mockito.mockConstruction(ProcessBuilder.class,
(mock, context) -> {
when(mock.start()).thenReturn(mockProcess);
})
){
service.startCropping(agentPrincipals, PROJ_NAME, "[email protected]");
// Verify a single ProcessBuilder instance was created
assertEquals(1,mocked.constructed().size());
}
}

@Test
public void startCroppingNoPermissionTest() {
when(globalPermissionEvaluator.hasGlobalPermission(any(AccessGroupSet.class), eq(Permission.ingest))).thenReturn(false);

assertThrows(AccessRestrictionException.class,
() -> service.startCropping(agentPrincipals, PROJ_NAME, "[email protected]"));
}

private void createDataJson() throws IOException {
createResultFile(JSON_FILENAME, DATA_JSON);
}
Expand Down
4 changes: 3 additions & 1 deletion web-admin-app/src/test/resources/server.properties
Original file line number Diff line number Diff line change
Expand Up @@ -48,4 +48,6 @@ file.max.uploadSize=100000000
repo.dir=/tmp/deposit
spoofing.enabled=false
[email protected]
chompb.projects.basePath=/tmp/projects
chompb.projects.basePath=/tmp/projects
boxctron.service.key=/path/to/key
boxctron.service.user=user
Loading