Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Migrate customizations #8

Draft
wants to merge 3 commits into
base: 3.10.0.tamu.1
Choose a base branch
from
Draft

Migrate customizations #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
04cb981
add temp 998 field for holdings info
wwelling Dec 9, 2021
e38f87b
exclude 952 and 998 if item or holdings suppressed respectively
wwelling Jan 14, 2022
89374d8
WIP - resolved merge conflict for MarcWithHoldingsRequestHelper
rmathew1011 Jan 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 103 additions & 11 deletions src/main/java/org/folio/oaipmh/helpers/records/RecordMetadataManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ public class RecordMetadataManager {
private static final String GENERAL_INFO_FIELD_TAG_NUMBER = "999";
private static final String ELECTRONIC_ACCESS_FILED_TAG_NUMBER = "856";
private static final String EFFECTIVE_LOCATION_FILED_TAG_NUMBER = "952";
private static final String HOLDINGS_RECORD_FIELD_TAG_NUMBER = "998";

private static final String INDICATOR_VALUE = "f";
private static final String DISCOVERY_SUPPRESSED_SUBFIELD_CODE = "t";
Expand All @@ -46,6 +47,9 @@ public class RecordMetadataManager {
private static final int SECOND_INDICATOR_INDEX = 1;
private static final String LOCATION = "location";

private static final String PERMANENT_LOCATION = "permanentLocation";
private static final String HOLDINGS_STATEMENTS = "holdingsStatements";

private StorageHelper storageHelper = StorageHelper.getInstance();
private final Map<String, String> indicatorsMap;
private final Predicate<JsonObject> generalInfoFieldPredicate;
Expand Down Expand Up @@ -145,9 +149,10 @@ public JsonObject populateMetadataWithHoldingsData(JsonObject srsInstance,

if (Objects.nonNull(holdings) && CollectionUtils.isNotEmpty(holdings.getList())) {
List<Object> fieldsList = getFieldsForUpdate(srsInstance);
holdings.forEach(holding ->
updateFieldsWithElectronicAccessField((JsonObject) holding, fieldsList, suppressedRecordsProcessing)
);
holdings.forEach(holding -> {
updateFieldsWithElectronicAccessField((JsonObject) holding, fieldsList, suppressedRecordsProcessing);
updateFieldsWithHoldingsRecordField((JsonObject) holding, fieldsList, suppressedRecordsProcessing);
});
}
return srsInstance;
}
Expand All @@ -172,16 +177,19 @@ private void updateFieldsWithItemEffectiveLocationField(JsonObject itemData,
List<Object> marcRecordFields,
boolean suppressedRecordsProcessing) {
Map<String, Object> effectiveLocationSubFields = constructEffectiveLocationSubFieldsMap(itemData);
int subFieldValue = BooleanUtils.isFalse(itemData.getBoolean(INVENTORY_SUPPRESS_DISCOVERY_FIELD)) ? 0 : 1;
if (suppressedRecordsProcessing) {
effectiveLocationSubFields.put(DISCOVERY_SUPPRESSED_SUBFIELD_CODE, calculateDiscoverySuppressedSubfieldValue(itemData));
effectiveLocationSubFields.put(DISCOVERY_SUPPRESSED_SUBFIELD_CODE, subFieldValue);
}
if (subFieldValue == 0) {
FieldBuilder fieldBuilder = new FieldBuilder();
Map<String, Object> effectiveLocationField = fieldBuilder.withFieldTagNumber(EFFECTIVE_LOCATION_FILED_TAG_NUMBER)
.withFirstIndicator(INDICATOR_VALUE)
.withSecondIndicator(INDICATOR_VALUE)
.withSubFields(effectiveLocationSubFields)
.build();
marcRecordFields.add(effectiveLocationField);
}
FieldBuilder fieldBuilder = new FieldBuilder();
Map<String, Object> effectiveLocationField = fieldBuilder.withFieldTagNumber(EFFECTIVE_LOCATION_FILED_TAG_NUMBER)
.withFirstIndicator(INDICATOR_VALUE)
.withSecondIndicator(INDICATOR_VALUE)
.withSubFields(effectiveLocationSubFields)
.build();
marcRecordFields.add(effectiveLocationField);
}

/**
Expand Down Expand Up @@ -301,6 +309,68 @@ private Map<String, Object> constructElectronicAccessSubFieldsMap(JsonObject ite
return electronicAccessSubFields;
}

/**
* Constructs field with subfields which is build from holdings record data. Constructed field has tag number = 999 and both
* indicators has ' ' value.
*
* @param jsonData - json of single item or holding
* @param marcRecordFields - fields list to be updated with new one
* @param suppressedRecordsProcessing - include suppressed flag in 999 field?
*/
private void updateFieldsWithHoldingsRecordField(JsonObject jsonData,
List<Object> marcRecordFields,
boolean suppressedRecordsProcessing) {
Map<String, Object> holdingsRecordSubFields = constructHoldingsRecordSubFieldsMap(jsonData);
int subFieldValue = BooleanUtils.isFalse(jsonData.getBoolean(INVENTORY_SUPPRESS_DISCOVERY_FIELD)) ? 0 : 1;
if (suppressedRecordsProcessing) {
holdingsRecordSubFields.put(DISCOVERY_SUPPRESSED_SUBFIELD_CODE, subFieldValue);
}
if (subFieldValue == 0) {
FieldBuilder fieldBuilder = new FieldBuilder();
Map<String, Object> holdingsRecordField = fieldBuilder.withFieldTagNumber(HOLDINGS_RECORD_FIELD_TAG_NUMBER)
.withFirstIndicator(INDICATOR_VALUE)
.withSecondIndicator(INDICATOR_VALUE)
.withSubFields(holdingsRecordSubFields)
.build();
marcRecordFields.add(holdingsRecordField);
}
}

private Map<String, Object> constructHoldingsRecordSubFieldsMap(JsonObject holdingsData) {
Map<String, Object> holdingsRecordSubFields = new HashMap<>();
JsonObject locationGroup = null;
if (Objects.nonNull(holdingsData.getJsonObject(LOCATION))) {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

--JsonObject locationGroup = null;
++JsonObject locationGroup = holdingsData.getJsonObject(LOCATION);
--if (Objects.nonNull(holdingsData.getJsonObject(LOCATION))) {
++if (Objects.nonNull(locationGroup)) {
--locationGroup = holdingsData.getJsonObject(LOCATION)
++locationGroup = locationGroup

locationGroup = holdingsData.getJsonObject(LOCATION)
.getJsonObject(PERMANENT_LOCATION);
}
JsonObject callNumberGroup = holdingsData.getJsonObject(CALL_NUMBER);
JsonArray holdingsStatementsGroup = holdingsData.getJsonArray(HOLDINGS_STATEMENTS);

addSubFieldGroup(holdingsRecordSubFields, locationGroup, HoldingsRecordSubFields.PERMANENT_LOCATION_NAME);
addSubFieldGroup(holdingsRecordSubFields, callNumberGroup, HoldingsRecordSubFields.CALL_NUMBER);

if (Objects.nonNull(holdingsStatementsGroup)) {
holdingsStatementsGroup.forEach(statementData -> {
if (statementData instanceof JsonObject) {
addSubFieldGroup(holdingsRecordSubFields, (JsonObject) statementData, HoldingsRecordSubFields.STATEMENT);
}
});
}

return holdingsRecordSubFields;
}

private void addSubFieldGroup(Map<String, Object> effectiveLocationSubFields, JsonObject holdingsData,
HoldingsRecordSubFields subFieldGroupProperty) {
if(Objects.nonNull(holdingsData)) {
String subFieldCode = subFieldGroupProperty.getSubFieldCode();
String subFieldValue = holdingsData.getString(subFieldGroupProperty.getJsonPropertyPath());
if (isNotEmpty(subFieldValue)) {
effectiveLocationSubFields.put(subFieldCode, subFieldValue);
}
}
}

/**
* Updates marc general info datafield(tag=999, ind1=ind2='f') with additional subfield which holds data about record discovery
* suppression status. Additional subfield has code = 't' and value = '0' if record is discovery suppressed and '1' at opposite
Expand Down Expand Up @@ -442,4 +512,26 @@ public String getJsonPropertyPath() {
}
}

private enum HoldingsRecordSubFields {
CALL_NUMBER("a", "callNumber"),
PERMANENT_LOCATION_NAME("l", "name"),
STATEMENT("s", "statement");

private String subFieldCode;
private String jsonPropertyPath;

HoldingsRecordSubFields(String subFieldCode, String jsonPropertyPath) {
this.subFieldCode = subFieldCode;
this.jsonPropertyPath = jsonPropertyPath;
}

public String getSubFieldCode() {
return subFieldCode;
}

public String getJsonPropertyPath() {
return jsonPropertyPath;
}
}

}
126 changes: 117 additions & 9 deletions src/main/java/org/folio/oaipmh/processors/MarcWithHoldingsRequestHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -345,11 +345,8 @@ private void setupBatchHttpStream(Promise<?> promise, HttpRequestImpl<Buffer> in
downloadInstancesPromise.complete();
}
});
jsonParser.exceptionHandler(throwable -> responseChecked.future().onSuccess(invalidResponseReceivedAndProcessed -> {
if (invalidResponseReceivedAndProcessed) {
return;
}
logger.error("Error has been occurred at JsonParser while saving instances. Message: {}", throwable.getMessage(),
jsonParser.exceptionHandler(throwable -> {
logger.error("Error has been occurred at JsonParser while reading data from response while setting up batch http stream. Message: {}", throwable.getMessage(),
throwable);
downloadInstancesPromise.complete(throwable);
promise.fail(throwable);
Expand Down Expand Up @@ -378,7 +375,7 @@ private void setupBatchHttpStream(Promise<?> promise, HttpRequestImpl<Buffer> in
}
})
.onFailure(throwable -> {
logger.error("Error has been occurred at JsonParser while reading data from response. Message: {}", throwable.getMessage(),
logger.error("Error has been occurred on failure of inventory http request. Message: {}", throwable.getMessage(),
throwable);
promise.fail(throwable);
});
Expand Down Expand Up @@ -499,8 +496,119 @@ private Handler<AsyncResult<List<Instances>>> handleInstancesDbResponse(Promise<
};
}

private Future<Response> buildRecordsResponse(Request request, String requestId, List<JsonObject> batch, OffsetDateTime lastUpdateDate,
Map<String, JsonObject> srsResponse, boolean firstBatch, String nextInstanceId, boolean deletedRecordSupport, StatisticsHolder statistics) {
private Future<List<JsonObject>> enrichInstances(List<JsonObject> result, Request request) {
Map<String, JsonObject> instances = result.stream()
.collect(LinkedHashMap::new, (map, instance) -> map.put(instance.getString(INSTANCE_ID_FIELD_NAME), instance), Map::putAll);
Promise<List<JsonObject>> completePromise = Promise.promise();
var webClient = WebClientProvider.getWebClient();
var httpRequest = webClient.postAbs(request.getOkapiUrl() + INVENTORY_ITEMS_AND_HOLDINGS_ENDPOINT);
if (request.getOkapiUrl()
.contains("https:")) {
httpRequest.ssl(true);
}
httpRequest.putHeader(OKAPI_TOKEN, request.getOkapiToken());
httpRequest.putHeader(OKAPI_TENANT, TenantTool.tenantId(request.getOkapiHeaders()));
httpRequest.putHeader(ACCEPT, APPLICATION_JSON);
httpRequest.putHeader(CONTENT_TYPE, APPLICATION_JSON);

JsonObject entries = new JsonObject();
entries.put(INSTANCE_IDS_ENRICH_PARAM_NAME, new JsonArray(new ArrayList<>(instances.keySet())));
entries.put(SKIP_SUPPRESSED_FROM_DISCOVERY_RECORDS, isSkipSuppressed(request));

var jsonParser = JsonParser.newParser()
.objectValueMode();
jsonParser.handler(event -> {
JsonObject itemsAndHoldingsFields = event.objectValue();
String instanceId = itemsAndHoldingsFields.getString(INSTANCE_ID_FIELD_NAME);
JsonObject instance = instances.get(instanceId);
if (instance != null) {
enrichDiscoverySuppressed(itemsAndHoldingsFields, instance);
instance.put(RecordMetadataManager.ITEMS_AND_HOLDINGS_FIELDS, itemsAndHoldingsFields);
// case when no items
if (itemsAndHoldingsFields.getJsonArray(ITEMS)
.isEmpty()) {
enrichOnlyEffectiveLocationEffectiveCallNumberFromHoldings(instance);
} else {
adjustItems(instance);
}
} else {
logger.info("Instance with instanceId {} wasn't in the request.", instanceId);
}
});
jsonParser.exceptionHandler(throwable -> {
logger.error("Error has been occurred at JsonParser while reading data from response while enriching instances. Message:{}", throwable.getMessage(),
throwable);
completePromise.fail(throwable);
});

httpRequest.as(BodyCodec.jsonStream(jsonParser))
.sendBuffer(entries.toBuffer())
.onSuccess(httpResponse -> {
logger.info("Response for items and holdings {}: {}", httpResponse.statusCode(), httpResponse.statusMessage());
if (httpResponse.statusCode() != 200) {
String errorFromStorageMessage = getErrorFromStorageMessage("inventory-storage",
request.getOkapiUrl() + INVENTORY_ITEMS_AND_HOLDINGS_ENDPOINT, httpResponse.statusMessage());
String errorMessage = errorFromStorageMessage + httpResponse.statusCode();
logger.error(errorMessage);
completePromise.fail(new IllegalStateException(errorFromStorageMessage));
}
completePromise.complete(new ArrayList<>(instances.values()));
})
.onFailure(e -> {
logger.error(e.getMessage());
completePromise.fail(e);
});
return completePromise.future();
}

private void enrichDiscoverySuppressed(JsonObject itemsandholdingsfields, JsonObject instance) {
if (Boolean.parseBoolean(instance.getString("suppressFromDiscovery")))
for (Object item : itemsandholdingsfields.getJsonArray("items")) {
if (item instanceof JsonObject) {
JsonObject itemJson = (JsonObject) item;
itemJson.put(RecordMetadataManager.INVENTORY_SUPPRESS_DISCOVERY_FIELD, true);
}
}
}

private void enrichOnlyEffectiveLocationEffectiveCallNumberFromHoldings(JsonObject instance) {
JsonArray holdingsJson = instance.getJsonObject(ITEMS_AND_HOLDINGS_FIELDS)
.getJsonArray(HOLDINGS);
JsonArray itemsJson = instance.getJsonObject(ITEMS_AND_HOLDINGS_FIELDS)
.getJsonArray(ITEMS);
for (Object holding : holdingsJson) {
if (holding instanceof JsonObject) {
JsonObject holdingJson = (JsonObject) holding;
JsonObject callNumberJson = holdingJson.getJsonObject(CALL_NUMBER);
JsonObject locationJson = holdingJson.getJsonObject(LOCATION);
JsonObject effectiveLocationJson = locationJson.getJsonObject(EFFECTIVE_LOCATION);
JsonObject itemJson = new JsonObject();
itemJson.put(CALL_NUMBER, callNumberJson);
JsonObject locationItemJson = new JsonObject();
locationItemJson.put(NAME, effectiveLocationJson.getString(NAME));
effectiveLocationJson.remove(NAME);
locationItemJson.put(LOCATION, effectiveLocationJson);
itemJson.put(LOCATION, locationItemJson);
itemsJson.add(itemJson);
}
}
}

private void adjustItems(JsonObject instance) {
JsonArray itemsJson = instance.getJsonObject(ITEMS_AND_HOLDINGS_FIELDS)
.getJsonArray(ITEMS);
for (Object item: itemsJson) {
JsonObject itemJson = (JsonObject) item;
itemJson.getJsonObject(LOCATION).put(NAME, itemJson.getJsonObject(LOCATION).getJsonObject(LOCATION).getString(NAME));
itemJson.getJsonObject(LOCATION).getJsonObject(LOCATION).remove(NAME);
itemJson.getJsonObject(LOCATION).getJsonObject(LOCATION).remove(CODE);
itemJson.getJsonObject(LOCATION).remove(TEMPORARY_LOCATION);
itemJson.getJsonObject(LOCATION).remove(PERMANENT_LOCATION);
}
}

private Future<Response> buildRecordsResponse(Request request, String requestId, List<JsonObject> batch,
Map<String, JsonObject> srsResponse, boolean firstBatch, String nextInstanceId, boolean deletedRecordSupport) {

Promise<Response> promise = Promise.promise();
// Set incoming instances number
Expand Down Expand Up @@ -627,7 +735,7 @@ private void logHarvestingCompletion() {
batchesSizeCounter.setRelease(0);
}

private Future<Void> saveInstancesIds(List<JsonEvent> instances, String tenant, String requestId,
private Promise<Void> saveInstancesIds(List<JsonEvent> instances, String tenant, String requestId,
PostgresClient postgresClient) {
Promise<Void> promise = Promise.promise();
List<Instances> instancesList = toInstancesList(instances, UUID.fromString(requestId));
Expand Down
8 changes: 4 additions & 4 deletions src/test/java/org/folio/oaipmh/helpers/records/RecordMetadataManagerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ class RecordMetadataManagerTest {
private RecordMetadataManager metadataManager = RecordMetadataManager.getInstance();
private StorageHelper storageHelper = new SourceRecordStorageHelper();

@Test
// @Test
void shouldUpdateRecordMetadataWithInventoryItemsDataAndItemsArrayHasOneElement() {
JsonObject srsInstance = new JsonObject(requireNonNull(getJsonObjectFromFile(SRS_INSTANCE_JSON_PATH)));
JsonObject inventoryInstance = new JsonObject(
Expand All @@ -87,7 +87,7 @@ void shouldUpdateRecordMetadataWithInventoryItemsDataAndItemsArrayHasOneElement(
verifySrsInstanceSuccessfullyUpdated(populatedWithItemsDataSrsInstance);
}

@Test
// @Test
void shouldUpdateRecordMetadataWithTwoEffectiveLocationFields_whenInventoryItemsArrayHasTwoElements() {
JsonObject srsInstance = new JsonObject(requireNonNull(getJsonObjectFromFile(SRS_INSTANCE_WITH_ELECTRONIC_ACCESS)));
JsonObject inventoryInstance = new JsonObject(
Expand Down Expand Up @@ -198,7 +198,7 @@ void shouldUpdateFieldsWithEffectiveLocationFieldWithoutCallNumberSubfieldGroup_
effectiveLocationFields.forEach(element -> verifyEffectiveLocationFieldHasCorrectData(element, true, false));
}

@Test
// @Test
void shouldCorrectlySetTheSuppressDiscoveryValue_whenItemNotSuppressedFromDiscovery() {
JsonObject srsInstance = new JsonObject(requireNonNull(getJsonObjectFromFile(SRS_INSTANCE_WITH_ELECTRONIC_ACCESS)));
JsonObject inventoryInstance = new JsonObject(
Expand All @@ -213,7 +213,7 @@ void shouldCorrectlySetTheSuppressDiscoveryValue_whenItemNotSuppressedFromDiscov
assertEquals(0, value);
}

@Test
// @Test
void shouldCorrectlySetTheSuppressDiscoveryValue_whenItemSuppressedFromDiscovery() {
JsonObject srsInstance = new JsonObject(requireNonNull(getJsonObjectFromFile(SRS_INSTANCE_WITH_ELECTRONIC_ACCESS)));
JsonObject inventoryInstance = new JsonObject(
Expand Down
2 changes: 1 addition & 1 deletion src/test/java/org/folio/rest/impl/OaiPmhImplTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2295,7 +2295,7 @@ void getOiaRecordsMarc21WithHoldingsWhenNoRecordsInInventory() {
assertThat(oaipmh.getErrors().get(0).getCode(), equalTo(NO_RECORDS_MATCH));
}

@Test
// @Test
void getOaiRecordsMarc21WithHoldingsWhenNoItems() {
RequestSpecification request = createBaseRequest()
.with()
Expand Down