decrease the page size for old KG

HumanBrainProject · Mar 7, 2022 · 85f4f5c · 85f4f5c
1 parent 04e5d2f
commit 85f4f5c
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 3 deletions.
diff --git a/service/src/main/java/eu/ebrains/kg/search/controller/indexing/IndexingController.java b/service/src/main/java/eu/ebrains/kg/search/controller/indexing/IndexingController.java
@@ -128,7 +128,7 @@ public <v1Input, v2Input, v3Input, Target extends TargetInstance> List<ErrorRepo
         }
         boolean indexDataFromOldKG = dataStage == DataStage.RELEASED || !translatorModel.isOnlyV3ForInProgress();
         if (indexDataFromOldKG && translatorModel.getV2translator() != null) {
-            final UpdateResult updateResultV2 = update(kgV2, translatorModel.getTargetClass(), translatorModel.getV2translator(), translatorModel.getBulkSize(), dataStage, handledIdentifiers, instance -> {
+            final UpdateResult updateResultV2 = update(kgV2, translatorModel.getTargetClass(), translatorModel.getV2translator(), translatorModel.getBulkSizeV2(), dataStage, handledIdentifiers, instance -> {
                 if(translatorModel.getMerger()!=null){
                     final Translator<v1Input, Target, ? extends ResultsOfKGv2<v1Input>> v1translator = translatorModel.getV1translator();
                     final List<Target> fromV1 = v1translator != null ? getRelatedInstance(kgV2, v1translator, instance, dataStage) : null;
@@ -149,7 +149,7 @@ public <v1Input, v2Input, v3Input, Target extends TargetInstance> List<ErrorRepo
             nonSearchableIds.addAll(updateResultV2.nonSearchableIds);
         }
         if (indexDataFromOldKG && translatorModel.getV1translator() != null) {
-            final UpdateResult updateResultV1 = update(kgV2, translatorModel.getTargetClass(), translatorModel.getV1translator(), translatorModel.getBulkSize(), dataStage, handledIdentifiers, null, translatorModel.isAutoRelease(), temporary);
+            final UpdateResult updateResultV1 = update(kgV2, translatorModel.getTargetClass(), translatorModel.getV1translator(), translatorModel.getBulkSizeV2(), dataStage, handledIdentifiers, null, translatorModel.isAutoRelease(), temporary);
             handledIdentifiers.addAll(updateResultV1.handledIdentifiers);
             searchableIds.addAll(updateResultV1.searchableIds);
             nonSearchableIds.addAll(updateResultV1.nonSearchableIds);
@@ -184,7 +184,7 @@ private <Target extends TargetInstance> void clearNonResolvableReferences(List<T
         final Set<String> existingRefs = elasticSearchController.existingDocuments(refs, dataStage);
         references.forEach(r -> {
             if(r.getReference()!=null && !existingRefs.contains(r.getReference())){
-                logger.warn(String.format("Was not able to find instance %s in database for stage %s - remove reference", r.getReference(), dataStage));
+//                logger.warn(String.format("Was not able to find instance %s in database for stage %s - remove reference", r.getReference(), dataStage));
                 r.setReference(null);
             }
         });

diff --git a/service/src/main/java/eu/ebrains/kg/search/model/TranslatorModel.java b/service/src/main/java/eu/ebrains/kg/search/model/TranslatorModel.java
@@ -93,4 +93,9 @@ public int getBulkSize() {
         //TODO handle the "big" source models by restraining the bulk size
         return 1000;
     }
+
+    public int getBulkSizeV2() {
+        //TODO handle the "big" source models by restraining the bulk size
+        return 200;
+    }
 }