Merge remote-tracking branch 'upstream/develop' into issues/133-merge…

…-upstream

docs/Migration-Instructions.md

@@ -2,5 +2,6 @@
 
 This page contains PowerAuth Enrollment Server migration instructions.
 
+- [PowerAuth Enrollment Server 1.6.0](./PowerAuth-Enrollment-Server-1.6.0.md)
 - [PowerAuth Enrollment Server 1.5.0](./PowerAuth-Enrollment-Server-1.5.0.md)
 - [PowerAuth Enrollment Server 1.4.0](./PowerAuth-Enrollment-Server-1.4.0.md)

docs/PowerAuth-Enrollment-Server-1.6.0.md

@@ -0,0 +1,5 @@
+# Migration from 1.5.x to 1.6.x
+
+This guide contains instructions for migration from PowerAuth Enrollment Server version `1.5.x` to version `1.6.0`.
+
+No migration steps nor database changes are required.

docs/onboarding/Configuration-Properties.md

@@ -144,7 +144,6 @@ The Onboarding Server uses the following public configuration properties:
 | `enrollment-server-onboarding.provider.innovatrics.serviceUserAgent`                                    | `Wultra/OnboardingServer`                                                                                 | User agent to use when making HTTP calls to Innovatrics REST service.                                                                                                                                                                                              |
 | `enrollment-server-onboarding.provider.innovatrics.presenceCheckConfiguration.score`                    | 0.875                                                                                                     | Presence check minimal score threshold.                                                                                                                                                                                                                            |
 | `enrollment-server-onboarding.provider.innovatrics.documentVerificationConfiguration.documentCountries` | `CZE`                                                                                                     | List of expected countries of issue of identification documents as three-letter country codes, i.e. ISO 3166-1 alpha-3. If empty, all countries of issue known to Innovatrics are considered during classification, which may have negative impact on performance. |
-| `enrollment-server-onboarding.provider.innovatrics.document-verification-configuration.crucialFields`   | `documentNumber`, `dateOfIssue`, `dateOfExpiry`, `surname`, `dateOfBirth`, `personalNumber`, `givenNames` | Set of fields in camelCase that are cross-validated between the machine-readable zone and visual zone. Only those specified fields that are actually extracted from a document are considered. If empty, no inconsistencies will be checked.                       |
 | `enrollment-server-onboarding.provider.innovatrics.restClientConfig.acceptInvalidSslCertificate`        | `false`                                                                                                   | Whether invalid SSL certificate is accepted when calling Zen ID REST service.                                                                                                                                                                                      |
 | `enrollment-server-onboarding.provider.innovatrics.restClientConfig.maxInMemorySize`                    | `10485760`                                                                                                | Maximum in memory size of HTTP requests when calling Innovatrics REST service.                                                                                                                                                                                     |
 | `enrollment-server-onboarding.provider.innovatrics.restClientConfig.proxyEnabled`                       | `false`                                                                                                   | Whether proxy server is enabled when calling Innovatrics REST service.                                                                                                                                                                                             |

docs/onboarding/Migration-Instructions.md

@@ -2,4 +2,5 @@
 
 This page contains PowerAuth Enrollment Onboarding Server migration instructions.
 
+- [PowerAuth Enrollment Onboarding Server 1.6.0](./PowerAuth-Enrollment-Onboarding-Server-1.6.0.md)
 - [PowerAuth Enrollment Onboarding Server 1.5.0](./PowerAuth-Enrollment-Onboarding-Server-1.5.0.md)

docs/onboarding/PowerAuth-Enrollment-Onboarding-Server-1.6.0.md

@@ -0,0 +1,5 @@
+# Migration from 1.5.x to 1.6.x
+
+This guide contains instructions for migration from PowerAuth Enrollment Onboarding Server version `1.5.x` to version `1.6.0`.
+
+No migration steps nor database changes are required.

enrollment-server-onboarding-common/pom.xml

@@ -61,6 +61,12 @@
             <artifactId>spring-boot-starter-test</artifactId>
             <scope>test</scope>
         </dependency>
+
+        <dependency>
+            <groupId>com.h2database</groupId>
+            <artifactId>h2</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
 </project>

enrollment-server-onboarding-common/src/main/java/com/wultra/app/onboardingserver/common/database/DocumentResultRepository.java

@@ -40,18 +40,20 @@ public interface DocumentResultRepository extends CrudRepository<DocumentResultE
      */
     @Query("SELECT doc FROM DocumentResultEntity doc WHERE" +
             " doc.documentVerification.status = com.wultra.app.enrollmentserver.model.enumeration.DocumentStatus.UPLOAD_IN_PROGRESS" +
+            " AND doc.documentVerification.providerName = :providerName " +
             " AND doc.extractedData IS NULL " +
             " ORDER BY doc.timestampCreated ASC")
-    Stream<DocumentResultEntity> streamAllInProgressDocumentSubmits();
+    Stream<DocumentResultEntity> streamAllInProgressDocumentSubmits(String providerName);
 
     /**
      * @return All not finished document submit verifications (upload is in progress and verification id exists)
      */
     @Query("SELECT doc FROM DocumentResultEntity doc WHERE" +
             " doc.documentVerification.status = com.wultra.app.enrollmentserver.model.enumeration.DocumentStatus.UPLOAD_IN_PROGRESS" +
+            " AND doc.documentVerification.providerName = :providerName " +
             " AND doc.documentVerification.verificationId IS NOT NULL" +
             " ORDER BY doc.timestampCreated ASC")
-    Stream<DocumentResultEntity> streamAllInProgressDocumentSubmitVerifications();
+    Stream<DocumentResultEntity> streamAllInProgressDocumentSubmitVerifications(String providerName);
 
     /**
      * @return All document results for the specified document verification and processing phase

enrollment-server-onboarding-common/src/test/java/com/wultra/app/onboardingserver/common/EnrollmentServerOnboardingCommonTestApplication.java

@@ -0,0 +1,24 @@
+/*
+ * PowerAuth Enrollment Server
+ * Copyright (C) 2023 Wultra s.r.o.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package com.wultra.app.onboardingserver.common;
+
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+public class EnrollmentServerOnboardingCommonTestApplication {
+}

enrollment-server-onboarding-common/src/test/java/com/wultra/app/onboardingserver/common/database/DocumentResultRepositoryTest.java

@@ -0,0 +1,65 @@
+/*
+ * PowerAuth Enrollment Server
+ * Copyright (C) 2023 Wultra s.r.o.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package com.wultra.app.onboardingserver.common.database;
+
+
+import com.wultra.app.onboardingserver.common.database.entity.DocumentResultEntity;
+import com.wultra.app.onboardingserver.common.database.entity.DocumentVerificationEntity;
+import jakarta.transaction.Transactional;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.jdbc.Sql;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Test for {@link DocumentResultRepository}.
+ *
+ * @author Jan Pesek, [email protected]
+ */
+@DataJpaTest
+@ActiveProfiles("test")
+@Transactional
+class DocumentResultRepositoryTest {
+
+    @Autowired
+    private DocumentResultRepository tested;
+
+    @Test
+    @Sql
+    void testStreamAllInProgressDocumentSubmits() {
+        assertThat(tested.streamAllInProgressDocumentSubmits("mock"))
+                .extracting(DocumentResultEntity::getDocumentVerification)
+                .extracting(DocumentVerificationEntity::getProviderName)
+                .containsOnly("mock")
+                .hasSize(1);
+    }
+
+    @Test
+    @Sql
+    void testStreamAllInProgressDocumentSubmitVerifications() {
+        assertThat(tested.streamAllInProgressDocumentSubmitVerifications("mock"))
+                .extracting(DocumentResultEntity::getDocumentVerification)
+                .extracting(DocumentVerificationEntity::getProviderName)
+                .containsOnly("mock")
+                .hasSize(1);
+    }
+
+}

enrollment-server-onboarding-common/src/test/resources/application-test.properties

@@ -0,0 +1,5 @@
+spring.datasource.url=jdbc:h2:mem:testdb;DB_CLOSE_ON_EXIT=FALSE
+spring.datasource.username=sa
+spring.datasource.password=password
+spring.datasource.driver-class-name=org.h2.Driver
+spring.jpa.hibernate.ddl-auto=create

enrollment-server-onboarding-common/src/test/resources/com/wultra/app/onboardingserver/common/database/DocumentResultRepositoryTest.testStreamAllInProgressDocumentSubmitVerifications.sql

@@ -0,0 +1,12 @@
+-- Documents that have been already submitted and data were extracted (multiple providers).
+INSERT INTO es_identity_verification(id, activation_id, user_id, process_id, status, phase, timestamp_created, timestamp_last_updated) VALUES
+('v3', 'a3', 'u3', 'p3', 'IN_PROGRESS', 'DOCUMENT_UPLOAD', now(), now()),
+('v4', 'a4', 'u4', 'p4', 'IN_PROGRESS', 'DOCUMENT_UPLOAD', now(), now());
+
+INSERT INTO es_document_verification(id, provider_name, activation_id, identity_verification_id, verification_id, type, status, filename, used_for_verification, timestamp_created, timestamp_last_updated) VALUES
+    ('d3', 'foreign', 'a3', 'v3', 'verification1', 'ID_CARD', 'UPLOAD_IN_PROGRESS', 'f3', true, now(), now()),
+    ('d4', 'mock',    'a4', 'v4', 'verification2', 'ID_CARD', 'UPLOAD_IN_PROGRESS', 'f4', true, now(), now());
+
+INSERT INTO es_document_result(id, document_verification_id, phase, extracted_data, timestamp_created) VALUES
+    (3, 'd3', 'UPLOAD', '{extracted_data}', now()),
+    (4, 'd4', 'UPLOAD', '{extracted_data}', now());

enrollment-server-onboarding-common/src/test/resources/com/wultra/app/onboardingserver/common/database/DocumentResultRepositoryTest.testStreamAllInProgressDocumentSubmits.sql

@@ -0,0 +1,12 @@
+-- Documents that have not been submitted yet (multiple providers).
+INSERT INTO es_identity_verification(id, activation_id, user_id, process_id, status, phase, timestamp_created, timestamp_last_updated) VALUES
+    ('v1', 'a1', 'u1', 'p1', 'IN_PROGRESS', 'DOCUMENT_UPLOAD', now(), now()),
+    ('v2', 'a2', 'u2', 'p2', 'IN_PROGRESS', 'DOCUMENT_UPLOAD', now(), now());
+
+INSERT INTO es_document_verification(id, provider_name, activation_id, identity_verification_id, type, status, filename, used_for_verification, timestamp_created, timestamp_last_updated) VALUES
+    ('d1', 'foreign', 'a1', 'v1', 'ID_CARD', 'UPLOAD_IN_PROGRESS', 'f1', true, now(), now()),
+    ('d2', 'mock',    'a2', 'v2', 'ID_CARD', 'UPLOAD_IN_PROGRESS', 'f2', true, now(), now());
+
+INSERT INTO es_document_result(id, document_verification_id, phase, timestamp_created) VALUES
+    (1, 'd1', 'UPLOAD', now()),
+    (2, 'd2', 'UPLOAD', now());

enrollment-server-onboarding-provider-innovatrics/src/main/java/com/wultra/app/onboardingserver/provider/innovatrics/InnovatricsConfigProps.java

@@ -81,11 +81,6 @@ public static class DocumentVerificationConfiguration {
          * Identifies expected document countries of issue in ISO 3166-1 alpha-3 format.
          */
         private List<String> documentCountries;
-
-        /**
-         * Set of fields in camelCase that are cross-validated between the machine-readable zone and visual zone, if extracted.
-         */
-        private Set<String> crucialFields;
     }
 
 }

enrollment-server-onboarding-provider-innovatrics/src/main/java/com/wultra/app/onboardingserver/provider/innovatrics/InnovatricsDocumentVerificationProvider.java

@@ -372,37 +372,23 @@ private List<String> parseVisualZoneInspection(final VisualZoneInspection visual
 
         final MrzConsistency mrzConsistency = textConsistentWith.getMrz();
         if (mrzConsistency != null) {
-            final List<String> inconsistentAttributes = getCrucial(mrzConsistency.getInconsistentTexts());
+            final List<String> inconsistentAttributes = mrzConsistency.getInconsistentTexts();
             if (!inconsistentAttributes.isEmpty()) {
-                rejectionReasons.add("Inconsistent crucial attributes with MRZ: %s".formatted(inconsistentAttributes));
+                rejectionReasons.add("Inconsistent attributes with MRZ: %s".formatted(inconsistentAttributes));
             }
         }
 
         final BarcodesConsistency barcodesConsistency = textConsistentWith.getBarcodes();
         if (barcodesConsistency != null) {
-            final List<String> inconsistentAttributes = getCrucial(barcodesConsistency.getInconsistentTexts());
+            final List<String> inconsistentAttributes = barcodesConsistency.getInconsistentTexts();
             if (!inconsistentAttributes.isEmpty()) {
-                rejectionReasons.add("Inconsistent crucial attributes with barcode: %s".formatted(inconsistentAttributes));
+                rejectionReasons.add("Inconsistent attributes with barcode: %s".formatted(inconsistentAttributes));
             }
         }
 
         return rejectionReasons;
     }
 
-    /**
-     * Intersects list of attributes with CRUCIAL_ATTRIBUTES.
-     * @param attributes Attributes to do the intersection on.
-     * @return Attributes intersection.
-     */
-    private List<String> getCrucial(final List<String> attributes) {
-        if (attributes == null) {
-            return Collections.emptyList();
-        }
-
-        final Set<String> crucialFields = configuration.getDocumentVerificationConfiguration().getCrucialFields();
-        return attributes.stream().filter(crucialFields::contains).toList();
-    }
-
     private <T> String serializeToString(T src) throws DocumentVerificationException {
         try {
             return objectMapper.writeValueAsString(src);

enrollment-server-onboarding-provider-innovatrics/src/test/resources/application-test.properties

@@ -3,6 +3,5 @@ enrollment-server-onboarding.presence-check.provider=innovatrics
 enrollment-server-onboarding.onboarding-process.enabled=false
 
 enrollment-server-onboarding.provider.innovatrics.documentVerificationConfiguration.documentCountries=CZE,SVK
-enrollment-server-onboarding.provider.innovatrics.document-verification-configuration.crucialFields=documentNumber, dateOfIssue, dateOfExpiry, surname, dateOfBirth, personalNumber, givenNames
 
 enrollment-server-onboarding.provider.innovatrics.restClientConfig.maxInMemorySize=1048576

enrollment-server-onboarding/src/main/java/com/wultra/app/onboardingserver/impl/service/document/DocumentProcessingBatchService.java

@@ -22,6 +22,8 @@
 import com.wultra.app.onboardingserver.common.database.entity.DocumentVerificationEntity;
 import com.wultra.app.enrollmentserver.model.enumeration.DocumentStatus;
 import com.wultra.app.enrollmentserver.model.integration.OwnerId;
+import com.wultra.app.onboardingserver.configuration.IdentityVerificationConfig;
+import lombok.AllArgsConstructor;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -37,6 +39,7 @@
  * @author Lukas Lukovsky, [email protected]
  */
 @Service
+@AllArgsConstructor
 public class DocumentProcessingBatchService {
 
     private static final Logger logger = LoggerFactory.getLogger(DocumentProcessingBatchService.class);
@@ -45,26 +48,15 @@ public class DocumentProcessingBatchService {
 
     private final DocumentProcessingService documentProcessingService;
 
-    /**
-     * Service constructor.
-     * @param documentResultRepository Document verification result repository.
-     * @param documentProcessingService Document processing service.
-     */
-    @Autowired
-    public DocumentProcessingBatchService(
-            DocumentResultRepository documentResultRepository,
-            DocumentProcessingService documentProcessingService) {
-        this.documentResultRepository = documentResultRepository;
-        this.documentProcessingService = documentProcessingService;
-    }
+    private final IdentityVerificationConfig identityVerificationConfig;
 
     /**
      * Checks in progress document submits on current provider status and data result
      */
     @Transactional
     public void checkInProgressDocumentSubmits() {
         AtomicInteger countFinished = new AtomicInteger(0);
-        try (Stream<DocumentResultEntity> stream = documentResultRepository.streamAllInProgressDocumentSubmits()) {
+        try (Stream<DocumentResultEntity> stream = documentResultRepository.streamAllInProgressDocumentSubmits(identityVerificationConfig.getDocumentVerificationProvider())) {
             stream.forEach(docResult -> {
                 DocumentVerificationEntity docVerification = docResult.getDocumentVerification();
                 final OwnerId ownerId = new OwnerId();