parametrize batch sizes

admin-ch · Jul 22, 2021 · 738bca3 · 738bca3
1 parent 0be200c
commit 738bca3
Show file tree

Hide file tree

Showing 13 changed files with 50 additions and 26 deletions.
diff --git a/...rc/main/java/ch/admin/bag/covidcertificate/backend/verifier/data/VerifierDataService.java b/...rc/main/java/ch/admin/bag/covidcertificate/backend/verifier/data/VerifierDataService.java
@@ -76,5 +76,5 @@ public interface VerifierDataService {
     /** returns the highest DSC pk id */
     public long findMaxDscPkId();
 
-    public int getMaxDscBatchCount();
+    public int getDscBatchSize();
 }
diff --git a/...admin/bag/covidcertificate/backend/verifier/data/impl/JdbcRevokedCertDataServiceImpl.java b/...admin/bag/covidcertificate/backend/verifier/data/impl/JdbcRevokedCertDataServiceImpl.java
@@ -29,11 +29,12 @@ public class JdbcRevokedCertDataServiceImpl implements RevokedCertDataService {
     private static final Logger logger =
             LoggerFactory.getLogger(JdbcRevokedCertDataServiceImpl.class);
 
-    private static final int MAX_REVOKED_CERT_BATCH_COUNT = 1000;
+    private final int revokedCertBatchSize;
     private final NamedParameterJdbcTemplate jt;
 
-    public JdbcRevokedCertDataServiceImpl(DataSource dataSource) {
+    public JdbcRevokedCertDataServiceImpl(DataSource dataSource, int revokedCertBatchSize) {
         this.jt = new NamedParameterJdbcTemplate(dataSource);
+        this.revokedCertBatchSize = revokedCertBatchSize;
     }
 
     @Transactional(readOnly = false)
@@ -90,10 +91,10 @@ public List<DbRevokedCert> findRevokedCerts(Long since) {
                 "select pk_revoked_cert_id, uvci from t_revoked_cert"
                         + " where pk_revoked_cert_id > :since"
                         + " order by pk_revoked_cert_id asc"
-                        + " limit :max_batch_count";
+                        + " limit :batch_size";
         MapSqlParameterSource params = new MapSqlParameterSource();
         params.addValue("since", since);
-        params.addValue("max_batch_count", MAX_REVOKED_CERT_BATCH_COUNT);
+        params.addValue("batch_size", revokedCertBatchSize);
         return jt.query(sql, params, new RevokedCertRowMapper());
     }
 

diff --git a/...ch/admin/bag/covidcertificate/backend/verifier/data/impl/JdbcVerifierDataServiceImpl.java b/...ch/admin/bag/covidcertificate/backend/verifier/data/impl/JdbcVerifierDataServiceImpl.java
@@ -36,12 +36,13 @@ public class JdbcVerifierDataServiceImpl implements VerifierDataService {
 
     private static final Logger logger = LoggerFactory.getLogger(JdbcVerifierDataServiceImpl.class);
 
-    private static final int MAX_DSC_BATCH_COUNT = 1000;
+    private final int dscBatchSize;
     private final NamedParameterJdbcTemplate jt;
     private final SimpleJdbcInsert cscaInsert;
     private final SimpleJdbcInsert dscInsert;
 
-    public JdbcVerifierDataServiceImpl(DataSource dataSource) {
+    public JdbcVerifierDataServiceImpl(DataSource dataSource, int dscBatchSize) {
+        this.dscBatchSize = dscBatchSize;
         this.jt = new NamedParameterJdbcTemplate(dataSource);
         this.cscaInsert =
                 new SimpleJdbcInsert(dataSource)
@@ -185,12 +186,12 @@ public List<ClientCert> findDscs(Long since, CertFormat certFormat, Long upTo) {
                         + " where pk_dsc_id > :since"
                         + (upTo != null ? " and pk_dsc_id <= :up_to" : "")
                         + " order by pk_dsc_id asc"
-                        + " limit :max_dsc_batch_count";
+                        + " limit :batch_size";
 
         var params = new MapSqlParameterSource();
         params.addValue("since", since);
         params.addValue("up_to", upTo);
-        params.addValue("max_dsc_batch_count", MAX_DSC_BATCH_COUNT);
+        params.addValue("batch_size", dscBatchSize);
 
         return jt.query(sql, params, new ClientCertRowMapper(certFormat));
     }
@@ -215,11 +216,11 @@ public List<ClientCert> findDscsBefore(Long since, CertFormat certFormat, Date i
                         + " where pk_dsc_id > :pk_dsc_id"
                         + " and imported_at < :before"
                         + " order by pk_dsc_id asc"
-                        + " limit :max_dsc_batch_count";
+                        + " limit :batch_size";
 
         var params = new MapSqlParameterSource();
         params.addValue("pk_dsc_id", since);
-        params.addValue("max_dsc_batch_count", MAX_DSC_BATCH_COUNT);
+        params.addValue("batch_size", dscBatchSize);
         params.addValue("before", importedBefore);
 
         return jt.query(sql, params, new ClientCertRowMapper(certFormat));
@@ -260,8 +261,8 @@ public long findMaxDscPkId() {
     }
 
     @Override
-    public int getMaxDscBatchCount() {
-        return MAX_DSC_BATCH_COUNT;
+    public int getDscBatchSize() {
+        return dscBatchSize;
     }
 
     private MapSqlParameterSource getCscaParams(DbCsca dbCsca) {

diff --git a/.../src/test/java/ch/admin/bag/covidcertificate/backend/verifier/data/config/TestConfig.java b/.../src/test/java/ch/admin/bag/covidcertificate/backend/verifier/data/config/TestConfig.java
@@ -15,6 +15,7 @@
 import ch.admin.bag.covidcertificate.backend.verifier.data.impl.JdbcAppTokenDataServiceImpl;
 import ch.admin.bag.covidcertificate.backend.verifier.data.impl.JdbcVerifierDataServiceImpl;
 import javax.sql.DataSource;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.test.context.TestConfiguration;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Profile;
@@ -23,9 +24,12 @@
 @TestConfiguration
 public class TestConfig {
 
+    @Value("${ws.keys.batch-size:1000}")
+    protected Integer dscBatchSize;
+
     @Bean
     public VerifierDataService verifierDataService(DataSource dataSource) {
-        return new JdbcVerifierDataServiceImpl(dataSource);
+        return new JdbcVerifierDataServiceImpl(dataSource, dscBatchSize);
     }
 
     @Bean

diff --git a/.../ch-covidcertificate-backend-verifier-data/src/test/resources/application-test.properties b/.../ch-covidcertificate-backend-verifier-data/src/test/resources/application-test.properties
@@ -0,0 +1,2 @@
+revocationList.batch-size=20000
+ws.keys.batch-size=1000
diff --git a/.../main/java/ch/admin/bag/covidcertificate/backend/verifier/sync/config/SyncBaseConfig.java b/.../main/java/ch/admin/bag/covidcertificate/backend/verifier/sync/config/SyncBaseConfig.java
@@ -37,6 +37,9 @@ public abstract class SyncBaseConfig {
     @Value("${dgc.clientcert.password}")
     String authClientCertPassword;
 
+    @Value("${ws.keys.batch-size:1000}")
+    protected Integer dscBatchSize;
+
     public abstract DataSource dataSource();
 
     public abstract Flyway flyway();
@@ -62,7 +65,7 @@ public RestTemplate restTemplate() {
 
     @Bean
     public VerifierDataService verifierDataService(DataSource dataSource) {
-        return new JdbcVerifierDataServiceImpl(dataSource);
+        return new JdbcVerifierDataServiceImpl(dataSource, dscBatchSize);
     }
 
     @Bean

diff --git a/.../src/test/java/ch/admin/bag/covidcertificate/backend/verifier/sync/config/TestConfig.java b/.../src/test/java/ch/admin/bag/covidcertificate/backend/verifier/sync/config/TestConfig.java
@@ -32,6 +32,9 @@ public class TestConfig {
     @Value("${dgc.baseurl}")
     String baseurl = "https://testurl.europa.eu";
 
+    @Value("${ws.keys.batch-size:1000}")
+    protected Integer dscBatchSize;
+
     @Bean
     public DgcSyncer dgcSyncer(DgcClient dgcClient, VerifierDataService verifierDataService) {
         logger.info("Instantiated DGC Syncer with baseurl: {}", baseurl);
@@ -46,7 +49,7 @@ public DgcClient dgcClient(RestTemplate restTemplate) {
 
     @Bean
     public VerifierDataService verifierDataService(DataSource dataSource) {
-        return new JdbcVerifierDataServiceImpl(dataSource);
+        return new JdbcVerifierDataServiceImpl(dataSource, dscBatchSize);
     }
 
     @Bean

diff --git a/.../src/main/java/ch/admin/bag/covidcertificate/backend/verifier/ws/config/WsBaseConfig.java b/.../src/main/java/ch/admin/bag/covidcertificate/backend/verifier/ws/config/WsBaseConfig.java
@@ -71,6 +71,12 @@ public abstract class WsBaseConfig implements WebMvcConfigurer {
     @Value("${revocationList.baseurl}")
     String revokedCertsBaseUrl;
 
+    @Value("${revocationList.batch-size:20000}")
+    protected Integer revokedCertBatchSize;
+
+    @Value("${ws.keys.batch-size:1000}")
+    protected Integer dscBatchSize;
+
     public abstract DataSource dataSource();
 
     public abstract Flyway flyway();
@@ -140,7 +146,7 @@ public void addInterceptors(InterceptorRegistry registry) {
 
     @Bean
     public VerifierDataService verifierDataService(DataSource dataSource) {
-        return new JdbcVerifierDataServiceImpl(dataSource);
+        return new JdbcVerifierDataServiceImpl(dataSource, dscBatchSize);
     }
 
     @Bean
@@ -177,7 +183,7 @@ public RevocationListSyncer revocationListSyncer(
 
     @Bean
     public RevokedCertDataService revokedCertDataService(DataSource dataSource) {
-        return new JdbcRevokedCertDataServiceImpl(dataSource);
+        return new JdbcRevokedCertDataServiceImpl(dataSource, revokedCertBatchSize);
     }
 
     @Bean

diff --git a/...main/java/ch/admin/bag/covidcertificate/backend/verifier/ws/controller/KeyController.java b/...main/java/ch/admin/bag/covidcertificate/backend/verifier/ws/controller/KeyController.java
@@ -97,7 +97,7 @@ private HttpHeaders getKeysUpdatesHeaders(List<ClientCert> dscs) {
                         .max()
                         .orElse(verifierDataService.findMaxDscPkId());
         headers.add(NEXT_SINCE_HEADER, nextSince.toString());
-        boolean upToDate = dscs.size() < verifierDataService.getMaxDscBatchCount();
+        boolean upToDate = dscs.size() < verifierDataService.getDscBatchSize();
         headers.add(UP_TO_DATE_HEADER, String.valueOf(upToDate));
         return headers;
     }

diff --git a/...r/ch-covidcertificate-backend-verifier-ws/src/main/resources/application-local.properties b/...r/ch-covidcertificate-backend-verifier-ws/src/main/resources/application-local.properties
@@ -10,4 +10,7 @@ ws.monitor.prometheus.password=prometheus
 server.port=8081
 
 revocationList.baseurl=http://localhost:8081
-revocationList.sync.cron=0 * * ? * *
+revocationList.sync.cron=0 * * ? * *
+
+revocationList.batch-size=4
+ws.keys.batch-size=3
diff --git a/...ava/ch/admin/bag/covidcertificate/backend/verifier/ws/controller/KeyControllerV2Test.java b/...ava/ch/admin/bag/covidcertificate/backend/verifier/ws/controller/KeyControllerV2Test.java
@@ -68,7 +68,7 @@ public abstract class KeyControllerV2Test extends BaseControllerTest {
 
     @BeforeAll
     public void setup() {
-        for (int i = 0; i < verifierDataService.getMaxDscBatchCount() * 10; i++) {
+        for (int i = 0; i < verifierDataService.getDscBatchSize() * 10; i++) {
             suffixes.add(i);
         }
     }
@@ -197,8 +197,7 @@ public void keysUpdatesTest() throws Exception {
 
         // test batching
         int batchCount = 4;
-        dscs.addAll(
-                insertNDscs(cscaId, verifierDataService.getMaxDscBatchCount() * (batchCount - 1)));
+        dscs.addAll(insertNDscs(cscaId, verifierDataService.getDscBatchSize() * (batchCount - 1)));
 
         // upTo set so no batching kicks in
         since = null;
@@ -255,7 +254,7 @@ private void assertUpdatesResponse(
         int upperCutCount = Math.max(0, (int) verifierDataService.findMaxDscPkId() - upTo);
         int expectedSize =
                 Math.min(
-                        verifierDataService.getMaxDscBatchCount(),
+                        verifierDataService.getDscBatchSize(),
                         Math.max(0, dscs.size() - lowerCutCount - upperCutCount));
         assertEquals(expectedSize, certs.size());
 

diff --git a/...er/ch-covidcertificate-backend-verifier-ws/src/test/resources/application-test.properties b/...er/ch-covidcertificate-backend-verifier-ws/src/test/resources/application-test.properties
@@ -1,4 +1,7 @@
 revocationList.baseurl=https://testurl.admin.ch/api/
 
 ws.keys.update.max-age=PT2M
-ws.keys.list.max-age=PT3M
+ws.keys.list.max-age=PT3M
+
+revocationList.batch-size=20000
+ws.keys.batch-size=1000
diff --git a/...verifier/ch-covidcertificate-backend-verifier-ws/src/test/resources/http/verifier-ws.http b/...verifier/ch-covidcertificate-backend-verifier-ws/src/test/resources/http/verifier-ws.http
@@ -10,7 +10,6 @@ Authorization: Bearer {{apiKey}}
 ### get active cert key ids
 GET {{baseUrl}}/v2/keys/list
 Accept: application/json
-If-None-Match: "-720957702"
 Authorization: Bearer {{apiKey}}
 
 ### get revocation list
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		revocationList.batch-size=20000
		ws.keys.batch-size=1000