Amélioration du temps d'indexation

.github/workflows/build-and-test.yml

@@ -35,5 +35,6 @@ jobs:
           curl -fsSL "$ELASTICSEARCH_URL/_cat/health?h=status"
       - name: Run tests
         env:
+          INDEX_CHUNK_SIZE: 100
           ELASTICSEARCH_URL: http://localhost:${{ job.services.elasticsearch.ports[9200] }}
         run: npm test -- --forceExit

package.json

@@ -4,11 +4,10 @@
   "description": "Indexation of french companies from INSEE's database",
   "main": "src/index.ts",
   "scripts": {
-    "scalingo-postbuild": "npm run build && npm prune --production",
     "build": "tsc",
     "watch": "tsc -w",
     "types": "tsc -noEmit",
-    "start": "npm run build",
+    "start": "npx --yes http-server",
     "index": "npm run index:sirene && npm run index:siret",
     "preindex:dev": "export NODE_ENV=dev && docker-compose up -d elasticsearch && npm run build && npm run index:sirene && npm run index:siret",
     "index:dev": "npm run index:sirene:dev && npm run index:siret:dev",

src/commands/__tests__/index.integration.ts

@@ -61,8 +61,6 @@ describe("Perform indexation", () => {
         }
       }
     };
-    // wait for the ES refresh cycle of 1sec
-    await new Promise(resolve => setTimeout(resolve, 1000));
 
     await elasticSearchClient.search(searchRequest).then(r => {
       if (r.body.timed_out) {
@@ -94,11 +92,10 @@ describe("Perform indexation", () => {
       body: {
         query: {
           match_all: {}
-        }
+        },
+        size: 300
       }
     };
-    // wait for the ES refresh cycle of 1sec
-    await new Promise(resolve => setTimeout(resolve, 1000));
 
     await elasticSearchClient.search(searchRequest).then(r => {
       if (r.body.timed_out) {

src/common/logger.ts

@@ -20,7 +20,7 @@ const LOG_TO_HTTP =
 const logFormat = format.combine(
   format.label({ label: "trackdechets-sirene-search" }),
   format.timestamp({
-    format: "HH-MM:ss YYYY-MM-DD"
+    format: "HH-mm:ss YYYY-MM-DD"
   }),
   format.prettyPrint(),
   format.colorize(),

src/indexation/__tests__/bulkIndex.test.ts

@@ -24,26 +24,16 @@ describe("bulkIndexByChunks", () => {
     dataFormatterFn: dataFormatterFnMock
   };
   const indexNameMock = "test_index";
-  const CHUNK_SIZE = 100;
+  const CHUNK_SIZE = parseInt(`${process.env.INDEX_CHUNK_SIZE}`, 10) || 100;
 
   beforeEach(() => {
     process.env.TD_SIRENE_INDEX_MAX_CONCURRENT_REQUESTS = "1";
-    process.env.INDEX_CHUNK_SIZE = `${CHUNK_SIZE}`;
     esClientMock.mockReset();
     dataFormatterFnMock.mockReset();
     esClientMock.mockImplementation(() => Promise.resolve());
     dataFormatterFnMock.mockImplementation((body, _) => Promise.resolve(body));
   });
 
-  test("Should immediately send a request if body size is less than CHUNK_SIZE", async () => {
-    // One document to index
-    const bodyMock: ElasticBulkNonFlatPayload = [
-      [{ index: { _id: "1", _index: indexNameMock } }, { field: "value1" }]
-    ];
-    await bulkIndexByChunks(bodyMock, indexConfigMock, indexNameMock);
-    expect(esClientMock).toHaveBeenCalledTimes(1);
-  });
-
   test("Should slice the body into chunks and send each as a separate request", async () => {
     // One Chunk + 1
     const bodyMock: ElasticBulkNonFlatPayload = Array(CHUNK_SIZE + 1)
@@ -72,13 +62,12 @@ describe("bulkIndexByChunks", () => {
   });
 
   test("Should call dataFormatterFn if it is a function", async () => {
-    // One document to index
-    const bodyMock: ElasticBulkNonFlatPayload = [
-      [
-        { index: { _id: "1", _index: indexNameMock } },
-        { my_document_field: "value1" }
-      ]
-    ];
+    const bodyMock: ElasticBulkNonFlatPayload = Array(CHUNK_SIZE + 1)
+      .fill(0)
+      .map((_, i) => [
+        { index: { _id: `${i}`, _index: indexNameMock } },
+        { my_document_field: `value${i}` }
+      ]);
     await bulkIndexByChunks(bodyMock, indexConfigMock, indexNameMock);
     expect(indexConfigMock.dataFormatterFn).toHaveBeenCalled();
   });