From ca49c34d0366c7d8fc3096be0eb9dd5e43c43773 Mon Sep 17 00:00:00 2001
From: ronitagarwala01 <34790361+ronitagarwala01@users.noreply.github.com>
Date: Wed, 22 Nov 2023 13:04:52 -0500
Subject: [PATCH] Apply sandbox init commands in small batches to avoid
 timeouts

Co-authored-by: Leo Singer <leo.singer@ligo.org>
---
 data.ts           |  9 +++++++--
 package-lock.json | 13 +++++++++++++
 package.json      |  2 ++
 3 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/data.ts b/data.ts
index a1d2f8d..950f13f 100644
--- a/data.ts
+++ b/data.ts
@@ -12,11 +12,12 @@ import { pathToFileURL } from 'url'
 import { Client } from '@opensearch-project/opensearch'
 import type { ClientOptions } from '@opensearch-project/opensearch'
 import { exists } from './paths'
+import chunk from 'lodash/chunk'
 
 const jsonFilename = 'sandbox-search.json'
 const jsFilename = 'sandbox-search.js'
 
-async function getData(path: string) {
+async function getData(path: string): Promise<object[]> {
   let result
   const jsonPath = join(path, jsonFilename)
   const jsPath = join(path, jsFilename)
@@ -44,6 +45,10 @@ export async function populate(path: string, opts: ClientOptions) {
   const data = await getData(path)
   if (data) {
     const client = new Client(opts)
-    await client.bulk({ body: data })
+    const batch_size = 10
+    const batches = chunk(data, batch_size)
+    for (const batch of batches) {
+      await client.bulk({ body: batch })
+    }
   }
 }
diff --git a/package-lock.json b/package-lock.json
index bc0ed54..91d575b 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -11,6 +11,7 @@
       "dependencies": {
         "@opensearch-project/opensearch": "^2.2.0",
         "env-paths": "^3.0.0",
+        "lodash": "^4.17.21",
         "make-fetch-happen": "^11.0.3",
         "rimraf": "^4.1.2",
         "tar": "^6.1.13",
@@ -20,6 +21,7 @@
       "devDependencies": {
         "@nasa-gcn/eslint-config-gitignore": "^0.0.1",
         "@tsconfig/node14": "^1.0.3",
+        "@types/lodash": "^4.14.202",
         "@types/make-fetch-happen": "^10.0.1",
         "@types/node": "^18.13.0",
         "@types/tar": "^6.1.4",
@@ -544,6 +546,12 @@
       "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==",
       "dev": true
     },
+    "node_modules/@types/lodash": {
+      "version": "4.14.202",
+      "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.202.tgz",
+      "integrity": "sha512-OvlIYQK9tNneDlS0VN54LLd5uiPCBOp7gS5Z0f1mjoJYBrtStzgmJBxONW3U6OZqdtNzZPmn9BS/7WI7BFFcFQ==",
+      "dev": true
+    },
     "node_modules/@types/make-fetch-happen": {
       "version": "10.0.1",
       "resolved": "https://registry.npmjs.org/@types/make-fetch-happen/-/make-fetch-happen-10.0.1.tgz",
@@ -2798,6 +2806,11 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/lodash": {
+      "version": "4.17.21",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
+      "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
+    },
     "node_modules/lodash.merge": {
       "version": "4.6.2",
       "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
diff --git a/package.json b/package.json
index bc7b329..f1ba9cf 100644
--- a/package.json
+++ b/package.json
@@ -27,6 +27,7 @@
   "dependencies": {
     "@opensearch-project/opensearch": "^2.2.0",
     "env-paths": "^3.0.0",
+    "lodash": "^4.17.21",
     "make-fetch-happen": "^11.0.3",
     "rimraf": "^4.1.2",
     "tar": "^6.1.13",
@@ -36,6 +37,7 @@
   "devDependencies": {
     "@nasa-gcn/eslint-config-gitignore": "^0.0.1",
     "@tsconfig/node14": "^1.0.3",
+    "@types/lodash": "^4.14.202",
     "@types/make-fetch-happen": "^10.0.1",
     "@types/node": "^18.13.0",
     "@types/tar": "^6.1.4",