log-viewer-webui: Add server implementation for submitting IR extract…

…ion jobs and serving IR files; Don't copy unnecessary files when building component. (#458)
y-scope · Jul 20, 2024 · 921a388 · 921a388
1 parent c62b758
commit 921a388
Show file tree

Hide file tree

Showing 18 changed files with 1,653 additions and 224 deletions.
diff --git a/Taskfile.yml b/Taskfile.yml
@@ -10,6 +10,7 @@ vars:
   G_BUILD_DIR: "{{.ROOT_DIR}}/build"
   G_CORE_COMPONENT_BUILD_DIR: "{{.G_BUILD_DIR}}/core"
   G_LOG_VIEWER_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/log-viewer-webui"
+  G_LOG_VIEWER_WEBUI_SRC_DIR: "{{.ROOT_DIR}}/components/log-viewer-webui"
   G_METEOR_BUILD_DIR: "{{.G_BUILD_DIR}}/meteor"
   G_NODEJS_14_BUILD_DIR: "{{.G_BUILD_DIR}}/nodejs-14"
   G_NODEJS_14_BIN_DIR: "{{.G_NODEJS_14_BUILD_DIR}}/bin"
@@ -73,6 +74,7 @@ tasks:
       CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5"
       OUTPUT_DIR: "{{.G_PACKAGE_BUILD_DIR}}"
     sources:
+      - "{{.G_BUILD_DIR}}/log-viewer-webui-client.md5"
       - "{{.G_BUILD_DIR}}/package-venv.md5"
       - "{{.G_BUILD_DIR}}/webui.md5"
       - "{{.G_BUILD_DIR}}/webui-nodejs.md5"
@@ -81,6 +83,10 @@ tasks:
       - "{{.G_CORE_COMPONENT_BUILD_DIR}}/clp"
       - "{{.G_CORE_COMPONENT_BUILD_DIR}}/clp-s"
       - "{{.G_CORE_COMPONENT_BUILD_DIR}}/reducer-server"
+      - "{{.G_LOG_VIEWER_WEBUI_SRC_DIR}}/server/package.json"
+      - "{{.G_LOG_VIEWER_WEBUI_SRC_DIR}}/server/package-lock.json"
+      - "{{.G_LOG_VIEWER_WEBUI_SRC_DIR}}/server/settings.json"
+      - "{{.G_LOG_VIEWER_WEBUI_SRC_DIR}}/server/src/**/*.js"
       - "{{.TASKFILE}}"
       - "/etc/os-release"
       - "components/clp-package-utils/dist/*.whl"
@@ -94,7 +100,7 @@ tasks:
       - "clp-py-utils"
       - "init"
       - "job-orchestration"
-      - "log-viewer-webui"
+      - "log-viewer-webui-client"
       - "nodejs-14"
       - "package-venv"
       - task: "utils:validate-checksum"
@@ -143,10 +149,15 @@ tasks:
         PATH="{{.G_NODEJS_14_BIN_DIR}}":$PATH npm install
       - >-
         rsync -a
-        "{{.G_LOG_VIEWER_WEBUI_BUILD_DIR}}/"
-        "{{.OUTPUT_DIR}}/var/www/log_viewer/"
+        "{{.G_LOG_VIEWER_WEBUI_BUILD_DIR}}/client"
+        "{{.OUTPUT_DIR}}/var/www/log_viewer_webui/"
       - |-
-        cd "{{.OUTPUT_DIR}}/var/www/log_viewer/server"
+        cd components/log-viewer-webui/server/
+        rsync -a \
+        package.json package-lock.json settings.json src \
+        "{{.OUTPUT_DIR}}/var/www/log_viewer_webui/server/"
+      - |-
+        cd "{{.OUTPUT_DIR}}/var/www/log_viewer_webui/server"
         PATH="{{.G_NODEJS_22_BIN_DIR}}":$PATH npm clean-install
       # This command must be last
       - task: "utils:compute-checksum"
@@ -195,19 +206,17 @@ tasks:
       vars:
         COMPONENT: "{{.TASK}}"
 
-  log-viewer-webui:
+  log-viewer-webui-client:
     vars:
       CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5"
       OUTPUT_DIR: "{{.G_LOG_VIEWER_WEBUI_BUILD_DIR}}"
     sources:
       - "{{.G_BUILD_DIR}}/log-viewer-modules.md5"
       - "{{.TASKFILE}}"
+      - "client/package.json"
       - "client/src/**/*.css"
       - "client/src/**/*.jsx"
-      - "client/src/package.json"
       - "client/src/webpack.config.js"
-      - "server/src/**/*.js"
-      - "server/src/**/package.json"
     dir: "components/log-viewer-webui"
     generates: ["{{.CHECKSUM_FILE}}"]
     deps:
@@ -219,14 +228,10 @@ tasks:
           DATA_DIR: "{{.OUTPUT_DIR}}"
     cmds:
       - "rm -rf '{{.OUTPUT_DIR}}'"
-      - "rsync -a client {{.OUTPUT_DIR}}/"
-      - |-
-        cd "{{.OUTPUT_DIR}}/client"
-        PATH="{{.G_NODEJS_22_BIN_DIR}}":$PATH npm run build
-      - "mkdir {{.OUTPUT_DIR}}/server"
       - |-
-        cd server
-        rsync -a src package-lock.json package.json {{.OUTPUT_DIR}}/server/
+        cd client
+        PATH="{{.G_NODEJS_22_BIN_DIR}}":$PATH npm run build -- \
+        --output-path "{{.OUTPUT_DIR}}/client"
       - task: "utils:compute-checksum"
         vars:
           DATA_DIR: "{{.OUTPUT_DIR}}"

diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py
@@ -501,11 +501,11 @@ def validate_webui_config(
     validate_port(f"{WEBUI_COMPONENT_NAME}.port", clp_config.webui.host, clp_config.webui.port)
 
 
-def validate_log_viewer_config(clp_config: CLPConfig, logs_dir: pathlib.Path):
-    try:
-        validate_path_could_be_dir(logs_dir)
-    except ValueError as ex:
-        raise ValueError(f"{LOG_VIEWER_WEBUI_COMPONENT_NAME} logs directory is invalid: {ex}")
+def validate_log_viewer_webui_config(clp_config: CLPConfig, settings_json_path: pathlib.Path):
+    if not settings_json_path.exists():
+        raise ValueError(
+            f"{WEBUI_COMPONENT_NAME} {settings_json_path} is not a valid path to settings.json"
+        )
 
     validate_port(
         f"{LOG_VIEWER_WEBUI_COMPONENT_NAME}.port",

diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py
@@ -50,7 +50,7 @@
     validate_and_load_queue_credentials_file,
     validate_and_load_redis_credentials_file,
     validate_db_config,
-    validate_log_viewer_config,
+    validate_log_viewer_webui_config,
     validate_queue_config,
     validate_redis_config,
     validate_reducer_config,
@@ -668,13 +668,13 @@ def generic_start_worker(
     logger.info(f"Started {component_name}.")
 
 
-def update_meteor_settings(
+def update_settings_object(
     parent_key_prefix: str,
     settings: Dict[str, Any],
     updates: Dict[str, Any],
 ):
     """
-    Recursively updates the given Meteor settings object with the values from `updates`.
+    Recursively updates the given settings object with the values from `updates`.
 
     :param parent_key_prefix: The prefix for keys at this level in the settings dictionary.
     :param settings: The settings to update.
@@ -686,11 +686,25 @@ def update_meteor_settings(
             error_msg = f"{parent_key_prefix}{key} is not a valid configuration key for the webui."
             raise ValueError(error_msg)
         if isinstance(value, dict):
-            update_meteor_settings(f"{parent_key_prefix}{key}.", settings[key], value)
+            update_settings_object(f"{parent_key_prefix}{key}.", settings[key], value)
         else:
             settings[key] = updates[key]
 
 
+def read_and_update_settings_json(settings_file_path: pathlib.Path, updates: Dict[str, Any]):
+    """
+    Reads and updates a settings JSON file.
+
+    :param settings_file_path:
+    :param updates:
+    """
+    with open(settings_file_path, "r") as settings_json_file:
+        settings_object = json.loads(settings_json_file.read())
+    update_settings_object("", settings_object, updates)
+
+    return settings_object
+
+
 def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts):
     component_name = WEBUI_COMPONENT_NAME
     logger.info(f"Starting {component_name}...")
@@ -710,8 +724,8 @@ def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts
     webui_logs_dir.mkdir(exist_ok=True, parents=True)
 
     container_webui_logs_dir = pathlib.Path("/") / "var" / "log" / component_name
-    with open(settings_json_path, "r") as settings_json_file:
-        meteor_settings = json.loads(settings_json_file.read())
+
+    # Read and update settings.json
     meteor_settings_updates = {
         "private": {
             "SqlDbHost": clp_config.database.host,
@@ -726,7 +740,7 @@ def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts
             "ClpStorageEngine": clp_config.package.storage_engine,
         },
     }
-    update_meteor_settings("", meteor_settings, meteor_settings_updates)
+    meteor_settings = read_and_update_settings_json(settings_json_path, meteor_settings_updates)
 
     # Start container
     # fmt: off
@@ -769,24 +783,43 @@ def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts
     logger.info(f"Started {component_name}.")
 
 
-def start_log_viewer_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts):
+def start_log_viewer_webui(
+    instance_id: str,
+    clp_config: CLPConfig,
+    container_clp_config: CLPConfig,
+    mounts: CLPDockerMounts,
+):
     component_name = LOG_VIEWER_WEBUI_COMPONENT_NAME
     logger.info(f"Starting {component_name}...")
 
     container_name = f"clp-{component_name}-{instance_id}"
     if container_exists(container_name):
         return
 
-    log_viewer_webui_logs_dir = clp_config.logs_directory / component_name
-    container_log_viewer_dir = CONTAINER_CLP_HOME / "var" / "www" / "log_viewer"
-    node_path = str(container_log_viewer_dir / "server" / "node_modules")
-
-    validate_log_viewer_config(clp_config, log_viewer_webui_logs_dir)
-
-    # Create directories
-    log_viewer_webui_logs_dir.mkdir(exist_ok=True, parents=True)
+    container_log_viewer_webui_dir = CONTAINER_CLP_HOME / "var" / "www" / "log_viewer_webui"
+    node_path = str(container_log_viewer_webui_dir / "server" / "node_modules")
+    settings_json_path = (
+        get_clp_home() / "var" / "www" / "log_viewer_webui" / "server" / "settings.json"
+    )
 
-    container_log_viewer_webui_logs_dir = pathlib.Path("/") / "var" / "log" / component_name
+    validate_log_viewer_webui_config(clp_config, settings_json_path)
+
+    # Read, update, and write back settings.json
+    settings_json_updates = {
+        "SqlDbHost": clp_config.database.host,
+        "SqlDbPort": clp_config.database.port,
+        "SqlDbName": clp_config.database.name,
+        "SqlDbQueryJobsTableName": QUERY_JOBS_TABLE_NAME,
+        "MongoDbHost": clp_config.results_cache.host,
+        "MongoDbPort": clp_config.results_cache.port,
+        "MongoDbName": clp_config.results_cache.db_name,
+        "MongoDbIrFilesCollectionName": clp_config.results_cache.ir_collection_name,
+        "ClientDir": str(container_log_viewer_webui_dir / "client"),
+        "IrFilesDir": str(container_clp_config.ir_output.directory),
+    }
+    settings_json = read_and_update_settings_json(settings_json_path, settings_json_updates)
+    with open(settings_json_path, "w") as settings_json_file:
+        settings_json_file.write(json.dumps(settings_json))
 
     # Start container
     # fmt: off
@@ -797,19 +830,17 @@ def start_log_viewer_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPD
         "--name", container_name,
         "--log-driver", "local",
         "-e", f"NODE_PATH={node_path}",
-        "-e", f"CLIENT_DIR={container_log_viewer_dir}/client/dist",
-        "-e", f"PORT={clp_config.log_viewer_webui.port}",
         "-e", f"HOST={clp_config.log_viewer_webui.host}",
+        "-e", f"PORT={clp_config.log_viewer_webui.port}",
+        "-e", f"CLP_DB_USER={clp_config.database.username}",
+        "-e", f"CLP_DB_PASS={clp_config.database.password}",
         "-e", f"NODE_ENV=production",
         "-u", f"{os.getuid()}:{os.getgid()}",
     ]
     # fmt: on
     necessary_mounts = [
         mounts.clp_home,
         mounts.ir_output_dir,
-        DockerMount(
-            DockerMountType.BIND, log_viewer_webui_logs_dir, container_log_viewer_webui_logs_dir
-        ),
     ]
     for mount in necessary_mounts:
         if mount:
@@ -819,7 +850,7 @@ def start_log_viewer_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPD
 
     node_cmd = [
         str(CONTAINER_CLP_HOME / "bin" / "node-22"),
-        str(container_log_viewer_dir / "server" / "src" / "main.js"),
+        str(container_log_viewer_webui_dir / "server" / "src" / "main.js"),
     ]
     cmd = container_cmd + node_cmd
     subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True)
@@ -1046,7 +1077,7 @@ def main(argv):
         if target in (ALL_TARGET_NAME, WEBUI_COMPONENT_NAME):
             start_webui(instance_id, clp_config, mounts)
         if target in (ALL_TARGET_NAME, LOG_VIEWER_WEBUI_COMPONENT_NAME):
-            start_log_viewer_webui(instance_id, clp_config, mounts)
+            start_log_viewer_webui(instance_id, clp_config, container_clp_config, mounts)
 
     except Exception as ex:
         if type(ex) == ValueError:

diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py
@@ -393,7 +393,6 @@ def validate_logging_level(cls, field):
 class LogViewerWebUi(BaseModel):
     host: str = "localhost"
     port: int = 3000
-    logging_level: str = "INFO"
 
     @validator("host")
     def validate_host(cls, field):

diff --git a/components/log-viewer-webui/server/.env b/components/log-viewer-webui/server/.env
@@ -1,3 +1,8 @@
 CLIENT_DIR=../client/dist
+IR_DATA_DIR=../../../build/clp-package/var/data/ir
+LOG_VIEWER_DIR=../yscope-log-viewer/dist
+
 HOST=localhost
 PORT=3000
+CLP_DB_USER=clp-user
+CLP_DB_PASS=
diff --git a/components/log-viewer-webui/server/.gitignore b/components/log-viewer-webui/server/.gitignore
@@ -1,2 +1,5 @@
+ # Local development
+.env.local
+
 # Testing
 /.tap