From 812e5fadca66c0d3936ea9e72fbb8b898929a503 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Tue, 9 Jul 2024 16:20:22 -0400 Subject: [PATCH 01/13] Initial change for packing log viewer --- Taskfile.yml | 54 ++++++++++++- .../clp_package_utils/general.py | 14 ++++ .../clp_package_utils/scripts/start_clp.py | 76 +++++++++++++++++-- .../clp-py-utils/clp_py_utils/clp_config.py | 25 ++++++ .../package-template/src/etc/clp-config.yml | 4 + 5 files changed, 162 insertions(+), 11 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 2e3422168..dd66bac70 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -15,7 +15,9 @@ vars: G_PACKAGE_VENV_DIR: "{{.G_BUILD_DIR}}/package-venv" G_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/webui" G_WEBUI_NODEJS_BUILD_DIR: "{{.G_BUILD_DIR}}/webui-nodejs" + G_LOG_VIEWER_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/log-viewer-webui" G_WEBUI_NODEJS_BIN_DIR: "{{.G_WEBUI_NODEJS_BUILD_DIR}}/bin" + G_LOG_VIEWER_WEBUI_BIN_DIR: "{{.G_NODEJS_22_DIR}}/bin" # Versions G_PACKAGE_VERSION: "0.2.0-dev" @@ -81,6 +83,7 @@ tasks: DATA_DIR: "{{.OUTPUT_DIR}}" - "webui" - "webui-nodejs" + - "log-viewer-webui" cmds: - "rm -rf '{{.OUTPUT_DIR}}'" - "rsync -a components/package-template/src/ '{{.OUTPUT_DIR}}'" @@ -101,16 +104,30 @@ tasks: "{{.G_CORE_COMPONENT_BUILD_DIR}}/clp" "{{.G_CORE_COMPONENT_BUILD_DIR}}/clp-s" "{{.G_CORE_COMPONENT_BUILD_DIR}}/reducer-server" - "{{.G_WEBUI_NODEJS_BIN_DIR}}/node" "{{.OUTPUT_DIR}}/bin/" + - >- + rsync -a + "{{.G_WEBUI_NODEJS_BIN_DIR}}/node" + "{{.OUTPUT_DIR}}/bin/node-14" + - >- + rsync -a + "{{.G_LOG_VIEWER_WEBUI_BIN_DIR}}/node" + "{{.OUTPUT_DIR}}/bin/node-22" - "mkdir -p '{{.OUTPUT_DIR}}/var/www/'" - >- rsync -a "{{.G_WEBUI_BUILD_DIR}}/" - "{{.OUTPUT_DIR}}/var/www/" + "{{.OUTPUT_DIR}}/var/www/webui/" - |- - cd "{{.OUTPUT_DIR}}/var/www/programs/server" + cd "{{.OUTPUT_DIR}}/var/www/webui/programs/server" PATH="{{.G_WEBUI_NODEJS_BIN_DIR}}":$PATH npm install + - >- + rsync -a + "{{.G_LOG_VIEWER_WEBUI_BUILD_DIR}}/" + "{{.OUTPUT_DIR}}/var/www/log_viewer/" + - |- + cd "{{.OUTPUT_DIR}}/var/www/log_viewer/server" + PATH="{{.G_LOG_VIEWER_WEBUI_BIN_DIR}}":$PATH npm i # This command must be last - task: "utils:compute-checksum" vars: @@ -174,6 +191,36 @@ tasks: vars: COMPONENT: "{{.TASK}}" + log-viewer-webui: + deps: + - "init" + - "log-viewer-webui-node-modules" + - task: "utils:validate-checksum" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + DATA_DIR: "{{.OUTPUT_DIR}}" + dir: "components/log-viewer-webui" + vars: + CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" + OUTPUT_DIR: "{{.G_LOG_VIEWER_WEBUI_BUILD_DIR}}" + cmds: + - "rm -rf '{{.OUTPUT_DIR}}'" + - "rsync -a client server {{.OUTPUT_DIR}}/" + - |- + cd "{{.OUTPUT_DIR}}/client" + PATH="{{.G_LOG_VIEWER_WEBUI_BIN_DIR}}":$PATH npm run build + - task: "utils:compute-checksum" + vars: + DATA_DIR: "{{.OUTPUT_DIR}}" + OUTPUT_FILE: "{{.CHECKSUM_FILE}}" + sources: + - "{{.G_BUILD_DIR}}/log-viewer-modules.md5" + - "{{.TASKFILE}}" + - "*" + - "client/**/*" + - "server/**/*" + generates: ["{{.CHECKSUM_FILE}}"] + webui: deps: - "init" @@ -306,7 +353,6 @@ tasks: # tasks which depend on this task to only have to check one checksum file, we concatenate the # three checksum files into one. log-viewer-webui-node-modules: - internal: true vars: # Checksum files CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index b84bf298e..a2e6e344f 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -15,6 +15,7 @@ CLP_DEFAULT_CREDENTIALS_FILE_PATH, CLPConfig, DB_COMPONENT_NAME, + LOG_VIEWER_WEBUI_COMPONENT_NAME, QUEUE_COMPONENT_NAME, REDIS_COMPONENT_NAME, REDUCER_COMPONENT_NAME, @@ -494,3 +495,16 @@ def validate_webui_config( raise ValueError(f"{WEBUI_COMPONENT_NAME} logs directory is invalid: {ex}") validate_port(f"{WEBUI_COMPONENT_NAME}.port", clp_config.webui.host, clp_config.webui.port) + + +def validate_log_viewer_config(clp_config: CLPConfig, logs_dir: pathlib.Path): + try: + validate_path_could_be_dir(logs_dir) + except ValueError as ex: + raise ValueError(f"{LOG_VIEWER_WEBUI_COMPONENT_NAME} logs directory is invalid: {ex}") + + validate_port( + f"{LOG_VIEWER_WEBUI_COMPONENT_NAME}.port", + clp_config.log_viewer_webui.host, + clp_config.log_viewer_webui.port, + ) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 9dba79886..584451e9f 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -21,6 +21,7 @@ COMPRESSION_WORKER_COMPONENT_NAME, CONTROLLER_TARGET_NAME, DB_COMPONENT_NAME, + LOG_VIEWER_WEBUI_COMPONENT_NAME, QUERY_JOBS_TABLE_NAME, QUERY_SCHEDULER_COMPONENT_NAME, QUERY_WORKER_COMPONENT_NAME, @@ -49,6 +50,7 @@ validate_and_load_queue_credentials_file, validate_and_load_redis_credentials_file, validate_db_config, + validate_log_viewer_config, validate_queue_config, validate_redis_config, validate_reducer_config, @@ -698,10 +700,9 @@ def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts return webui_logs_dir = clp_config.logs_directory / component_name - node_path = str( - CONTAINER_CLP_HOME / "var" / "www" / "programs" / "server" / "npm" / "node_modules" - ) - settings_json_path = get_clp_home() / "var" / "www" / "settings.json" + container_webui_dir = CONTAINER_CLP_HOME / "var" / "www" / "webui" + node_path = str(container_webui_dir / "programs" / "server" / "npm" / "node_modules") + settings_json_path = get_clp_home() / "var" / "www" / "webui" / "settings.json" validate_webui_config(clp_config, webui_logs_dir, settings_json_path) @@ -758,9 +759,67 @@ def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts container_cmd.append(clp_config.execution_container) node_cmd = [ - str(CONTAINER_CLP_HOME / "bin" / "node"), - str(CONTAINER_CLP_HOME / "var" / "www" / "launcher.js"), - str(CONTAINER_CLP_HOME / "var" / "www" / "main.js"), + str(CONTAINER_CLP_HOME / "bin" / "node-14"), + str(container_webui_dir / "launcher.js"), + str(container_webui_dir / "main.js"), + ] + cmd = container_cmd + node_cmd + subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) + + logger.info(f"Started {component_name}.") + + +def start_log_viewer_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts): + component_name = LOG_VIEWER_WEBUI_COMPONENT_NAME + logger.info(f"Starting {component_name}...") + + container_name = f"clp-{component_name}-{instance_id}" + if container_exists(container_name): + return + + log_viewer_webui_logs_dir = clp_config.logs_directory / component_name + container_log_viewer_dir = CONTAINER_CLP_HOME / "var" / "www" / "log_viewer" + node_path = str(container_log_viewer_dir / "server" / "node_modules") + + validate_log_viewer_config(clp_config, log_viewer_webui_logs_dir) + + # Create directories + log_viewer_webui_logs_dir.mkdir(exist_ok=True, parents=True) + + container_log_viewer_webui_logs_dir = pathlib.Path("/") / "var" / "log" / component_name + + # Start container + # fmt: off + container_cmd = [ + "docker", "run", + "-d", + "--network", "host", + "--name", container_name, + "--log-driver", "local", + "-e", f"NODE_PATH={node_path}", + "-e", f"CLIENT_DIR=../client/dist", + "-e", f"PORT={clp_config.log_viewer_webui.port}", + "-e", f"HOST={clp_config.log_viewer_webui.host}", + "-e", f"NODE_ENV=production", + "-u", f"{os.getuid()}:{os.getgid()}", + ] + # fmt: on + necessary_mounts = [ + mounts.clp_home, + mounts.ir_output_dir, + DockerMount( + DockerMountType.BIND, log_viewer_webui_logs_dir, container_log_viewer_webui_logs_dir + ), + ] + for mount in necessary_mounts: + if mount: + container_cmd.append("--mount") + container_cmd.append(str(mount)) + container_cmd.append(clp_config.execution_container) + + node_cmd = [ + str(CONTAINER_CLP_HOME / "bin" / "node-22"), + str(container_log_viewer_dir / "server" / "src" / "main.js"), ] cmd = container_cmd + node_cmd subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) @@ -897,6 +956,7 @@ def main(argv): COMPRESSION_SCHEDULER_COMPONENT_NAME, QUERY_SCHEDULER_COMPONENT_NAME, WEBUI_COMPONENT_NAME, + LOG_VIEWER_WEBUI_COMPONENT_NAME, ): validate_and_load_db_credentials_file(clp_config, clp_home, True) if target in ( @@ -984,6 +1044,8 @@ def main(argv): start_reducer(instance_id, clp_config, container_clp_config, num_workers, mounts) if target in (ALL_TARGET_NAME, WEBUI_COMPONENT_NAME): start_webui(instance_id, clp_config, mounts) + if target in (ALL_TARGET_NAME, LOG_VIEWER_WEBUI_COMPONENT_NAME): + start_log_viewer_webui(instance_id, clp_config, mounts) except Exception as ex: if type(ex) == ValueError: diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 0c0ce6893..d9e40c5c5 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -26,6 +26,7 @@ COMPRESSION_WORKER_COMPONENT_NAME = "compression_worker" QUERY_WORKER_COMPONENT_NAME = "query_worker" WEBUI_COMPONENT_NAME = "webui" +LOG_VIEWER_WEBUI_COMPONENT_NAME = "log_viewer_webui" # Target names ALL_TARGET_NAME = "" @@ -382,6 +383,29 @@ def validate_logging_level(cls, field): return field +class LogViewerWebUi(BaseModel): + host: str = "localhost" + port: int = 3000 + logging_level: str = "INFO" + + @validator("host") + def validate_host(cls, field): + if "" == field: + raise ValueError(f"{LOG_VIEWER_WEBUI_COMPONENT_NAME}.host cannot be empty.") + return field + + @validator("port") + def validate_port(cls, field): + min_valid_port = 0 + max_valid_port = 2**16 - 1 + if min_valid_port > field or max_valid_port < field: + raise ValueError( + f"{LOG_VIEWER_WEBUI_COMPONENT_NAME}.port is not within valid range " + f"{min_valid_port}-{max_valid_port}." + ) + return field + + class CLPConfig(BaseModel): execution_container: typing.Optional[str] @@ -398,6 +422,7 @@ class CLPConfig(BaseModel): compression_worker: CompressionWorker = CompressionWorker() query_worker: QueryWorker = QueryWorker() webui: WebUi = WebUi() + log_viewer_webui: LogViewerWebUi = LogViewerWebUi() credentials_file_path: pathlib.Path = CLP_DEFAULT_CREDENTIALS_FILE_PATH archive_output: ArchiveOutput = ArchiveOutput() diff --git a/components/package-template/src/etc/clp-config.yml b/components/package-template/src/etc/clp-config.yml index 3f658211e..ce626afa3 100644 --- a/components/package-template/src/etc/clp-config.yml +++ b/components/package-template/src/etc/clp-config.yml @@ -60,6 +60,10 @@ # port: 4000 # logging_level: "INFO" # +#log_viewer_webui +# host: "localhost" +# port: 3000 +# ## Where archives should be output to #archive_output: # directory: "var/data/archives" From cedf6d3bbf9ad6f4c1edfff4c8dba68c3d77c7c4 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Tue, 9 Jul 2024 22:23:53 -0400 Subject: [PATCH 02/13] add missing stop statement --- .../clp-package-utils/clp_package_utils/scripts/stop_clp.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index 7971dd7d8..f0c9893b9 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -11,6 +11,7 @@ COMPRESSION_WORKER_COMPONENT_NAME, CONTROLLER_TARGET_NAME, DB_COMPONENT_NAME, + LOG_VIEWER_WEBUI_COMPONENT_NAME, QUERY_SCHEDULER_COMPONENT_NAME, QUERY_WORKER_COMPONENT_NAME, QUEUE_COMPONENT_NAME, @@ -134,6 +135,9 @@ def main(argv): already_exited_containers = [] force = parsed_args.force + if target in (ALL_TARGET_NAME, LOG_VIEWER_WEBUI_COMPONENT_NAME): + container_name = f"clp-{LOG_VIEWER_WEBUI_COMPONENT_NAME}-{instance_id}" + stop_running_container(container_name, already_exited_containers, force) if target in (ALL_TARGET_NAME, WEBUI_COMPONENT_NAME): container_name = f"clp-{WEBUI_COMPONENT_NAME}-{instance_id}" stop_running_container(container_name, already_exited_containers, force) From e02f4a6d92167acb16683db25190b54a1052e6c2 Mon Sep 17 00:00:00 2001 From: haiqi96 <14502009+haiqi96@users.noreply.github.com> Date: Wed, 10 Jul 2024 10:09:54 -0400 Subject: [PATCH 03/13] Apply suggestions from code review Co-authored-by: Junhao Liao --- Taskfile.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index dd66bac70..ce7a2c563 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -120,14 +120,14 @@ tasks: "{{.OUTPUT_DIR}}/var/www/webui/" - |- cd "{{.OUTPUT_DIR}}/var/www/webui/programs/server" - PATH="{{.G_WEBUI_NODEJS_BIN_DIR}}":$PATH npm install + PATH="{{.G_WEBUI_NODEJS_BIN_DIR}}":$PATH npm clean-install - >- rsync -a "{{.G_LOG_VIEWER_WEBUI_BUILD_DIR}}/" "{{.OUTPUT_DIR}}/var/www/log_viewer/" - |- cd "{{.OUTPUT_DIR}}/var/www/log_viewer/server" - PATH="{{.G_LOG_VIEWER_WEBUI_BIN_DIR}}":$PATH npm i + PATH="{{.G_LOG_VIEWER_WEBUI_BIN_DIR}}":$PATH npm clean-install # This command must be last - task: "utils:compute-checksum" vars: @@ -216,9 +216,12 @@ tasks: sources: - "{{.G_BUILD_DIR}}/log-viewer-modules.md5" - "{{.TASKFILE}}" - - "*" - - "client/**/*" - - "server/**/*" + - "client/src/**/*.css" + - "client/src/**/*.jsx" + - "client/src/package.json" + - "client/src/webpack.config.js" + - "server/src/**/*.js" + - "server/src/**/package.json" generates: ["{{.CHECKSUM_FILE}}"] webui: From 99c58723947928117e465cb82cf5b445b91ac9f9 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Wed, 10 Jul 2024 11:37:16 -0400 Subject: [PATCH 04/13] address code review comments --- Taskfile.yml | 5 ++- .../clp-py-utils/clp_py_utils/clp_config.py | 37 ++++++++++--------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index dd66bac70..e626b0ca4 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -9,15 +9,15 @@ vars: # Paths G_BUILD_DIR: "{{.ROOT_DIR}}/build" G_CORE_COMPONENT_BUILD_DIR: "{{.G_BUILD_DIR}}/core" + G_LOG_VIEWER_WEBUI_BIN_DIR: "{{.G_NODEJS_22_DIR}}/bin" + G_LOG_VIEWER_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/log-viewer-webui" G_METEOR_BUILD_DIR: "{{.G_BUILD_DIR}}/meteor" G_NODEJS_22_DIR: "{{.G_BUILD_DIR}}/nodejs-22" G_PACKAGE_BUILD_DIR: "{{.G_BUILD_DIR}}/clp-package" G_PACKAGE_VENV_DIR: "{{.G_BUILD_DIR}}/package-venv" G_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/webui" G_WEBUI_NODEJS_BUILD_DIR: "{{.G_BUILD_DIR}}/webui-nodejs" - G_LOG_VIEWER_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/log-viewer-webui" G_WEBUI_NODEJS_BIN_DIR: "{{.G_WEBUI_NODEJS_BUILD_DIR}}/bin" - G_LOG_VIEWER_WEBUI_BIN_DIR: "{{.G_NODEJS_22_DIR}}/bin" # Versions G_PACKAGE_VERSION: "0.2.0-dev" @@ -353,6 +353,7 @@ tasks: # tasks which depend on this task to only have to check one checksum file, we concatenate the # three checksum files into one. log-viewer-webui-node-modules: + internal: true vars: # Checksum files CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index d9e40c5c5..64c663285 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -154,6 +154,21 @@ def _validate_logging_level(cls, field): ) +def _validate_host(cls, field): + if "" == field: + raise ValueError(f"{cls.__name__}.host cannot be empty.") + + +def _validate_port(cls, field): + min_valid_port = 0 + max_valid_port = 2 ** 16 - 1 + if min_valid_port > field or max_valid_port < field: + raise ValueError( + f"{cls.__name__}.port is not within valid range " + f"{min_valid_port}-{max_valid_port}." + ) + + class CompressionScheduler(BaseModel): jobs_poll_delay: float = 0.1 # seconds logging_level: str = "INFO" @@ -362,19 +377,12 @@ class WebUi(BaseModel): @validator("host") def validate_host(cls, field): - if "" == field: - raise ValueError(f"{WEBUI_COMPONENT_NAME}.host cannot be empty.") + _validate_host(cls, field) return field @validator("port") def validate_port(cls, field): - min_valid_port = 0 - max_valid_port = 2**16 - 1 - if min_valid_port > field or max_valid_port < field: - raise ValueError( - f"{WEBUI_COMPONENT_NAME}.port is not within valid range " - f"{min_valid_port}-{max_valid_port}." - ) + _validate_port(cls, field) return field @validator("logging_level") @@ -390,19 +398,12 @@ class LogViewerWebUi(BaseModel): @validator("host") def validate_host(cls, field): - if "" == field: - raise ValueError(f"{LOG_VIEWER_WEBUI_COMPONENT_NAME}.host cannot be empty.") + _validate_host(cls, field) return field @validator("port") def validate_port(cls, field): - min_valid_port = 0 - max_valid_port = 2**16 - 1 - if min_valid_port > field or max_valid_port < field: - raise ValueError( - f"{LOG_VIEWER_WEBUI_COMPONENT_NAME}.port is not within valid range " - f"{min_valid_port}-{max_valid_port}." - ) + _validate_port(cls, field) return field From 89b78543dcb8559739fb7360e80d223b9ace9c71 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Wed, 10 Jul 2024 16:27:18 -0400 Subject: [PATCH 05/13] fix --- Taskfile.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 2b30c4aae..367172b18 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -9,10 +9,10 @@ vars: # Paths G_BUILD_DIR: "{{.ROOT_DIR}}/build" G_CORE_COMPONENT_BUILD_DIR: "{{.G_BUILD_DIR}}/core" - G_LOG_VIEWER_WEBUI_BIN_DIR: "{{.G_NODEJS_22_DIR}}/bin" - G_LOG_VIEWER_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/log-viewer-webui" G_METEOR_BUILD_DIR: "{{.G_BUILD_DIR}}/meteor" G_NODEJS_22_DIR: "{{.G_BUILD_DIR}}/nodejs-22" + G_LOG_VIEWER_WEBUI_BIN_DIR: "{{.G_NODEJS_22_DIR}}/bin" + G_LOG_VIEWER_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/log-viewer-webui" G_PACKAGE_BUILD_DIR: "{{.G_BUILD_DIR}}/clp-package" G_PACKAGE_VENV_DIR: "{{.G_BUILD_DIR}}/package-venv" G_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/webui" From 01d502c969b1b5c080f72fab9e4a3b40fe31b0f5 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Wed, 10 Jul 2024 17:26:41 -0400 Subject: [PATCH 06/13] fixes --- Taskfile.yml | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 367172b18..7a8373889 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -9,10 +9,10 @@ vars: # Paths G_BUILD_DIR: "{{.ROOT_DIR}}/build" G_CORE_COMPONENT_BUILD_DIR: "{{.G_BUILD_DIR}}/core" + G_LOG_VIEWER_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/log-viewer-webui" G_METEOR_BUILD_DIR: "{{.G_BUILD_DIR}}/meteor" G_NODEJS_22_DIR: "{{.G_BUILD_DIR}}/nodejs-22" - G_LOG_VIEWER_WEBUI_BIN_DIR: "{{.G_NODEJS_22_DIR}}/bin" - G_LOG_VIEWER_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/log-viewer-webui" + G_NODEJS_22_BIN_DIR: "{{.G_NODEJS_22_DIR}}/bin" G_PACKAGE_BUILD_DIR: "{{.G_BUILD_DIR}}/clp-package" G_PACKAGE_VENV_DIR: "{{.G_BUILD_DIR}}/package-venv" G_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/webui" @@ -70,12 +70,15 @@ tasks: vars: CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" OUTPUT_DIR: "{{.G_PACKAGE_BUILD_DIR}}" + env: + NODE_ENV: "production" deps: - "core" - "clp-package-utils" - "clp-py-utils" - "init" - "job-orchestration" + - "log-viewer-webui" - "package-venv" - task: "utils:validate-checksum" vars: @@ -83,7 +86,6 @@ tasks: DATA_DIR: "{{.OUTPUT_DIR}}" - "webui" - "webui-nodejs" - - "log-viewer-webui" cmds: - "rm -rf '{{.OUTPUT_DIR}}'" - "rsync -a components/package-template/src/ '{{.OUTPUT_DIR}}'" @@ -111,23 +113,25 @@ tasks: "{{.OUTPUT_DIR}}/bin/node-14" - >- rsync -a - "{{.G_LOG_VIEWER_WEBUI_BIN_DIR}}/node" + "{{.G_NODEJS_22_BIN_DIR}}/node" "{{.OUTPUT_DIR}}/bin/node-22" - "mkdir -p '{{.OUTPUT_DIR}}/var/www/'" - >- rsync -a "{{.G_WEBUI_BUILD_DIR}}/" "{{.OUTPUT_DIR}}/var/www/webui/" + # Don't use clean-install because meteor doesn't generate + # package-lock.json that clean-install depends on - |- cd "{{.OUTPUT_DIR}}/var/www/webui/programs/server" - PATH="{{.G_WEBUI_NODEJS_BIN_DIR}}":$PATH npm clean-install + PATH="{{.G_WEBUI_NODEJS_BIN_DIR}}":$PATH npm install - >- rsync -a "{{.G_LOG_VIEWER_WEBUI_BUILD_DIR}}/" "{{.OUTPUT_DIR}}/var/www/log_viewer/" - |- cd "{{.OUTPUT_DIR}}/var/www/log_viewer/server" - PATH="{{.G_LOG_VIEWER_WEBUI_BIN_DIR}}":$PATH npm clean-install + PATH="{{.G_NODEJS_22_BIN_DIR}}":$PATH npm clean-install # This command must be last - task: "utils:compute-checksum" vars: @@ -205,10 +209,14 @@ tasks: OUTPUT_DIR: "{{.G_LOG_VIEWER_WEBUI_BUILD_DIR}}" cmds: - "rm -rf '{{.OUTPUT_DIR}}'" - - "rsync -a client server {{.OUTPUT_DIR}}/" + - "rsync -a client {{.OUTPUT_DIR}}/" - |- cd "{{.OUTPUT_DIR}}/client" - PATH="{{.G_LOG_VIEWER_WEBUI_BIN_DIR}}":$PATH npm run build + PATH="{{.G_NODEJS_22_BIN_DIR}}":$PATH npm run build + - "mkdir {{.OUTPUT_DIR}}/server" + - |- + cd server + rsync -a src package-lock.json package.json {{.OUTPUT_DIR}}/server/ - task: "utils:compute-checksum" vars: DATA_DIR: "{{.OUTPUT_DIR}}" From ddbf688e9085985668f9f2c633aa5982e2f40769 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:01:48 -0400 Subject: [PATCH 07/13] linter --- components/clp-py-utils/clp_py_utils/clp_config.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 64c663285..9485b43c2 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -161,11 +161,10 @@ def _validate_host(cls, field): def _validate_port(cls, field): min_valid_port = 0 - max_valid_port = 2 ** 16 - 1 + max_valid_port = 2**16 - 1 if min_valid_port > field or max_valid_port < field: raise ValueError( - f"{cls.__name__}.port is not within valid range " - f"{min_valid_port}-{max_valid_port}." + f"{cls.__name__}.port is not within valid range " f"{min_valid_port}-{max_valid_port}." ) From 614d6b050d529834f80c7da9a344eeafa88b0e5d Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:58:44 -0400 Subject: [PATCH 08/13] buggy but don't understand why --- Taskfile.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 7a8373889..0cf407fd4 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -11,13 +11,13 @@ vars: G_CORE_COMPONENT_BUILD_DIR: "{{.G_BUILD_DIR}}/core" G_LOG_VIEWER_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/log-viewer-webui" G_METEOR_BUILD_DIR: "{{.G_BUILD_DIR}}/meteor" - G_NODEJS_22_DIR: "{{.G_BUILD_DIR}}/nodejs-22" - G_NODEJS_22_BIN_DIR: "{{.G_NODEJS_22_DIR}}/bin" + G_NODEJS_14_BUILD_DIR: "{{.G_BUILD_DIR}}/nodejs-14" + G_NODEJS_14_BIN_DIR: "{{.G_NODEJS_14_BUILD_DIR}}/bin" + G_NODEJS_22_BUILD_DIR: "{{.G_BUILD_DIR}}/nodejs-22" + G_NODEJS_22_BIN_DIR: "{{.G_NODEJS_22_BUILD_DIR}}/bin" G_PACKAGE_BUILD_DIR: "{{.G_BUILD_DIR}}/clp-package" G_PACKAGE_VENV_DIR: "{{.G_BUILD_DIR}}/package-venv" G_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/webui" - G_WEBUI_NODEJS_BUILD_DIR: "{{.G_BUILD_DIR}}/webui-nodejs" - G_WEBUI_NODEJS_BIN_DIR: "{{.G_WEBUI_NODEJS_BUILD_DIR}}/bin" # Versions G_PACKAGE_VERSION: "0.2.0-dev" @@ -85,7 +85,7 @@ tasks: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" DATA_DIR: "{{.OUTPUT_DIR}}" - "webui" - - "webui-nodejs" + - "nodejs-14" cmds: - "rm -rf '{{.OUTPUT_DIR}}'" - "rsync -a components/package-template/src/ '{{.OUTPUT_DIR}}'" @@ -109,7 +109,7 @@ tasks: "{{.OUTPUT_DIR}}/bin/" - >- rsync -a - "{{.G_WEBUI_NODEJS_BIN_DIR}}/node" + "{{.G_NODEJS_14_BIN_DIR}}/node" "{{.OUTPUT_DIR}}/bin/node-14" - >- rsync -a @@ -124,7 +124,7 @@ tasks: # package-lock.json that clean-install depends on - |- cd "{{.OUTPUT_DIR}}/var/www/webui/programs/server" - PATH="{{.G_WEBUI_NODEJS_BIN_DIR}}":$PATH npm install + PATH="{{.G_NODEJS_14_BIN_DIR}}":$PATH npm install - >- rsync -a "{{.G_LOG_VIEWER_WEBUI_BUILD_DIR}}/" @@ -280,7 +280,7 @@ tasks: internal: true vars: CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" - OUTPUT_DIR: "{{.G_NODEJS_22_DIR}}" + OUTPUT_DIR: "{{.G_NODEJS_22_BUILD_DIR}}" run: "once" cmds: - task: "nodejs" @@ -289,11 +289,11 @@ tasks: NODEJS_VERSION: "v22.4.0" OUTPUT_DIR: "{{.OUTPUT_DIR}}" - webui-nodejs: + nodejs-14: internal: true vars: CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" - OUTPUT_DIR: "{{.G_WEBUI_NODEJS_BUILD_DIR}}" + OUTPUT_DIR: "{{.G_NODEJS_14_BUILD_DIR}}" cmds: - task: "nodejs" vars: @@ -395,7 +395,7 @@ tasks: cmds: - "rm -f {{.CHECKSUM_FILE}}" - task: "clean-log-viewer-webui" - - "PATH='{{.G_NODEJS_22_DIR}}/bin':$PATH npm run init" + - "PATH='{{.G_NODEJS_22_BIN_DIR}}':$PATH npm run init" # These commands must be last - task: "utils:compute-checksum" vars: From aad6068a360c77243d4dd9a8c6e3dc2c4b60535e Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Thu, 11 Jul 2024 15:07:24 -0400 Subject: [PATCH 09/13] fixes --- lint-tasks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint-tasks.yml b/lint-tasks.yml index 075ee2ec8..6b3f3ca26 100644 --- a/lint-tasks.yml +++ b/lint-tasks.yml @@ -3,7 +3,7 @@ version: "3" vars: G_LINT_VENV_DIR: "{{.G_BUILD_DIR}}/lint-venv" G_LOG_VIEWER_WEBUI_SRC_DIR: "{{.ROOT_DIR}}/components/log-viewer-webui" - G_NODEJS_22_BIN_DIR: "{{.G_NODEJS_22_DIR}}/bin" + G_NODEJS_22_BIN_DIR: "{{.G_NODEJS_22_BUILD_DIR}}/bin" G_WEBUI_SRC_DIR: "{{.ROOT_DIR}}/components/webui" tasks: From 9943627fc36bba10e4950a2d7035fb57afa56a36 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Thu, 11 Jul 2024 17:05:02 -0400 Subject: [PATCH 10/13] deduplicate --- lint-tasks.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/lint-tasks.yml b/lint-tasks.yml index 6b3f3ca26..9cd3f1435 100644 --- a/lint-tasks.yml +++ b/lint-tasks.yml @@ -3,7 +3,6 @@ version: "3" vars: G_LINT_VENV_DIR: "{{.G_BUILD_DIR}}/lint-venv" G_LOG_VIEWER_WEBUI_SRC_DIR: "{{.ROOT_DIR}}/components/log-viewer-webui" - G_NODEJS_22_BIN_DIR: "{{.G_NODEJS_22_BUILD_DIR}}/bin" G_WEBUI_SRC_DIR: "{{.ROOT_DIR}}/components/webui" tasks: From 55a7453c1d4b9c38157708745ab20f5cf5974050 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Fri, 12 Jul 2024 09:55:27 -0400 Subject: [PATCH 11/13] reorder target --- Taskfile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Taskfile.yml b/Taskfile.yml index 0cf407fd4..3baa609fd 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -79,13 +79,13 @@ tasks: - "init" - "job-orchestration" - "log-viewer-webui" + - "nodejs-14" - "package-venv" - task: "utils:validate-checksum" vars: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" DATA_DIR: "{{.OUTPUT_DIR}}" - "webui" - - "nodejs-14" cmds: - "rm -rf '{{.OUTPUT_DIR}}'" - "rsync -a components/package-template/src/ '{{.OUTPUT_DIR}}'" From e602f157807bf0e0a98e4a9d9c6661fae431eb0e Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Fri, 12 Jul 2024 10:15:19 -0400 Subject: [PATCH 12/13] add missing changes in clp scripts --- .../clp_package_utils/scripts/start_clp.py | 3 ++- .../clp_package_utils/scripts/stop_clp.py | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 584451e9f..4c072752e 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -797,7 +797,7 @@ def start_log_viewer_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPD "--name", container_name, "--log-driver", "local", "-e", f"NODE_PATH={node_path}", - "-e", f"CLIENT_DIR=../client/dist", + "-e", f"CLIENT_DIR={container_log_viewer_dir}/client/dist", "-e", f"PORT={clp_config.log_viewer_webui.port}", "-e", f"HOST={clp_config.log_viewer_webui.host}", "-e", f"NODE_ENV=production", @@ -929,6 +929,7 @@ def main(argv): reducer_server_parser = component_args_parser.add_parser(REDUCER_COMPONENT_NAME) add_num_workers_argument(reducer_server_parser) component_args_parser.add_parser(WEBUI_COMPONENT_NAME) + component_args_parser.add_parser(LOG_VIEWER_WEBUI_COMPONENT_NAME) parsed_args = args_parser.parse_args(argv[1:]) diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index f0c9893b9..f100a098a 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -93,6 +93,7 @@ def main(argv): component_args_parser.add_parser(COMPRESSION_WORKER_COMPONENT_NAME) component_args_parser.add_parser(QUERY_WORKER_COMPONENT_NAME) component_args_parser.add_parser(WEBUI_COMPONENT_NAME) + component_args_parser.add_parser(LOG_VIEWER_WEBUI_COMPONENT_NAME) parsed_args = args_parser.parse_args(argv[1:]) @@ -107,7 +108,12 @@ def main(argv): clp_config = load_config_file(config_file_path, default_config_file_path, clp_home) # Validate and load necessary credentials - if target in (ALL_TARGET_NAME, CONTROLLER_TARGET_NAME, DB_COMPONENT_NAME): + if target in ( + ALL_TARGET_NAME, + CONTROLLER_TARGET_NAME, + DB_COMPONENT_NAME, + LOG_VIEWER_WEBUI_COMPONENT_NAME, + ): validate_and_load_db_credentials_file(clp_config, clp_home, False) if target in ( ALL_TARGET_NAME, From c97816b4523cb900b424bf4baca8ae3141733752 Mon Sep 17 00:00:00 2001 From: haiqi96 <14502009+haiqi96@users.noreply.github.com> Date: Fri, 12 Jul 2024 15:24:25 -0400 Subject: [PATCH 13/13] Apply suggestions from code review Co-authored-by: Junhao Liao --- Taskfile.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 3baa609fd..9763307a7 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -67,11 +67,11 @@ tasks: STORAGE_ENGINE: "clp" package: + env: + NODE_ENV: "production" vars: CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" OUTPUT_DIR: "{{.G_PACKAGE_BUILD_DIR}}" - env: - NODE_ENV: "production" deps: - "core" - "clp-package-utils" @@ -120,8 +120,8 @@ tasks: rsync -a "{{.G_WEBUI_BUILD_DIR}}/" "{{.OUTPUT_DIR}}/var/www/webui/" - # Don't use clean-install because meteor doesn't generate - # package-lock.json that clean-install depends on + # Avoid using `npm clean-install` because Meteor does not generate a `package-lock.json` file, + # which `clean-install` depends on. - |- cd "{{.OUTPUT_DIR}}/var/www/webui/programs/server" PATH="{{.G_NODEJS_14_BIN_DIR}}":$PATH npm install