Add Web UI to support querying from browsers.

Taskfile.yml

@@ -3,6 +3,9 @@ version: "3"
 vars:
   BUILD_DIR: "{{.TASKFILE_DIR}}/build"
   CORE_COMPONENT_BUILD_DIR: "{{.TASKFILE_DIR}}/build/core"
+  WEBUI_BUILD_DIR: "{{.TASKFILE_DIR}}/build/webui"
+  NODEJS_BUILD_DIR: "{{.TASKFILE_DIR}}/build/nodejs"
+  NODEJS_BIN_DIR: "{{.NODEJS_BUILD_DIR}}/node/bin"
   LINT_VENV_DIR: "{{.TASKFILE_DIR}}/.lint-venv"
   PACKAGE_BUILD_DIR: "{{.TASKFILE_DIR}}/build/clp-package"
   PACKAGE_VENV_DIR: "{{.TASKFILE_DIR}}/build/package-venv"
@@ -29,7 +32,6 @@ tasks:
       - task: "clean-python-component"
         vars:
           COMPONENT: "job-orchestration"
-
   clean-package:
     cmds:
       - "rm -rf '{{.PACKAGE_BUILD_DIR}}'"
@@ -64,6 +66,7 @@ tasks:
       - "compression-job-handler"
       - "job-orchestration"
       - "package-venv"
+      - "webui"
     cmds:
       - task: "clean-package"
       - "mkdir -p '{{.PACKAGE_BUILD_DIR}}'"
@@ -83,7 +86,13 @@ tasks:
         "{{.CORE_COMPONENT_BUILD_DIR}}/clg"
         "{{.CORE_COMPONENT_BUILD_DIR}}/clo"
         "{{.CORE_COMPONENT_BUILD_DIR}}/clp"
+        "{{.BUILD_DIR}}/nodejs/node/bin/node"
         "{{.PACKAGE_BUILD_DIR}}/bin/"
+      - "mkdir -p '{{.PACKAGE_BUILD_DIR}}/var/www/'"
+      - >-
+        rsync -a --delete
+        "{{.WEBUI_BUILD_DIR}}/"
+        "{{.PACKAGE_BUILD_DIR}}/var/www/"
       # This step must be last since we use this file to detect whether the package was built
       # successfully
       - "echo {{.PACKAGE_VERSION}} > '{{.PACKAGE_VERSION_FILE}}'"
@@ -92,6 +101,7 @@ tasks:
       - "{{.CORE_COMPONENT_BUILD_DIR}}/clg"
       - "{{.CORE_COMPONENT_BUILD_DIR}}/clo"
       - "{{.CORE_COMPONENT_BUILD_DIR}}/clp"
+      - "{{.BUILD_DIR}}/webui/built/**/*"
       - "{{.TASKFILE_DIR}}/Taskfile.yml"
       - "components/clp-package-utils/dist/*.whl"
       - "components/clp-py-utils/dist/*.whl"
@@ -102,6 +112,52 @@ tasks:
       - "test -e '{{.PACKAGE_VERSION_FILE}}'"
       - "test {{.TIMESTAMP | unixEpoch}} -lt $(stat --format %Y '{{.PACKAGE_VERSION_FILE}}')"
 
+  webui:
+    deps:
+      - "nodejs"
+    dir: "components/webui"
+    cmds:
+      - "rm -rf '{{.WEBUI_BUILD_DIR}}'"
+      - "mkdir -p {{.WEBUI_BUILD_DIR}}"
+      - "meteor npm install --production"
+      - "meteor build --directory {{.WEBUI_BUILD_DIR}}"
+      - >-
+        rsync -a
+        "{{.WEBUI_BUILD_DIR}}/bundle/"
+        "$PWD/settings.json"
+        "$PWD/launcher.js"
+        "{{.WEBUI_BUILD_DIR}}/"
+      - "rm -rf '{{.WEBUI_BUILD_DIR}}/bundle/'"
+      - |-
+        cd {{.WEBUI_BUILD_DIR}}/programs/server
+        PATH={{.NODEJS_BIN_DIR}}:$PATH $(readlink -f {{.NODEJS_BIN_DIR}}/npm) install
+    sources:
+      - "./.meteor/*"
+      - "./client/**/*"
+      - "./imports/**/*"
+      - "./server/**/*"
+      - "./tests/**/*"
+      - "./*"
+
+  nodejs:
+    vars:
+      NODEJS_VERSION: "14.21.3"
+      NODEJS_RELEASE: "v{{.NODEJS_VERSION}}-linux-x64"
+      TAR_FILENAME: "node-{{.NODEJS_RELEASE}}.tar.xz"
+      TAR_FILE_PATH: "{{.NODEJS_BUILD_DIR}}/{{.TAR_FILENAME}}"
+    cmds:
+      - "rm -rf '{{.NODEJS_BUILD_DIR}}/node'"
+      - "mkdir -p {{.NODEJS_BUILD_DIR}}"
+      - >-
+        curl -fsSL
+        https://nodejs.org/dist/v{{.NODEJS_VERSION}}/{{.TAR_FILENAME}}
+        -o {{.TAR_FILE_PATH}}
+      - "tar xf {{.TAR_FILE_PATH}} -C {{.NODEJS_BUILD_DIR}}"
+      - "mv {{.NODEJS_BUILD_DIR}}/node-{{.NODEJS_RELEASE}} {{.NODEJS_BUILD_DIR}}/node"
+      - "rm -f '{{.TAR_FILE_PATH}}'"
+    sources:
+      - "{{.NODEJS_BUILD_DIR}}/node/**/*"
+
   core:
     deps: ["core-submodules"]
     vars:

components/clp-package-utils/clp_package_utils/general.py

@@ -14,13 +14,14 @@
     CLP_DEFAULT_CREDENTIALS_FILE_PATH,
     DB_COMPONENT_NAME,
     QUEUE_COMPONENT_NAME,
-    RESULTS_CACHE_COMPONENT_NAME
+    RESULTS_CACHE_COMPONENT_NAME,
+    WEBUI_COMPONENT_NAME,
 )
 from clp_py_utils.core import (
     get_config_value,
     make_config_path_absolute,
     read_yaml_config_file,
-    validate_path_could_be_dir
+    validate_path_could_be_dir,
 )
 
 # CONSTANTS
@@ -60,6 +61,7 @@ def __init__(self, clp_home: pathlib.Path, docker_clp_home: pathlib.Path):
         self.logs_dir: typing.Optional[DockerMount] = None
         self.archives_output_dir: typing.Optional[DockerMount] = None
 
+
 def get_clp_home():
     # Determine CLP_HOME from an environment variable or this script's path
     clp_home = None
@@ -78,6 +80,7 @@ def get_clp_home():
 
     return clp_home.resolve()
 
+
 def check_dependencies():
     try:
         subprocess.run("command -v docker", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True)
@@ -277,3 +280,16 @@ def validate_results_cache_config(clp_config: CLPConfig, data_dir: pathlib.Path,
 def validate_worker_config(clp_config: CLPConfig):
     clp_config.validate_input_logs_dir()
     clp_config.validate_archive_output_dir()
+
+
+def validate_webui_config(clp_config: CLPConfig, logs_dir: pathlib.Path, settings_json_path: pathlib.Path):
+    if not settings_json_path.exists():
+        raise ValueError(f"{WEBUI_COMPONENT_NAME} {settings_json_path} is not a valid path to Meteor settings.json")
+
+    try:
+        validate_path_could_be_dir(logs_dir)
+    except ValueError as ex:
+        raise ValueError(f"{WEBUI_COMPONENT_NAME} logs directory is invalid: {ex}")
+
+    validate_port(f"{WEBUI_COMPONENT_NAME}.port", clp_config.webui.host,
+                  clp_config.webui.port)

components/clp-package-utils/clp_package_utils/scripts/start_clp.py

@@ -1,4 +1,5 @@
 import argparse
+import json
 import logging
 import multiprocessing
 import os
@@ -29,6 +30,7 @@
     validate_db_config,
     validate_queue_config,
     validate_results_cache_config,
+    validate_webui_config,
     validate_worker_config
 )
 from clp_py_utils.clp_config import (
@@ -40,6 +42,7 @@
     SEARCH_SCHEDULER_COMPONENT_NAME,
     SEARCH_WORKER_COMPONENT_NAME,
     WORKER_COMPONENT_NAME,
+    WEBUI_COMPONENT_NAME,
 )
 from job_orchestration.scheduler.constants import QueueName
 
@@ -544,6 +547,70 @@ def start_worker(instance_id: str, clp_config: CLPConfig, container_clp_config:
     logger.info(f"Started {WORKER_COMPONENT_NAME}.")
 
 
+def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts):
+    logger.info(f"Starting {WEBUI_COMPONENT_NAME}...")
+
+    container_name = f'clp-{WEBUI_COMPONENT_NAME}-{instance_id}'
+    if container_exists(container_name):
+        logger.info(f"{WEBUI_COMPONENT_NAME} already running.")
+        return
+
+    webui_logs_dir = clp_config.logs_directory / WEBUI_COMPONENT_NAME
+    node_path = str(CONTAINER_CLP_HOME / 'var' / 'www' / 'programs'/'server'/'npm'/'node_modules')
+    settings_json_path = get_clp_home() / 'var' / 'www' / 'settings.json'
+
+    validate_webui_config(clp_config, webui_logs_dir, settings_json_path)
+
+    # Create directories
+    webui_logs_dir.mkdir(exist_ok=True, parents=True)
+
+    container_webui_logs_dir = pathlib.Path('/') / 'var' / 'log' / WEBUI_COMPONENT_NAME
+    with open(settings_json_path, 'r') as settings_json_file:
+        settings_json_content = settings_json_file.read()
+        meteor_settings = json.loads(settings_json_content)
+
+    # Start container
+    container_cmd = [
+        'docker', 'run',
+        '-d',
+        '--network', 'host',
+        '--rm',
+        '--name', container_name,
+        '-e', f"NODE_PATH={node_path}",
+        '-e', f'MONGO_URL={clp_config.results_cache.get_uri()}',
+        '-e', f'PORT={clp_config.webui.port}',
+        '-e', f'ROOT_URL=http://{clp_config.webui.host}',
+        '-e', f'METEOR_SETTINGS={json.dumps(meteor_settings)}',
+        '-e', f'CLP_DB_HOST={clp_config.database.host}',
+        '-e', f'CLP_DB_PORT={clp_config.database.port}',
+        '-e', f'CLP_DB_NAME={clp_config.database.name}',
+        '-e', f'CLP_DB_USER={clp_config.database.username}',
+        '-e', f'CLP_DB_PASS={clp_config.database.password}',
+        '-e', f'WEBUI_LOGS_DIR={container_webui_logs_dir}',
+        '-e', f'WEBUI_LOGGING_LEVEL={clp_config.webui.logging_level}',
+        '-u', f'{os.getuid()}:{os.getgid()}',
+    ]
+    necessary_mounts = [
+        mounts.clp_home,
+        DockerMount(DockerMountType.BIND, webui_logs_dir, container_webui_logs_dir),
+    ]
+    for mount in necessary_mounts:
+        if mount:
+            container_cmd.append('--mount')
+            container_cmd.append(str(mount))
+    container_cmd.append(clp_config.execution_container)
+
+    node_cmd = [
+        str(CONTAINER_CLP_HOME / 'bin' / 'node'),
+        str(CONTAINER_CLP_HOME / 'var' / 'www' / 'launcher.js'),
+        str(CONTAINER_CLP_HOME / 'var' / 'www' / 'main.js')
+    ]
+    cmd = container_cmd + node_cmd
+    subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True)
+
+    logger.info(f"Started {WEBUI_COMPONENT_NAME}.")
+
+
 def main(argv):
     clp_home = get_clp_home()
     default_config_file_path = clp_home / CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH
@@ -557,6 +624,7 @@ def main(argv):
     component_args_parser.add_parser(QUEUE_COMPONENT_NAME)
     component_args_parser.add_parser(RESULTS_CACHE_COMPONENT_NAME)
     component_args_parser.add_parser(SCHEDULER_COMPONENT_NAME)
+    component_args_parser.add_parser(WEBUI_COMPONENT_NAME)
     worker_args_parser = component_args_parser.add_parser(WORKER_COMPONENT_NAME)
     worker_args_parser.add_argument('--num-cpus', type=int, default=0,
                                     help="Number of logical CPU cores to use for compression")
@@ -581,7 +649,7 @@ def main(argv):
 
         # Validate and load necessary credentials
         if component_name in ['', DB_COMPONENT_NAME, SCHEDULER_COMPONENT_NAME,
-                              SEARCH_SCHEDULER_COMPONENT_NAME]:
+                              SEARCH_SCHEDULER_COMPONENT_NAME, WEBUI_COMPONENT_NAME]:
             validate_and_load_db_credentials_file(clp_config, clp_home, True)
         if component_name in ['', QUEUE_COMPONENT_NAME, SCHEDULER_COMPONENT_NAME,
                               WORKER_COMPONENT_NAME, SEARCH_SCHEDULER_COMPONENT_NAME,
@@ -635,6 +703,8 @@ def main(argv):
             start_search_worker(instance_id, clp_config, container_clp_config, num_cpus, mounts)
         if '' == component_name or WORKER_COMPONENT_NAME == component_name:
             start_worker(instance_id, clp_config, container_clp_config, num_cpus, mounts)
+        if '' == component_name or WEBUI_COMPONENT_NAME == component_name:
+            start_webui(instance_id, clp_config, mounts)
     except Exception as ex:
         # Stop CLP
         subprocess.run([str(clp_home / 'sbin' / 'stop-clp.sh')], check=True)

components/clp-package-utils/clp_package_utils/scripts/stop_clp.py

@@ -19,7 +19,8 @@
     SEARCH_SCHEDULER_COMPONENT_NAME,
     SEARCH_WORKER_COMPONENT_NAME,
     SCHEDULER_COMPONENT_NAME,
-    WORKER_COMPONENT_NAME
+    WORKER_COMPONENT_NAME,
+    WEBUI_COMPONENT_NAME,
 )
 
 # Setup logging
@@ -57,6 +58,7 @@ def main(argv):
     component_args_parser.add_parser(RESULTS_CACHE_COMPONENT_NAME)
     component_args_parser.add_parser(SCHEDULER_COMPONENT_NAME)
     component_args_parser.add_parser(WORKER_COMPONENT_NAME)
+    component_args_parser.add_parser(WEBUI_COMPONENT_NAME)
 
     parsed_args = args_parser.parse_args(argv[1:])
 
@@ -89,6 +91,8 @@ def main(argv):
         with open(instance_id_file_path, 'r') as f:
             instance_id = f.readline()
 
+        if '' == component_name or WEBUI_COMPONENT_NAME == component_name:
+            stop_container(f'clp-{WEBUI_COMPONENT_NAME}-{instance_id}')
         if '' == component_name or WORKER_COMPONENT_NAME == component_name:
             stop_container(f'clp-{WORKER_COMPONENT_NAME}-{instance_id}')
         if '' == component_name or SEARCH_WORKER_COMPONENT_NAME == component_name:

components/clp-py-utils/clp_py_utils/clp_config.py

@@ -15,6 +15,7 @@
 SEARCH_SCHEDULER_COMPONENT_NAME = 'search_scheduler'
 SEARCH_WORKER_COMPONENT_NAME = 'search_worker'
 WORKER_COMPONENT_NAME = 'worker'
+WEBUI_COMPONENT_NAME = 'webui'
 
 CLP_DEFAULT_CREDENTIALS_FILE_PATH = pathlib.Path('etc') / 'credentials.yml'
 CLP_METADATA_TABLE_PREFIX = 'clp_'
@@ -100,6 +101,7 @@ def get_clp_connection_params_and_type(self, disable_localhost_socket_connection
             connection_params_and_type['ssl_cert'] = self.ssl_cert
         return connection_params_and_type
 
+
 def _validate_logging_level(cls, field):
     if not is_valid_logging_level(field):
         raise ValueError(
@@ -142,6 +144,12 @@ def validate_host(cls, field):
             raise ValueError(f'{RESULTS_CACHE_COMPONENT_NAME}.host cannot be empty.')
         return field
 
+    @validator('db_name')
+    def validate_db_name(cls, field):
+        if '' == field:
+            raise ValueError(f'{RESULTS_CACHE_COMPONENT_NAME}.db_name cannot be empty.')
+        return field
+
     def get_uri(self):
         return f"mongodb://{self.host}:{self.port}/{self.db_name}"
 
@@ -195,6 +203,32 @@ def dump_to_primitive_dict(self):
         return d
 
 
+class WebUi(BaseModel):
+    host: str = 'localhost'
+    port: int = 4000
+    logging_level: str = 'INFO'
+
+    @validator('host')
+    def validate_host(cls, field):
+        if '' == field:
+            raise ValueError(f'{WEBUI_COMPONENT_NAME}.host cannot be empty.')
+        return field
+
+    @validator('port')
+    def validate_port(cls, field):
+        min_valid_port = 0
+        max_valid_port = 2 ** 16 - 1
+        if min_valid_port > field or max_valid_port < field:
+            raise ValueError(f'{WEBUI_COMPONENT_NAME}.port is not within valid range '
+                             f'{min_valid_port}-{max_valid_port}.')
+        return field
+
+    @validator('logging_level')
+    def validate_logging_level(cls, field):
+        _validate_logging_level(cls, field)
+        return field
+
+
 class CLPConfig(BaseModel):
     execution_container: str = 'ghcr.io/y-scope/clp/clp-execution-x86-ubuntu-focal:main'
 
@@ -206,6 +240,7 @@ class CLPConfig(BaseModel):
     scheduler: Scheduler = Scheduler()
     search_scheduler: SearchScheduler = SearchScheduler()
     search_worker: SearchWorker = SearchWorker()
+    webui: WebUi = WebUi()
     credentials_file_path: pathlib.Path = CLP_DEFAULT_CREDENTIALS_FILE_PATH
 
     archive_output: ArchiveOutput = ArchiveOutput()

components/package-template/src/etc/clp-config.yml

@@ -25,6 +25,11 @@
 #  port: 27017
 #  db_name: "clp-search"
 #
+#webui:
+#  host: "localhost"
+#  port: 4000
+#  logging_level: "INFO"
+#
 #search_scheduler:
 #  jobs_poll_delay: 0.1  # seconds
 #  logging_level: "INFO"

components/webui/.gitignore

@@ -0,0 +1 @@
+node_modules/