Add Web UI to support querying from browsers.

Taskfile.yml

@@ -3,6 +3,9 @@ version: "3"
 vars:
   BUILD_DIR: "{{.TASKFILE_DIR}}/build"
   CORE_COMPONENT_BUILD_DIR: "{{.TASKFILE_DIR}}/build/core"
+  WEBUI_BUILD_DIR: "{{.TASKFILE_DIR}}/build/webui"
+  NODEJS_BUILD_DIR: "{{.TASKFILE_DIR}}/build/nodejs"
+  NODEJS_BIN_DIR: "{{.NODEJS_BUILD_DIR}}/node/bin"
   LINT_VENV_DIR: "{{.TASKFILE_DIR}}/.lint-venv"
   PACKAGE_BUILD_DIR: "{{.TASKFILE_DIR}}/build/clp-package"
   PACKAGE_VENV_DIR: "{{.TASKFILE_DIR}}/build/package-venv"
@@ -29,7 +32,6 @@ tasks:
       - task: "clean-python-component"
         vars:
           COMPONENT: "job-orchestration"
-
   clean-package:
     cmds:
       - "rm -rf '{{.PACKAGE_BUILD_DIR}}'"
@@ -64,6 +66,7 @@ tasks:
       - "compression-job-handler"
       - "job-orchestration"
       - "package-venv"
+      - "webui"
     cmds:
       - task: "clean-package"
       - "mkdir -p '{{.PACKAGE_BUILD_DIR}}'"
@@ -83,7 +86,13 @@ tasks:
         "{{.CORE_COMPONENT_BUILD_DIR}}/clg"
         "{{.CORE_COMPONENT_BUILD_DIR}}/clo"
         "{{.CORE_COMPONENT_BUILD_DIR}}/clp"
+        "{{.BUILD_DIR}}/nodejs/node/bin/node"
         "{{.PACKAGE_BUILD_DIR}}/bin/"
+      - "mkdir -p '{{.PACKAGE_BUILD_DIR}}/var/www/'"
+      - >-
+        rsync -a --delete
+        "{{.WEBUI_BUILD_DIR}}/"
+        "{{.PACKAGE_BUILD_DIR}}/var/www/"
       # This step must be last since we use this file to detect whether the package was built
       # successfully
       - "echo {{.PACKAGE_VERSION}} > '{{.PACKAGE_VERSION_FILE}}'"
@@ -92,6 +101,7 @@ tasks:
       - "{{.CORE_COMPONENT_BUILD_DIR}}/clg"
       - "{{.CORE_COMPONENT_BUILD_DIR}}/clo"
       - "{{.CORE_COMPONENT_BUILD_DIR}}/clp"
+      - "{{.BUILD_DIR}}/webui/built/**/*"
       - "{{.TASKFILE_DIR}}/Taskfile.yml"
       - "components/clp-package-utils/dist/*.whl"
       - "components/clp-py-utils/dist/*.whl"
@@ -102,6 +112,51 @@ tasks:
       - "test -e '{{.PACKAGE_VERSION_FILE}}'"
       - "test {{.TIMESTAMP | unixEpoch}} -lt $(stat --format %Y '{{.PACKAGE_VERSION_FILE}}')"
 
+  webui:
+    deps:
+      - "nodejs"
+    dir: "components/webui"
+    cmds:
+      - "rm -rf '{{.WEBUI_BUILD_DIR}}'"
+      - "mkdir -p {{.WEBUI_BUILD_DIR}}"
+      - "meteor npm install --production"
+      - "meteor build --directory {{.WEBUI_BUILD_DIR}}"
+      - >-
+        rsync -a
+        "{{.WEBUI_BUILD_DIR}}/bundle/"
+        "{{.WEBUI_BUILD_DIR}}/"
+      - "rm -rf '{{.WEBUI_BUILD_DIR}}/bundle/'"
+      - "cp $PWD/settings.json {{.WEBUI_BUILD_DIR}}/"
+      - |-
+        cd {{.WEBUI_BUILD_DIR}}/programs/server
+        PATH={{.NODEJS_BIN_DIR}}:$PATH $(readlink -f {{.NODEJS_BIN_DIR}}/npm) install
+    sources:
+      - "./.meteor/*"
+      - "./client/**/*"
+      - "./imports/**/*"
+      - "./server/**/*"
+      - "./tests/**/*"
+      - "./*"
+
+  nodejs:
+    vars:
+      NODEJS_VERSION: "14.21.3"
+      NODEJS_RELEASE: "v{{.NODEJS_VERSION}}-linux-x64"
+      TAR_FILENAME: "node-{{.NODEJS_RELEASE}}.tar.xz"
+      TAR_FILE_PATH: "{{.NODEJS_BUILD_DIR}}/{{.TAR_FILENAME}}"
+    cmds:
+      - "rm -rf '{{.NODEJS_BUILD_DIR}}/node'"
+      - "mkdir -p {{.NODEJS_BUILD_DIR}}"
+      - >-
+        curl -fsSL
+        https://nodejs.org/dist/v{{.NODEJS_VERSION}}/{{.TAR_FILENAME}}
+        -o {{.TAR_FILE_PATH}}
+      - "tar xf {{.TAR_FILE_PATH}} -C {{.NODEJS_BUILD_DIR}}"
+      - "mv {{.NODEJS_BUILD_DIR}}/node-{{.NODEJS_RELEASE}} {{.NODEJS_BUILD_DIR}}/node"
+      - "rm -f '{{.TAR_FILE_PATH}}'"
+    sources:
+      - "{{.NODEJS_BUILD_DIR}}/node/**/*"
+
   core:
     deps: ["core-submodules"]
     vars:

components/clp-package-utils/clp_package_utils/general.py

@@ -14,13 +14,14 @@
     CLP_DEFAULT_CREDENTIALS_FILE_PATH,
     DB_COMPONENT_NAME,
     QUEUE_COMPONENT_NAME,
-    RESULTS_CACHE_COMPONENT_NAME
+    RESULTS_CACHE_COMPONENT_NAME,
+    WEBUI_COMPONENT_NAME,
 )
 from clp_py_utils.core import (
     get_config_value,
     make_config_path_absolute,
     read_yaml_config_file,
-    validate_path_could_be_dir
+    validate_path_could_be_dir,
 )
 
 # CONSTANTS
@@ -41,12 +42,18 @@ class DockerMountType(enum.IntEnum):
 class DockerMount:
     def __init__(self, type: DockerMountType, src: pathlib.Path, dst: pathlib.Path, is_read_only: bool = False):
         self.__type = type
-        self.__src = src
-        self.__dst = dst
+        self._src = src
+        self._dst = dst
         self.__is_read_only = is_read_only
 
+    def get_src(self):
+        return self._src
+
+    def get_dst(self):
+        return self._dst
+
     def __str__(self):
-        mount_str = f"type={DOCKER_MOUNT_TYPE_STRINGS[self.__type]},src={self.__src},dst={self.__dst}"
+        mount_str = f"type={DOCKER_MOUNT_TYPE_STRINGS[self.__type]},src={self._src},dst={self._dst}"
         if self.__is_read_only:
             mount_str += ",readonly"
         return mount_str
@@ -60,6 +67,7 @@ def __init__(self, clp_home: pathlib.Path, docker_clp_home: pathlib.Path):
         self.logs_dir: typing.Optional[DockerMount] = None
         self.archives_output_dir: typing.Optional[DockerMount] = None
 
+
 def get_clp_home():
     # Determine CLP_HOME from an environment variable or this script's path
     clp_home = None
@@ -78,6 +86,7 @@ def get_clp_home():
 
     return clp_home.resolve()
 
+
 def check_dependencies():
     try:
         subprocess.run("command -v docker", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True)
@@ -277,3 +286,10 @@ def validate_results_cache_config(clp_config: CLPConfig, data_dir: pathlib.Path,
 def validate_worker_config(clp_config: CLPConfig):
     clp_config.validate_input_logs_dir()
     clp_config.validate_archive_output_dir()
+
+
+def validate_webui_config(clp_config: CLPConfig):
+    component_name = WEBUI_COMPONENT_NAME
+
+    validate_port(f"{component_name}.port", clp_config.webui.host,
+                  clp_config.webui.port)

components/clp-package-utils/clp_package_utils/scripts/start_clp.py

@@ -1,4 +1,5 @@
 import argparse
+import json
 import logging
 import multiprocessing
 import os
@@ -29,6 +30,7 @@
     validate_db_config,
     validate_queue_config,
     validate_results_cache_config,
+    validate_webui_config,
     validate_worker_config
 )
 from clp_py_utils.clp_config import (
@@ -40,6 +42,7 @@
     SEARCH_SCHEDULER_COMPONENT_NAME,
     SEARCH_WORKER_COMPONENT_NAME,
     WORKER_COMPONENT_NAME,
+    WEBUI_COMPONENT_NAME,
 )
 from job_orchestration.scheduler.constants import QueueName
 
@@ -544,6 +547,58 @@ def start_worker(instance_id: str, clp_config: CLPConfig, container_clp_config:
     logger.info(f"Started {WORKER_COMPONENT_NAME}.")
 
 
+def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts):
+    logger.info(f"Starting {WEBUI_COMPONENT_NAME}...")
+
+    container_name = f'clp-{WEBUI_COMPONENT_NAME}-{instance_id}'
+    if container_exists(container_name):
+        logger.info(f"{WEBUI_COMPONENT_NAME} already running.")
+        return
+
+    validate_webui_config(clp_config)
+
+    settings_json_path = str(mounts.clp_home.get_src() / 'var' / 'www' / 'settings.json')
+    with open(settings_json_path, 'r') as settings_json_file:
+        settings_json_content = settings_json_file.read()
+        meteor_settings = json.loads(settings_json_content)
+
+    # Start container
+    container_cmd = [
+        'docker', 'run',
+        '-d',
+        '--network', 'host',
+        '--rm',
+        '--name', container_name,
+        '-e', f'MONGO_URL={clp_config.results_cache.get_uri()}',
+        '-e', f'PORT={clp_config.webui.port}',
+        '-e', f'ROOT_URL=http://{clp_config.webui.host}',
+        '-e', f'METEOR_SETTINGS={json.dumps(meteor_settings)}',
+        '-e', f'CLP_DB_HOST={clp_config.database.host}',
+        '-e', f'CLP_DB_PORT={clp_config.database.port}',
+        '-e', f'CLP_DB_NAME={clp_config.database.name}',
+        '-e', f'CLP_DB_USER={clp_config.database.username}',
+        '-e', f'CLP_DB_PASS={clp_config.database.password}',
+        '-u', f'{os.getuid()}:{os.getgid()}',
+    ]
+    necessary_mounts = [
+        mounts.clp_home,
+    ]
+    for mount in necessary_mounts:
+        if mount:
+            container_cmd.append('--mount')
+            container_cmd.append(str(mount))
+    container_cmd.append(clp_config.execution_container)
+
+    node_cmd = [
+        str(CONTAINER_CLP_HOME / 'bin' / 'node'),
+        str(CONTAINER_CLP_HOME / 'var' / 'www' / 'main.js')
+    ]
+    cmd = container_cmd + node_cmd
+    subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True)
+
+    logger.info(f"Started {WEBUI_COMPONENT_NAME}.")
+
+
 def main(argv):
     clp_home = get_clp_home()
     default_config_file_path = clp_home / CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH
@@ -557,6 +612,7 @@ def main(argv):
     component_args_parser.add_parser(QUEUE_COMPONENT_NAME)
     component_args_parser.add_parser(RESULTS_CACHE_COMPONENT_NAME)
     component_args_parser.add_parser(SCHEDULER_COMPONENT_NAME)
+    component_args_parser.add_parser(WEBUI_COMPONENT_NAME)
     worker_args_parser = component_args_parser.add_parser(WORKER_COMPONENT_NAME)
     worker_args_parser.add_argument('--num-cpus', type=int, default=0,
                                     help="Number of logical CPU cores to use for compression")
@@ -581,7 +637,7 @@ def main(argv):
 
         # Validate and load necessary credentials
         if component_name in ['', DB_COMPONENT_NAME, SCHEDULER_COMPONENT_NAME,
-                              SEARCH_SCHEDULER_COMPONENT_NAME]:
+                              SEARCH_SCHEDULER_COMPONENT_NAME, WEBUI_COMPONENT_NAME]:
             validate_and_load_db_credentials_file(clp_config, clp_home, True)
         if component_name in ['', QUEUE_COMPONENT_NAME, SCHEDULER_COMPONENT_NAME,
                               WORKER_COMPONENT_NAME, SEARCH_SCHEDULER_COMPONENT_NAME,
@@ -635,6 +691,8 @@ def main(argv):
             start_search_worker(instance_id, clp_config, container_clp_config, num_cpus, mounts)
         if '' == component_name or WORKER_COMPONENT_NAME == component_name:
             start_worker(instance_id, clp_config, container_clp_config, num_cpus, mounts)
+        if '' == component_name or WEBUI_COMPONENT_NAME == component_name:
+            start_webui(instance_id, clp_config, mounts)
     except Exception as ex:
         # Stop CLP
         subprocess.run([str(clp_home / 'sbin' / 'stop-clp.sh')], check=True)

components/clp-package-utils/clp_package_utils/scripts/stop_clp.py

@@ -19,7 +19,8 @@
     SEARCH_SCHEDULER_COMPONENT_NAME,
     SEARCH_WORKER_COMPONENT_NAME,
     SCHEDULER_COMPONENT_NAME,
-    WORKER_COMPONENT_NAME
+    WORKER_COMPONENT_NAME,
+    WEBUI_COMPONENT_NAME,
 )
 
 # Setup logging
@@ -57,6 +58,7 @@ def main(argv):
     component_args_parser.add_parser(RESULTS_CACHE_COMPONENT_NAME)
     component_args_parser.add_parser(SCHEDULER_COMPONENT_NAME)
     component_args_parser.add_parser(WORKER_COMPONENT_NAME)
+    component_args_parser.add_parser(WEBUI_COMPONENT_NAME)
 
     parsed_args = args_parser.parse_args(argv[1:])
 
@@ -89,6 +91,8 @@ def main(argv):
         with open(instance_id_file_path, 'r') as f:
             instance_id = f.readline()
 
+        if '' == component_name or WEBUI_COMPONENT_NAME == component_name:
+            stop_container(f'clp-{WEBUI_COMPONENT_NAME}-{instance_id}')
         if '' == component_name or WORKER_COMPONENT_NAME == component_name:
             stop_container(f'clp-{WORKER_COMPONENT_NAME}-{instance_id}')
         if '' == component_name or SEARCH_WORKER_COMPONENT_NAME == component_name:

components/clp-py-utils/clp_py_utils/clp_config.py

@@ -15,6 +15,7 @@
 SEARCH_SCHEDULER_COMPONENT_NAME = 'search_scheduler'
 SEARCH_WORKER_COMPONENT_NAME = 'search_worker'
 WORKER_COMPONENT_NAME = 'worker'
+WEBUI_COMPONENT_NAME = 'webui'
 
 CLP_DEFAULT_CREDENTIALS_FILE_PATH = pathlib.Path('etc') / 'credentials.yml'
 CLP_METADATA_TABLE_PREFIX = 'clp_'
@@ -100,6 +101,7 @@ def get_clp_connection_params_and_type(self, disable_localhost_socket_connection
             connection_params_and_type['ssl_cert'] = self.ssl_cert
         return connection_params_and_type
 
+
 def _validate_logging_level(cls, field):
     if not is_valid_logging_level(field):
         raise ValueError(
@@ -142,6 +144,12 @@ def validate_host(cls, field):
             raise ValueError(f'{RESULTS_CACHE_COMPONENT_NAME}.host cannot be empty.')
         return field
 
+    @validator('db_name')
+    def validate_db_name(cls, field):
+        if '' == field:
+            raise ValueError(f'{RESULTS_CACHE_COMPONENT_NAME}.db_name cannot be empty.')
+        return field
+
     def get_uri(self):
         return f"mongodb://{self.host}:{self.port}/{self.db_name}"
 
@@ -195,6 +203,11 @@ def dump_to_primitive_dict(self):
         return d
 
 
+class WebUi(BaseModel):
+    host: str = 'localhost'
+    port: int = 4000
+
+
 class CLPConfig(BaseModel):
     execution_container: str = 'ghcr.io/y-scope/clp/clp-execution-x86-ubuntu-focal:main'
 
@@ -206,6 +219,7 @@ class CLPConfig(BaseModel):
     scheduler: Scheduler = Scheduler()
     search_scheduler: SearchScheduler = SearchScheduler()
     search_worker: SearchWorker = SearchWorker()
+    webui: WebUi = WebUi()
     credentials_file_path: pathlib.Path = CLP_DEFAULT_CREDENTIALS_FILE_PATH
 
     archive_output: ArchiveOutput = ArchiveOutput()

components/package-template/src/etc/clp-config.yml

@@ -25,6 +25,10 @@
 #  port: 27017
 #  db_name: "clp-search"
 #
+#webui:
+#  host: "localhost"
+#  port: 4000
+#
 #search_scheduler:
 #  jobs_poll_delay: 0.1  # seconds
 #  logging_level: "INFO"

components/webui/.gitignore

@@ -0,0 +1 @@
+node_modules/

components/webui/.meteor/.finished-upgraders

@@ -0,0 +1,19 @@
+# This file contains information which helps Meteor properly upgrade your
+# app when you run 'meteor update'. You should check it into version control
+# with your project.
+
+notices-for-0.9.0
+notices-for-0.9.1
+0.9.4-platform-file
+notices-for-facebook-graph-api-2
+1.2.0-standard-minifiers-package
+1.2.0-meteor-platform-split
+1.2.0-cordova-changes
+1.2.0-breaking-changes
+1.3.0-split-minifiers-package
+1.4.0-remove-old-dev-bundle-link
+1.4.1-add-shell-server-package
+1.4.3-split-account-service-packages
+1.5-add-dynamic-import-package
+1.7-split-underscore-from-meteor-base
+1.8.3-split-jquery-from-blaze

components/webui/.meteor/.gitignore

@@ -0,0 +1 @@
+local

components/webui/.meteor/.id

@@ -0,0 +1,7 @@
+# This file contains a token that is unique to your project.
+# Check it into your repository along with the rest of this directory.
+# It can be used for purposes such as:
+#   - ensuring you don't accidentally deploy one app on top of another
+#   - providing package authors with aggregated statistics
+
+w2i4drbtdntc.kdbz5fehx6g