From a8fee2415d6bcde2c472961214e67465388ab687 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 26 Jan 2024 05:09:59 -0500 Subject: [PATCH 01/60] Add Web UI to support querying from browser clients. --- Taskfile.yml | 57 +- .../clp_package_utils/general.py | 26 +- .../clp_package_utils/scripts/start_clp.py | 60 +- .../clp_package_utils/scripts/stop_clp.py | 6 +- .../clp-py-utils/clp_py_utils/clp_config.py | 14 + .../package-template/src/etc/clp-config.yml | 4 + components/webui/.gitignore | 1 + components/webui/.meteor/.finished-upgraders | 19 + components/webui/.meteor/.gitignore | 1 + components/webui/.meteor/.id | 7 + components/webui/.meteor/packages | 23 + components/webui/.meteor/platforms | 2 + components/webui/.meteor/release | 1 + components/webui/.meteor/versions | 80 + components/webui/README.md | 41 + components/webui/client/main.css | 3 + components/webui/client/main.html | 7 + components/webui/client/main.jsx | 18 + .../webui/imports/api/search/collections.js | 28 + .../webui/imports/api/search/constants.js | 33 + .../imports/api/search/server/methods.js | 152 ++ .../imports/api/search/server/publications.js | 31 + components/webui/imports/api/search/sql.js | 31 + .../webui/imports/api/user/client/methods.js | 65 + .../webui/imports/api/user/server/methods.js | 8 + components/webui/imports/ui/App.jsx | 69 + components/webui/imports/ui/App.scss | 15 + .../imports/ui/SearchView/SearchControls.js | 226 +++ .../imports/ui/SearchView/SearchResults.jsx | 42 + .../ui/SearchView/SearchResultsHeader.jsx | 74 + .../ui/SearchView/SearchResultsTable.jsx | 116 ++ .../imports/ui/SearchView/SearchView.jsx | 234 +++ .../imports/ui/SearchView/SearchView.scss | 94 ++ .../webui/imports/ui/SearchView/datetime.js | 211 +++ .../webui/imports/ui/Sidebar/Sidebar.jsx | 48 + .../webui/imports/ui/Sidebar/Sidebar.scss | 153 ++ .../imports/ui/bootstrap-customized.scss | 5 + .../ui/constants/LOCAL_STORAGE_KEYS.js | 7 + components/webui/package-lock.json | 1429 +++++++++++++++++ components/webui/package.json | 39 + components/webui/server/main.js | 18 + components/webui/settings.json | 6 + components/webui/tests/main.js | 20 + 43 files changed, 3516 insertions(+), 8 deletions(-) create mode 100644 components/webui/.gitignore create mode 100644 components/webui/.meteor/.finished-upgraders create mode 100644 components/webui/.meteor/.gitignore create mode 100644 components/webui/.meteor/.id create mode 100644 components/webui/.meteor/packages create mode 100644 components/webui/.meteor/platforms create mode 100644 components/webui/.meteor/release create mode 100644 components/webui/.meteor/versions create mode 100644 components/webui/README.md create mode 100644 components/webui/client/main.css create mode 100644 components/webui/client/main.html create mode 100644 components/webui/client/main.jsx create mode 100644 components/webui/imports/api/search/collections.js create mode 100644 components/webui/imports/api/search/constants.js create mode 100644 components/webui/imports/api/search/server/methods.js create mode 100644 components/webui/imports/api/search/server/publications.js create mode 100644 components/webui/imports/api/search/sql.js create mode 100644 components/webui/imports/api/user/client/methods.js create mode 100644 components/webui/imports/api/user/server/methods.js create mode 100644 components/webui/imports/ui/App.jsx create mode 100644 components/webui/imports/ui/App.scss create mode 100644 components/webui/imports/ui/SearchView/SearchControls.js create mode 100644 components/webui/imports/ui/SearchView/SearchResults.jsx create mode 100644 components/webui/imports/ui/SearchView/SearchResultsHeader.jsx create mode 100644 components/webui/imports/ui/SearchView/SearchResultsTable.jsx create mode 100644 components/webui/imports/ui/SearchView/SearchView.jsx create mode 100644 components/webui/imports/ui/SearchView/SearchView.scss create mode 100644 components/webui/imports/ui/SearchView/datetime.js create mode 100644 components/webui/imports/ui/Sidebar/Sidebar.jsx create mode 100644 components/webui/imports/ui/Sidebar/Sidebar.scss create mode 100644 components/webui/imports/ui/bootstrap-customized.scss create mode 100644 components/webui/imports/ui/constants/LOCAL_STORAGE_KEYS.js create mode 100644 components/webui/package-lock.json create mode 100644 components/webui/package.json create mode 100644 components/webui/server/main.js create mode 100644 components/webui/settings.json create mode 100644 components/webui/tests/main.js diff --git a/Taskfile.yml b/Taskfile.yml index 0e2855f5a..ede927919 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -3,6 +3,9 @@ version: "3" vars: BUILD_DIR: "{{.TASKFILE_DIR}}/build" CORE_COMPONENT_BUILD_DIR: "{{.TASKFILE_DIR}}/build/core" + WEBUI_BUILD_DIR: "{{.TASKFILE_DIR}}/build/webui" + NODEJS_BUILD_DIR: "{{.TASKFILE_DIR}}/build/nodejs" + NODEJS_BIN_DIR: "{{.NODEJS_BUILD_DIR}}/node/bin" LINT_VENV_DIR: "{{.TASKFILE_DIR}}/.lint-venv" PACKAGE_BUILD_DIR: "{{.TASKFILE_DIR}}/build/clp-package" PACKAGE_VENV_DIR: "{{.TASKFILE_DIR}}/build/package-venv" @@ -29,7 +32,6 @@ tasks: - task: "clean-python-component" vars: COMPONENT: "job-orchestration" - clean-package: cmds: - "rm -rf '{{.PACKAGE_BUILD_DIR}}'" @@ -64,6 +66,7 @@ tasks: - "compression-job-handler" - "job-orchestration" - "package-venv" + - "webui" cmds: - task: "clean-package" - "mkdir -p '{{.PACKAGE_BUILD_DIR}}'" @@ -83,7 +86,13 @@ tasks: "{{.CORE_COMPONENT_BUILD_DIR}}/clg" "{{.CORE_COMPONENT_BUILD_DIR}}/clo" "{{.CORE_COMPONENT_BUILD_DIR}}/clp" + "{{.BUILD_DIR}}/nodejs/node/bin/node" "{{.PACKAGE_BUILD_DIR}}/bin/" + - "mkdir -p '{{.PACKAGE_BUILD_DIR}}/var/www/'" + - >- + rsync -a --delete + "{{.WEBUI_BUILD_DIR}}/" + "{{.PACKAGE_BUILD_DIR}}/var/www/" # This step must be last since we use this file to detect whether the package was built # successfully - "echo {{.PACKAGE_VERSION}} > '{{.PACKAGE_VERSION_FILE}}'" @@ -92,6 +101,7 @@ tasks: - "{{.CORE_COMPONENT_BUILD_DIR}}/clg" - "{{.CORE_COMPONENT_BUILD_DIR}}/clo" - "{{.CORE_COMPONENT_BUILD_DIR}}/clp" + - "{{.BUILD_DIR}}/webui/built/**/*" - "{{.TASKFILE_DIR}}/Taskfile.yml" - "components/clp-package-utils/dist/*.whl" - "components/clp-py-utils/dist/*.whl" @@ -102,6 +112,51 @@ tasks: - "test -e '{{.PACKAGE_VERSION_FILE}}'" - "test {{.TIMESTAMP | unixEpoch}} -lt $(stat --format %Y '{{.PACKAGE_VERSION_FILE}}')" + webui: + deps: + - "nodejs" + dir: "components/webui" + cmds: + - "rm -rf '{{.WEBUI_BUILD_DIR}}'" + - "mkdir -p {{.WEBUI_BUILD_DIR}}" + - "meteor npm install --production" + - "meteor build --directory {{.WEBUI_BUILD_DIR}}" + - >- + rsync -a + "{{.WEBUI_BUILD_DIR}}/bundle/" + "{{.WEBUI_BUILD_DIR}}/" + - "rm -rf '{{.WEBUI_BUILD_DIR}}/bundle/'" + - "cp $PWD/settings.json {{.WEBUI_BUILD_DIR}}/" + - |- + cd {{.WEBUI_BUILD_DIR}}/programs/server + PATH={{.NODEJS_BIN_DIR}}:$PATH $(readlink -f {{.NODEJS_BIN_DIR}}/npm) install + sources: + - "./.meteor/*" + - "./client/**/*" + - "./imports/**/*" + - "./server/**/*" + - "./tests/**/*" + - "./*" + + nodejs: + vars: + NODEJS_VERSION: "14.21.3" + NODEJS_RELEASE: "v{{.NODEJS_VERSION}}-linux-x64" + TAR_FILENAME: "node-{{.NODEJS_RELEASE}}.tar.xz" + TAR_FILE_PATH: "{{.NODEJS_BUILD_DIR}}/{{.TAR_FILENAME}}" + cmds: + - "rm -rf '{{.NODEJS_BUILD_DIR}}/node'" + - "mkdir -p {{.NODEJS_BUILD_DIR}}" + - >- + curl -fsSL + https://nodejs.org/dist/v{{.NODEJS_VERSION}}/{{.TAR_FILENAME}} + -o {{.TAR_FILE_PATH}} + - "tar xf {{.TAR_FILE_PATH}} -C {{.NODEJS_BUILD_DIR}}" + - "mv {{.NODEJS_BUILD_DIR}}/node-{{.NODEJS_RELEASE}} {{.NODEJS_BUILD_DIR}}/node" + - "rm -f '{{.TAR_FILE_PATH}}'" + sources: + - "{{.NODEJS_BUILD_DIR}}/node/**/*" + core: deps: ["core-submodules"] vars: diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index cbb95b765..77331db82 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -14,13 +14,14 @@ CLP_DEFAULT_CREDENTIALS_FILE_PATH, DB_COMPONENT_NAME, QUEUE_COMPONENT_NAME, - RESULTS_CACHE_COMPONENT_NAME + RESULTS_CACHE_COMPONENT_NAME, + WEBUI_COMPONENT_NAME, ) from clp_py_utils.core import ( get_config_value, make_config_path_absolute, read_yaml_config_file, - validate_path_could_be_dir + validate_path_could_be_dir, ) # CONSTANTS @@ -41,12 +42,18 @@ class DockerMountType(enum.IntEnum): class DockerMount: def __init__(self, type: DockerMountType, src: pathlib.Path, dst: pathlib.Path, is_read_only: bool = False): self.__type = type - self.__src = src - self.__dst = dst + self._src = src + self._dst = dst self.__is_read_only = is_read_only + def get_src(self): + return self._src + + def get_dst(self): + return self._dst + def __str__(self): - mount_str = f"type={DOCKER_MOUNT_TYPE_STRINGS[self.__type]},src={self.__src},dst={self.__dst}" + mount_str = f"type={DOCKER_MOUNT_TYPE_STRINGS[self.__type]},src={self._src},dst={self._dst}" if self.__is_read_only: mount_str += ",readonly" return mount_str @@ -60,6 +67,7 @@ def __init__(self, clp_home: pathlib.Path, docker_clp_home: pathlib.Path): self.logs_dir: typing.Optional[DockerMount] = None self.archives_output_dir: typing.Optional[DockerMount] = None + def get_clp_home(): # Determine CLP_HOME from an environment variable or this script's path clp_home = None @@ -78,6 +86,7 @@ def get_clp_home(): return clp_home.resolve() + def check_dependencies(): try: subprocess.run("command -v docker", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True) @@ -277,3 +286,10 @@ def validate_results_cache_config(clp_config: CLPConfig, data_dir: pathlib.Path, def validate_worker_config(clp_config: CLPConfig): clp_config.validate_input_logs_dir() clp_config.validate_archive_output_dir() + + +def validate_webui_config(clp_config: CLPConfig): + component_name = WEBUI_COMPONENT_NAME + + validate_port(f"{component_name}.port", clp_config.webui.host, + clp_config.webui.port) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index e2c41b06d..6ea26eb1a 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -1,4 +1,5 @@ import argparse +import json import logging import multiprocessing import os @@ -29,6 +30,7 @@ validate_db_config, validate_queue_config, validate_results_cache_config, + validate_webui_config, validate_worker_config ) from clp_py_utils.clp_config import ( @@ -40,6 +42,7 @@ SEARCH_SCHEDULER_COMPONENT_NAME, SEARCH_WORKER_COMPONENT_NAME, WORKER_COMPONENT_NAME, + WEBUI_COMPONENT_NAME, ) from job_orchestration.scheduler.constants import QueueName @@ -544,6 +547,58 @@ def start_worker(instance_id: str, clp_config: CLPConfig, container_clp_config: logger.info(f"Started {WORKER_COMPONENT_NAME}.") +def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts): + logger.info(f"Starting {WEBUI_COMPONENT_NAME}...") + + container_name = f'clp-{WEBUI_COMPONENT_NAME}-{instance_id}' + if container_exists(container_name): + logger.info(f"{WEBUI_COMPONENT_NAME} already running.") + return + + validate_webui_config(clp_config) + + settings_json_path = str(mounts.clp_home.get_src() / 'var' / 'www' / 'settings.json') + with open(settings_json_path, 'r') as settings_json_file: + settings_json_content = settings_json_file.read() + meteor_settings = json.loads(settings_json_content) + + # Start container + container_cmd = [ + 'docker', 'run', + '-d', + '--network', 'host', + '--rm', + '--name', container_name, + '-e', f'MONGO_URL={clp_config.results_cache.get_uri()}', + '-e', f'PORT={clp_config.webui.port}', + '-e', f'ROOT_URL=http://{clp_config.webui.host}', + '-e', f'METEOR_SETTINGS={json.dumps(meteor_settings)}', + '-e', f'CLP_DB_HOST={clp_config.database.host}', + '-e', f'CLP_DB_PORT={clp_config.database.port}', + '-e', f'CLP_DB_NAME={clp_config.database.name}', + '-e', f'CLP_DB_USER={clp_config.database.username}', + '-e', f'CLP_DB_PASS={clp_config.database.password}', + '-u', f'{os.getuid()}:{os.getgid()}', + ] + necessary_mounts = [ + mounts.clp_home, + ] + for mount in necessary_mounts: + if mount: + container_cmd.append('--mount') + container_cmd.append(str(mount)) + container_cmd.append(clp_config.execution_container) + + node_cmd = [ + str(CONTAINER_CLP_HOME / 'bin' / 'node'), + str(CONTAINER_CLP_HOME / 'var' / 'www' / 'main.js') + ] + cmd = container_cmd + node_cmd + subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True) + + logger.info(f"Started {WEBUI_COMPONENT_NAME}.") + + def main(argv): clp_home = get_clp_home() default_config_file_path = clp_home / CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH @@ -557,6 +612,7 @@ def main(argv): component_args_parser.add_parser(QUEUE_COMPONENT_NAME) component_args_parser.add_parser(RESULTS_CACHE_COMPONENT_NAME) component_args_parser.add_parser(SCHEDULER_COMPONENT_NAME) + component_args_parser.add_parser(WEBUI_COMPONENT_NAME) worker_args_parser = component_args_parser.add_parser(WORKER_COMPONENT_NAME) worker_args_parser.add_argument('--num-cpus', type=int, default=0, help="Number of logical CPU cores to use for compression") @@ -581,7 +637,7 @@ def main(argv): # Validate and load necessary credentials if component_name in ['', DB_COMPONENT_NAME, SCHEDULER_COMPONENT_NAME, - SEARCH_SCHEDULER_COMPONENT_NAME]: + SEARCH_SCHEDULER_COMPONENT_NAME, WEBUI_COMPONENT_NAME]: validate_and_load_db_credentials_file(clp_config, clp_home, True) if component_name in ['', QUEUE_COMPONENT_NAME, SCHEDULER_COMPONENT_NAME, WORKER_COMPONENT_NAME, SEARCH_SCHEDULER_COMPONENT_NAME, @@ -635,6 +691,8 @@ def main(argv): start_search_worker(instance_id, clp_config, container_clp_config, num_cpus, mounts) if '' == component_name or WORKER_COMPONENT_NAME == component_name: start_worker(instance_id, clp_config, container_clp_config, num_cpus, mounts) + if '' == component_name or WEBUI_COMPONENT_NAME == component_name: + start_webui(instance_id, clp_config, mounts) except Exception as ex: # Stop CLP subprocess.run([str(clp_home / 'sbin' / 'stop-clp.sh')], check=True) diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index 44186bc0e..a22ab121f 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -19,7 +19,8 @@ SEARCH_SCHEDULER_COMPONENT_NAME, SEARCH_WORKER_COMPONENT_NAME, SCHEDULER_COMPONENT_NAME, - WORKER_COMPONENT_NAME + WORKER_COMPONENT_NAME, + WEBUI_COMPONENT_NAME, ) # Setup logging @@ -57,6 +58,7 @@ def main(argv): component_args_parser.add_parser(RESULTS_CACHE_COMPONENT_NAME) component_args_parser.add_parser(SCHEDULER_COMPONENT_NAME) component_args_parser.add_parser(WORKER_COMPONENT_NAME) + component_args_parser.add_parser(WEBUI_COMPONENT_NAME) parsed_args = args_parser.parse_args(argv[1:]) @@ -89,6 +91,8 @@ def main(argv): with open(instance_id_file_path, 'r') as f: instance_id = f.readline() + if '' == component_name or WEBUI_COMPONENT_NAME == component_name: + stop_container(f'clp-{WEBUI_COMPONENT_NAME}-{instance_id}') if '' == component_name or WORKER_COMPONENT_NAME == component_name: stop_container(f'clp-{WORKER_COMPONENT_NAME}-{instance_id}') if '' == component_name or SEARCH_WORKER_COMPONENT_NAME == component_name: diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 7ccb56da7..bb930e8e2 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -15,6 +15,7 @@ SEARCH_SCHEDULER_COMPONENT_NAME = 'search_scheduler' SEARCH_WORKER_COMPONENT_NAME = 'search_worker' WORKER_COMPONENT_NAME = 'worker' +WEBUI_COMPONENT_NAME = 'webui' CLP_DEFAULT_CREDENTIALS_FILE_PATH = pathlib.Path('etc') / 'credentials.yml' CLP_METADATA_TABLE_PREFIX = 'clp_' @@ -100,6 +101,7 @@ def get_clp_connection_params_and_type(self, disable_localhost_socket_connection connection_params_and_type['ssl_cert'] = self.ssl_cert return connection_params_and_type + def _validate_logging_level(cls, field): if not is_valid_logging_level(field): raise ValueError( @@ -142,6 +144,12 @@ def validate_host(cls, field): raise ValueError(f'{RESULTS_CACHE_COMPONENT_NAME}.host cannot be empty.') return field + @validator('db_name') + def validate_db_name(cls, field): + if '' == field: + raise ValueError(f'{RESULTS_CACHE_COMPONENT_NAME}.db_name cannot be empty.') + return field + def get_uri(self): return f"mongodb://{self.host}:{self.port}/{self.db_name}" @@ -195,6 +203,11 @@ def dump_to_primitive_dict(self): return d +class WebUi(BaseModel): + host: str = 'localhost' + port: int = 4000 + + class CLPConfig(BaseModel): execution_container: str = 'ghcr.io/y-scope/clp/clp-execution-x86-ubuntu-focal:main' @@ -206,6 +219,7 @@ class CLPConfig(BaseModel): scheduler: Scheduler = Scheduler() search_scheduler: SearchScheduler = SearchScheduler() search_worker: SearchWorker = SearchWorker() + webui: WebUi = WebUi() credentials_file_path: pathlib.Path = CLP_DEFAULT_CREDENTIALS_FILE_PATH archive_output: ArchiveOutput = ArchiveOutput() diff --git a/components/package-template/src/etc/clp-config.yml b/components/package-template/src/etc/clp-config.yml index 291752f53..68803588a 100644 --- a/components/package-template/src/etc/clp-config.yml +++ b/components/package-template/src/etc/clp-config.yml @@ -25,6 +25,10 @@ # port: 27017 # db_name: "clp-search" # +#webui: +# host: "localhost" +# port: 4000 +# #search_scheduler: # jobs_poll_delay: 0.1 # seconds # logging_level: "INFO" diff --git a/components/webui/.gitignore b/components/webui/.gitignore new file mode 100644 index 000000000..c2658d7d1 --- /dev/null +++ b/components/webui/.gitignore @@ -0,0 +1 @@ +node_modules/ diff --git a/components/webui/.meteor/.finished-upgraders b/components/webui/.meteor/.finished-upgraders new file mode 100644 index 000000000..c07b6ff75 --- /dev/null +++ b/components/webui/.meteor/.finished-upgraders @@ -0,0 +1,19 @@ +# This file contains information which helps Meteor properly upgrade your +# app when you run 'meteor update'. You should check it into version control +# with your project. + +notices-for-0.9.0 +notices-for-0.9.1 +0.9.4-platform-file +notices-for-facebook-graph-api-2 +1.2.0-standard-minifiers-package +1.2.0-meteor-platform-split +1.2.0-cordova-changes +1.2.0-breaking-changes +1.3.0-split-minifiers-package +1.4.0-remove-old-dev-bundle-link +1.4.1-add-shell-server-package +1.4.3-split-account-service-packages +1.5-add-dynamic-import-package +1.7-split-underscore-from-meteor-base +1.8.3-split-jquery-from-blaze diff --git a/components/webui/.meteor/.gitignore b/components/webui/.meteor/.gitignore new file mode 100644 index 000000000..408303742 --- /dev/null +++ b/components/webui/.meteor/.gitignore @@ -0,0 +1 @@ +local diff --git a/components/webui/.meteor/.id b/components/webui/.meteor/.id new file mode 100644 index 000000000..0428be3da --- /dev/null +++ b/components/webui/.meteor/.id @@ -0,0 +1,7 @@ +# This file contains a token that is unique to your project. +# Check it into your repository along with the rest of this directory. +# It can be used for purposes such as: +# - ensuring you don't accidentally deploy one app on top of another +# - providing package authors with aggregated statistics + +w2i4drbtdntc.kdbz5fehx6g diff --git a/components/webui/.meteor/packages b/components/webui/.meteor/packages new file mode 100644 index 000000000..78e004d54 --- /dev/null +++ b/components/webui/.meteor/packages @@ -0,0 +1,23 @@ +# Meteor packages used by this project, one per line. +# Check this file (and the other files in this directory) into your repository. +# +# 'meteor add' and 'meteor remove' will edit this file for you, +# but you can also edit it by hand. + +meteor-base@1.5.1 # Packages every Meteor app needs to have +mobile-experience@1.1.1 # Packages for a great mobile UX +mongo@1.16.8 # The database Meteor supports right now +reactive-var@1.0.12 # Reactive variable for tracker + +standard-minifier-css@1.9.2 # CSS minifier run for production mode +standard-minifier-js@2.8.1 # JS minifier run for production mode +es5-shim@4.8.0 # ECMAScript 5 compatibility for older browsers +ecmascript@0.16.8 # Enable ECMAScript2015+ syntax in app code +typescript@4.9.5 # Enable TypeScript syntax in .ts and .tsx modules +shell-server@0.5.0 # Server-side component of the `meteor shell` command +hot-module-replacement@0.5.3 # Update client in development without reloading the page + +static-html@1.3.2 # Define static page content in .html files +react-meteor-data # React higher-order component for reactively tracking Meteor data +fourseven:scss +accounts-password@2.4.0 # user authentication via password diff --git a/components/webui/.meteor/platforms b/components/webui/.meteor/platforms new file mode 100644 index 000000000..efeba1b50 --- /dev/null +++ b/components/webui/.meteor/platforms @@ -0,0 +1,2 @@ +server +browser diff --git a/components/webui/.meteor/release b/components/webui/.meteor/release new file mode 100644 index 000000000..c500c39d6 --- /dev/null +++ b/components/webui/.meteor/release @@ -0,0 +1 @@ +METEOR@2.14 diff --git a/components/webui/.meteor/versions b/components/webui/.meteor/versions new file mode 100644 index 000000000..b65159172 --- /dev/null +++ b/components/webui/.meteor/versions @@ -0,0 +1,80 @@ +accounts-base@2.2.9 +accounts-password@2.4.0 +allow-deny@1.1.1 +autoupdate@1.8.0 +babel-compiler@7.10.5 +babel-runtime@1.5.1 +base64@1.0.12 +binary-heap@1.0.11 +blaze-tools@1.1.3 +boilerplate-generator@1.7.2 +caching-compiler@1.2.2 +caching-html-compiler@1.2.1 +callback-hook@1.5.1 +check@1.3.2 +ddp@1.4.1 +ddp-client@2.6.1 +ddp-common@1.4.0 +ddp-rate-limiter@1.2.1 +ddp-server@2.7.0 +diff-sequence@1.1.2 +dynamic-import@0.7.3 +ecmascript@0.16.8 +ecmascript-runtime@0.8.1 +ecmascript-runtime-client@0.12.1 +ecmascript-runtime-server@0.11.0 +ejson@1.1.3 +email@2.2.5 +es5-shim@4.8.0 +fetch@0.1.4 +fourseven:scss@4.16.0 +geojson-utils@1.0.11 +hot-code-push@1.0.4 +hot-module-replacement@0.5.3 +html-tools@1.1.3 +htmljs@1.1.1 +id-map@1.1.1 +inter-process-messaging@0.1.1 +launch-screen@2.0.0 +localstorage@1.2.0 +logging@1.3.3 +meteor@1.11.4 +meteor-base@1.5.1 +minifier-css@1.6.4 +minifier-js@2.7.5 +minimongo@1.9.3 +mobile-experience@1.1.1 +mobile-status-bar@1.1.0 +modern-browsers@0.1.10 +modules@0.20.0 +modules-runtime@0.13.1 +modules-runtime-hot@0.14.2 +mongo@1.16.8 +mongo-decimal@0.1.3 +mongo-dev-server@1.1.0 +mongo-id@1.0.8 +npm-mongo@4.17.2 +ordered-dict@1.1.0 +promise@0.12.2 +random@1.2.1 +rate-limit@1.1.1 +react-fast-refresh@0.2.8 +react-meteor-data@2.7.2 +reactive-var@1.0.12 +reload@1.3.1 +retry@1.1.0 +routepolicy@1.1.1 +sha@1.0.9 +shell-server@0.5.0 +socket-stream-client@0.5.2 +spacebars-compiler@1.3.1 +standard-minifier-css@1.9.2 +standard-minifier-js@2.8.1 +static-html@1.3.2 +templating-tools@1.2.2 +tracker@1.3.3 +typescript@4.9.5 +underscore@1.0.13 +url@1.3.2 +webapp@1.13.6 +webapp-hashing@1.1.1 diff --git a/components/webui/README.md b/components/webui/README.md new file mode 100644 index 000000000..0f6164fc3 --- /dev/null +++ b/components/webui/README.md @@ -0,0 +1,41 @@ +# Setup + +## Requirements + +* [NodeJS 14](https://nodejs.org/download/release/v14.21.3/) +* [Meteor](https://docs.meteor.com/install.html#installation) + +## Install the dependencies + +```shell +meteor npm install +``` + +If you ever add a package manually to `package.json` or `package.json` changes +for some other reason, you should rerun this command. + +# Running in development + +The full functionality of the webui depends on other components in the CLP +package: + +* Build the CLP package +* Start the package: `/sbin/start-clp` +* Then run `/sbin/stop-clp webui` to stop the webui instance started + in the package. +* Start meteor: + + ```shell + MONGO_URL="mongodb://localhost:27017/clp-search" \ + ROOT_URL="http://" \ + CLP_DB_HOST="localhost" \ + CLP_DB_PORT=3306 \ + CLP_DB_NAME="clp-db" \ + CLP_DB_USER="clp-user" \ + CLP_DB_PASS="" \ + meteor --port --settings settings.json + ``` + + * Change `` to the IP of the machine you're running the webui on. + +* The webui should now be available at `http://:3000` diff --git a/components/webui/client/main.css b/components/webui/client/main.css new file mode 100644 index 000000000..1c25edaaf --- /dev/null +++ b/components/webui/client/main.css @@ -0,0 +1,3 @@ +#react-target { + height: 100%; +} diff --git a/components/webui/client/main.html b/components/webui/client/main.html new file mode 100644 index 000000000..e7718bfa7 --- /dev/null +++ b/components/webui/client/main.html @@ -0,0 +1,7 @@ + + YScope CLP + + + +
+ diff --git a/components/webui/client/main.jsx b/components/webui/client/main.jsx new file mode 100644 index 000000000..1516793fc --- /dev/null +++ b/components/webui/client/main.jsx @@ -0,0 +1,18 @@ +import React from "react"; +import {Meteor} from "meteor/meteor"; +import {render} from "react-dom"; +import {App} from "/imports/ui/App.jsx"; +import {Router, Switch} from "react-router"; +import {createBrowserHistory} from "history"; + +Meteor.startup(() => { + const routes = ( + + + + + + ); + + render(routes, document.getElementById("react-target")); +}); diff --git a/components/webui/imports/api/search/collections.js b/components/webui/imports/api/search/collections.js new file mode 100644 index 000000000..52e2df43a --- /dev/null +++ b/components/webui/imports/api/search/collections.js @@ -0,0 +1,28 @@ +import {Mongo} from "meteor/mongo"; +import {INVALID_JOB_ID, SearchSignal} from "./constants"; + + +export const SearchResultsMetadataCollection = new Mongo.Collection(Meteor.settings.public.SearchResultsMetadataCollectionName); +export const initSearchEventCollection = () => { + // create the collection if not exists + if (SearchResultsMetadataCollection.find().count() === 0) { + SearchResultsMetadataCollection.insert({ + _id: INVALID_JOB_ID.toString(), + lastEvent: SearchSignal.NONE, + errorMsg: null, + numTotalResults: -1 + }); + } +} + +// this should only be accessed by the server; client should use a React referenced-object +export const MY_MONGO_DB = {} + +// Retrieves Mongo Collection object by name if it has been created +// as Meteor.js does not permit creating more than one Collection object with a same name. +export const getCollection = (dbRef, name) => { + if (dbRef[name] === undefined) { + dbRef[name] = new Mongo.Collection(name); + } + return dbRef[name]; +}; diff --git a/components/webui/imports/api/search/constants.js b/components/webui/imports/api/search/constants.js new file mode 100644 index 000000000..050bfa3f6 --- /dev/null +++ b/components/webui/imports/api/search/constants.js @@ -0,0 +1,33 @@ +let enumSearchSignal; +export const SearchSignal = Object.freeze({ + NONE: (enumSearchSignal=0), + REQ_MASK: (enumSearchSignal = 0x10000000), + REQ_CANCELLING: ++enumSearchSignal, + REQ_QUERYING: ++enumSearchSignal, + REQ_CLEARING: ++enumSearchSignal, + RSP_MASK: (enumSearchSignal = 0x20000000), + RSP_DONE: ++enumSearchSignal, + RSP_ERROR: ++enumSearchSignal, + RSP_SEARCHING: ++enumSearchSignal, +}); +export const isSearchSignalReq = (e) => (0 !== (SearchSignal.REQ_MASK & e)); +export const isSearchSignalRsp = (e) => (0 !== (SearchSignal.RSP_MASK & e)); + +// below should match `job_orchestration.search_scheduler.common`: class JobStatus +let enumJobStatus; +export const JobStatus = Object.freeze({ + PENDING: (enumJobStatus=0), + RUNNING: ++enumJobStatus, + SUCCESS: ++enumJobStatus, + FAILED: ++enumJobStatus, + CANCELLING: ++enumJobStatus, + CANCELLED: ++enumJobStatus +}) + +export const JOB_STATUS_WAITING_STATES = [ + JobStatus.PENDING, + JobStatus.RUNNING, + JobStatus.CANCELLING +] + +export const INVALID_JOB_ID = -1; diff --git a/components/webui/imports/api/search/server/methods.js b/components/webui/imports/api/search/server/methods.js new file mode 100644 index 000000000..02df7b5b3 --- /dev/null +++ b/components/webui/imports/api/search/server/methods.js @@ -0,0 +1,152 @@ +import {Meteor} from "meteor/meteor"; +import msgpack from "@msgpack/msgpack"; + +import {JOB_STATUS_WAITING_STATES, JobStatus, SearchSignal} from "../constants"; +import {SQL_CONNECTION} from "../sql"; +import {getCollection, MY_MONGO_DB, SearchResultsMetadataCollection} from "../collections"; + + +const SEARCH_JOBS_TABLE_NAME = "distributed_search_jobs"; + +const sleep = (s) => new Promise(r => setTimeout(r, s * 1000)); + + +const submitQuery = async (args) => { + let jobId = null; + + try { + const [queryInsertResults] = await SQL_CONNECTION.query(`INSERT INTO ${SEARCH_JOBS_TABLE_NAME} (search_config) + VALUES (?) `, [Buffer.from(msgpack.encode(args))]); + jobId = queryInsertResults["insertId"]; + } catch (e) { + console.error("Unable to submit query job to SQL DB", e); + } + + return jobId; +}; + + +const waitTillJobFinishes = async (jobId) => { + let errorMsg = null; + + try { + while (true) { + const [rows, _] = await SQL_CONNECTION.query(`SELECT status + from ${SEARCH_JOBS_TABLE_NAME} + where id = ${jobId}`); + const status = rows[0]["status"]; + if (!JOB_STATUS_WAITING_STATES.includes(status)) { + console.debug(`Job ${jobId} exited with status = ${status}`); + + if (JobStatus.SUCCESS !== status) { + if (JobStatus.NO_MATCHING_ARCHIVE === status) { + errorMsg = "No matching archive by query string and time range"; + } else { + errorMsg = `Job exited in an unexpected status=${status}: ${Object.keys(JobStatus)[status]}`; + } + } + break; + } + + await sleep(0.5); + } + } catch (e) { + errorMsg = `Error querying job status, jobId=${jobId}: ${e}`; + console.error(errorMsg); + } + + return errorMsg; +}; + +const cancelQuery = async (jobId) => { + const [rows, _] = await SQL_CONNECTION.query(`UPDATE ${SEARCH_JOBS_TABLE_NAME} + SET status = ${JobStatus.CANCELLING} + WHERE id = (?)`, [jobId]); +}; + + +const updateSearchEventWhenJobFinishes = async (jobId) => { + const errorMsg = await waitTillJobFinishes(jobId); + const filter = { + _id: jobId.toString() + }; + const modifier = { + $set: { + lastSignal: SearchSignal.RSP_DONE, + errorMsg: errorMsg, + numTotalResults: await getCollection(MY_MONGO_DB, jobId).countDocuments() + } + }; + + SearchResultsMetadataCollection.update(filter, modifier); +}; + +const createMongoIndexes = async (jobId) => { + const timestampAscendingIndex = { + key: {timestamp: 1, _id: 1}, + name: "timestamp-ascending" + }; + + const timestampDescendingIndex = { + key: {timestamp: -1, _id: -1}, + name: "timestamp-descending" + }; + + if (null !== jobId) { + const queryJobCollection = getCollection(MY_MONGO_DB, jobId.toString()); + const queryJobRawCollection = queryJobCollection.rawCollection(); + await queryJobRawCollection.createIndexes([timestampAscendingIndex, timestampDescendingIndex]); + } +}; + +Meteor.methods({ + async "search.submitQuery"({ + queryString, + timestampBegin, + timestampEnd, + }) { + let jobId = null; + + const args = { + query_string: queryString, + begin_timestamp: timestampBegin, + end_timestamp: timestampEnd, + }; + + jobId = await submitQuery(args); + if (null !== jobId) { + SearchResultsMetadataCollection.insert({ + _id: jobId.toString(), + lastSignal: SearchSignal.RSP_SEARCHING, + errorMsg: null + }); + + Meteor.defer(async () => { + await updateSearchEventWhenJobFinishes(jobId); + }); + + // Create indexes for ascending / descending sort + await createMongoIndexes(jobId); + } + + return {jobId}; + }, + + async "search.clearResults"({ + jobId + }) + { + console.debug(`Got request to clear results. jobid = ${jobId}`); + + const resultsCollection = getCollection(MY_MONGO_DB, jobId.toString()); + await resultsCollection.dropCollectionAsync(); + + delete MY_MONGO_DB[jobId.toString()]; + }, + + async "search.cancelOperation"({ + jobId + }) { + await cancelQuery(jobId); + }, +}); diff --git a/components/webui/imports/api/search/server/publications.js b/components/webui/imports/api/search/server/publications.js new file mode 100644 index 000000000..162ef5cd6 --- /dev/null +++ b/components/webui/imports/api/search/server/publications.js @@ -0,0 +1,31 @@ +import {Meteor} from "meteor/meteor"; + +import {SearchResultsMetadataCollection, getCollection, MY_MONGO_DB} from "../collections"; + +Meteor.publish(Meteor.settings.public.SearchResultsMetadataCollectionName, ({jobId}) => { + const filter = { + _id: jobId.toString() + } + + return SearchResultsMetadataCollection.find(filter); +}); + +Meteor.publish(Meteor.settings.public.SearchResultsCollectionName, ({ + jobId, fieldToSortBy, visibleSearchResultsLimit + }) => { + const collection = getCollection(MY_MONGO_DB, jobId.toString()) + + const findOptions = { + limit: visibleSearchResultsLimit, + disableOplog: true, + pollingIntervalMs: 250 + }; + if (fieldToSortBy) { + const sort = {}; + sort[fieldToSortBy.name] = fieldToSortBy.direction; + sort["_id"] = fieldToSortBy.direction; + findOptions["sort"] = sort; + } + + return collection.find({}, findOptions); +}); diff --git a/components/webui/imports/api/search/sql.js b/components/webui/imports/api/search/sql.js new file mode 100644 index 000000000..a000a2a02 --- /dev/null +++ b/components/webui/imports/api/search/sql.js @@ -0,0 +1,31 @@ +import mysql from "mysql2/promise"; + +// SQL connection for submitting queries to the backend +export let SQL_CONNECTION = null; + +export const initSQL = async () => { + const CLP_DB_HOST = process.env['CLP_DB_HOST']; + const CLP_DB_PORT = process.env['CLP_DB_PORT']; + const CLP_DB_NAME = process.env['CLP_DB_NAME']; + const CLP_DB_USER = process.env['CLP_DB_USER']; + const CLP_DB_PASS = process.env['CLP_DB_PASS']; + if ([CLP_DB_HOST, CLP_DB_PORT, CLP_DB_NAME, CLP_DB_USER, CLP_DB_PASS].includes(undefined)) { + console.error("Environment variables CLP_DB_URL, CLP_DB_USER and CLP_DB_PASS need to be defined"); + process.exit(1); + } + + SQL_CONNECTION = await mysql.createConnection({ + host: CLP_DB_HOST, + port: parseInt(CLP_DB_PORT), + database: CLP_DB_NAME, + user: CLP_DB_USER, + password: CLP_DB_PASS, + }); + await SQL_CONNECTION.connect(); +} +export const deinitSQL = async () => { + if (null !== SQL_CONNECTION) { + await SQL_CONNECTION.end(); + SQL_CONNECTION = null; + } +} diff --git a/components/webui/imports/api/user/client/methods.js b/components/webui/imports/api/user/client/methods.js new file mode 100644 index 000000000..9d4fc611b --- /dev/null +++ b/components/webui/imports/api/user/client/methods.js @@ -0,0 +1,65 @@ +import {Meteor} from "meteor/meteor"; +import {v4 as uuidv4} from "uuid"; + +// TODO: implement a full-fledged registration sys +const LOCAL_STORAGE_KEY_USERNAME = 'username'; +const DUMMY_PASSWORD = 'DummyPassword'; + +let LoginRetryCount = 0; +const CONST_MAX_LOGIN_RETRY = 3; + +export const registerAndLoginWithUsername = async (username) => { + try { + await new Promise((resolve, reject) => { + Meteor.call('user.create', {username, password: DUMMY_PASSWORD}, (error) => { + if (error) { + console.log('create user error', error); + reject(error); + } else { + localStorage.setItem(LOCAL_STORAGE_KEY_USERNAME, username); + resolve(true); + } + }); + }); + + return await loginWithUsername(username); + } catch (error) { + return false; + } +}; + +export const loginWithUsername = async (username) => { + try { + return await new Promise((resolve, reject) => { + Meteor.loginWithPassword(username, DUMMY_PASSWORD, (error) => { + if (!error) { + resolve(true); + } else { + console.log('login error', error, 'LOGIN_RETRY_COUNT:', LoginRetryCount); + if (LoginRetryCount < CONST_MAX_LOGIN_RETRY) { + LoginRetryCount++; + resolve(registerAndLoginWithUsername(username)); + } else { + reject(error); + } + } + }); + }); + } catch (error) { + return false; + } +}; + +export const login = async () => { + let username = localStorage.getItem(LOCAL_STORAGE_KEY_USERNAME); + let result; + + if (username === null) { + username = uuidv4(); + result = await registerAndLoginWithUsername(username); + } else { + result = await loginWithUsername(username); + } + + return result +} diff --git a/components/webui/imports/api/user/server/methods.js b/components/webui/imports/api/user/server/methods.js new file mode 100644 index 000000000..9a429a256 --- /dev/null +++ b/components/webui/imports/api/user/server/methods.js @@ -0,0 +1,8 @@ +import {Meteor} from "meteor/meteor"; +import {Accounts} from 'meteor/accounts-base'; + +Meteor.methods({ + 'user.create'({username, password}) { + Accounts.createUser({username, password}); + } +}) diff --git a/components/webui/imports/ui/App.jsx b/components/webui/imports/ui/App.jsx new file mode 100644 index 000000000..dae67f769 --- /dev/null +++ b/components/webui/imports/ui/App.jsx @@ -0,0 +1,69 @@ +import React from "react"; +import {Redirect, Route, Switch} from "react-router"; +import {faSearch} from "@fortawesome/free-solid-svg-icons"; + +import SearchView from "./SearchView/SearchView.jsx"; +import Sidebar from "./Sidebar/Sidebar.jsx"; + +import {login} from "../api/user/client/methods"; +import "./App.scss"; +import LOCAL_STORAGE_KEYS from "./constants/LOCAL_STORAGE_KEYS"; + + +const ROUTES = [ + {path: "/search", label: "Search", icon: faSearch, component: SearchView}, +]; + +export const App = () => { + const [loggedIn, setLoggedIn] = React.useState(false); + const [isSidebarStateCollapsed, setSidebarStateCollapsed] = React.useState( + "true" === localStorage.getItem(LOCAL_STORAGE_KEYS.IS_SIDEBAR_COLLAPSED) + ); + + React.useEffect(async () => { + const result = await login() + setLoggedIn(result) + }, []); + + React.useEffect(() => { + localStorage.setItem(LOCAL_STORAGE_KEYS.IS_SIDEBAR_COLLAPSED, isSidebarStateCollapsed.toString()); + }, [isSidebarStateCollapsed]); + const [isSidebarVisuallyCollapsed, setSidebarVisuallyCollapsed] = React.useState( + "true" === localStorage.getItem(LOCAL_STORAGE_KEYS.IS_SIDEBAR_COLLAPSED) + ); + + const handleSidebarToggle = () => { + setSidebarStateCollapsed(!isSidebarStateCollapsed); + } + + const handleSidebarTransitioned = () => { + setSidebarVisuallyCollapsed(isSidebarStateCollapsed); + } + + return (
+ +
+
+ {!loggedIn ?
+
+
+ Loading... +
+
+
: + + + + + + + } +
+
+
); +} diff --git a/components/webui/imports/ui/App.scss b/components/webui/imports/ui/App.scss new file mode 100644 index 000000000..e4230d432 --- /dev/null +++ b/components/webui/imports/ui/App.scss @@ -0,0 +1,15 @@ +@import "bootstrap-customized.scss"; + +html { + font-size: 14px; + height: 100%; +} + +body { + background-image: linear-gradient(135deg, white, rgba(255, 255, 255, 0.5)); + font-family: 'Source Sans Pro', sans-serif; + height: 100%; +} + +@import "SearchView/SearchView.scss"; +@import "Sidebar/Sidebar.scss"; diff --git a/components/webui/imports/ui/SearchView/SearchControls.js b/components/webui/imports/ui/SearchView/SearchControls.js new file mode 100644 index 000000000..d0d377469 --- /dev/null +++ b/components/webui/imports/ui/SearchView/SearchControls.js @@ -0,0 +1,226 @@ +import * as PropTypes from "prop-types"; +import React, {useEffect, useState} from "react"; + +import {Button, Col, Container, Dropdown, DropdownButton, Form, InputGroup, Row} from "react-bootstrap"; +import DatePicker from "react-datepicker"; +import {FontAwesomeIcon} from "@fortawesome/react-fontawesome"; +import {faBars, faSearch, faTimes, faTrash} from "@fortawesome/free-solid-svg-icons"; + +import {cTimePresets} from "./datetime"; +import LOCAL_STORAGE_KEYS from "../constants/LOCAL_STORAGE_KEYS"; +import {isSearchSignalReq, SearchSignal} from "../../api/search/constants"; + + +const SearchControlsDatePicker = (props) => () + +const SearchControlsFilterLabel = (props) => () + +const SearchFilterControlsDrawer = ({ + timeRange, setTimeRange + }) => { + const updateBeginTimestamp = (date) => { + if (date.getTime() > timeRange.end.getTime()) { + setTimeRange({begin: date, end: date}); + } else { + setTimeRange({begin: date, end: timeRange.end}); + } + } + const updateEndTimestamp = (date) => { + setTimeRange({begin: timeRange.begin, end: date}); + } + + const handleTimeRangePresetSelection = (event) => { + event.preventDefault(); + + let preset_ix = parseInt(event.target.getAttribute("data-preset")); + if (isNaN(preset_ix) || preset_ix >= cTimePresets.length) { + console.error(`Unknown time range preset index: ${preset_ix}`); + } + let preset = cTimePresets[preset_ix]; + let timeRange = preset.compute(); + setTimeRange(timeRange); + } + + let timeRangePresetItems = []; + for (let i = 0; i < cTimePresets.length; ++i) { + timeRangePresetItems.push( + + {cTimePresets[i].label} + + ); + } + + // Compute range of end timestamp so that it's after the begin timestamp + let timestampEndMin = null; + let timestampEndMax = null; + if (timeRange.begin.getFullYear() === timeRange.end.getFullYear() && timeRange.begin.getMonth() === timeRange.end.getMonth() && timeRange.begin.getDate() === timeRange.end.getDate()) { + timestampEndMin = new Date(timeRange.begin); + // TODO This doesn't handle leap seconds + timestampEndMax = new Date(timeRange.end).setHours(23, 59, 59, 999); + } + + return (
+ + + + Time Range + + + + + {timeRangePresetItems} + + + + to + + + + + + +
); +} + +export const SearchControls = ({ + queryString, + setQueryString, + timeRange, + setTimeRange, + resultsMetadata, + submitQuery, + handleClearResults, + cancelOperation, + }) => { + const [drawerOpen, setDrawerOpen] = useState("true" === localStorage.getItem(LOCAL_STORAGE_KEYS.SEARCH_CONTROLS_VISIBLE)); + const [canceling, setCanceling] = useState(false); + + useEffect(() => { + localStorage.setItem(LOCAL_STORAGE_KEYS.SEARCH_CONTROLS_VISIBLE, drawerOpen.toString()); + }, [drawerOpen]); + + const queryChangeHandler = (e) => { + setQueryString(e.target.value); + } + + const handleDrawerToggleClick = () => { + setDrawerOpen(!drawerOpen); + } + + const handleQuerySubmission = (e) => { + e.preventDefault(); + + setCanceling(false); + submitQuery(); + } + + const handleCancelOperation = () => { + setCanceling(true); + cancelOperation(); + } + + return <> +
+ + + + + { + (SearchSignal.RSP_DONE === resultsMetadata["lastSignal"]) && + + } + { + (SearchSignal.RSP_SEARCHING === resultsMetadata["lastSignal"]) ? + : + + } + + +
+ + {drawerOpen && } + +} diff --git a/components/webui/imports/ui/SearchView/SearchResults.jsx b/components/webui/imports/ui/SearchView/SearchResults.jsx new file mode 100644 index 000000000..16b841806 --- /dev/null +++ b/components/webui/imports/ui/SearchView/SearchResults.jsx @@ -0,0 +1,42 @@ +import React from "react"; +import {SearchResultsHeader} from "./SearchResultsHeader"; +import {SearchResultsTable} from "./SearchResultsTable"; + +export const SearchResults = ({ + jobId, + searchResults, + resultsMetadata, + fieldToSortBy, + setFieldToSortBy, + visibleSearchResultsLimit, + setVisibleSearchResultsLimit, + maxLinesPerResult, + setMaxLinesPerResult, +}) => { + const numResultsOnServer = resultsMetadata["numTotalResults"] || searchResults.length; + const isMessageTable = searchResults.length === 0 || + Object.keys(searchResults[0]).includes("timestamp"); + return <> + {isMessageTable &&
+ +
} + {(0 < searchResults.length) &&
+ +
} + ; +}; diff --git a/components/webui/imports/ui/SearchView/SearchResultsHeader.jsx b/components/webui/imports/ui/SearchView/SearchResultsHeader.jsx new file mode 100644 index 000000000..bc1e4211a --- /dev/null +++ b/components/webui/imports/ui/SearchView/SearchResultsHeader.jsx @@ -0,0 +1,74 @@ +import {faCog} from "@fortawesome/free-solid-svg-icons"; +import {FontAwesomeIcon} from "@fortawesome/react-fontawesome"; +import React from "react"; +import { + Button, + Col, + Container, + Form, + InputGroup, + OverlayTrigger, + Popover, + Row, +} from "react-bootstrap"; +import {SearchSignal} from "../../api/search/constants"; + +export const SearchResultsHeader = ({ + jobId, + resultsMetadata, + numResultsOnServer, + maxLinesPerResult, + setMaxLinesPerResult, +}) => { + const handleMaxLinesPerResultSubmission = (e) => { + e.preventDefault(); + const value = parseInt(e.target.elements["maxLinesPerResult"].value); + if (value > 0) { + setMaxLinesPerResult(value); + } + }; + + let numResultsText = `(Job ID ${jobId}) `; + if (0 === numResultsOnServer) { + numResultsText += SearchSignal.RSP_DONE !== resultsMetadata["lastSignal"] ? + "Query is running" : + "No results found"; + } else if (1 === numResultsOnServer) { + numResultsText += "1 result found"; + } else { + numResultsText += `${numResultsOnServer} results found`; + } + + return (<> + + + {numResultsText} + + +
+ + Max lines per result + + +
+ }> + + + + + + ); +}; \ No newline at end of file diff --git a/components/webui/imports/ui/SearchView/SearchResultsTable.jsx b/components/webui/imports/ui/SearchView/SearchResultsTable.jsx new file mode 100644 index 000000000..7361a70d1 --- /dev/null +++ b/components/webui/imports/ui/SearchView/SearchResultsTable.jsx @@ -0,0 +1,116 @@ +import {faSort, faSortDown, faSortUp} from "@fortawesome/free-solid-svg-icons"; +import {FontAwesomeIcon} from "@fortawesome/react-fontawesome"; +import React, {useEffect, useState} from "react"; +import {Spinner, Table} from "react-bootstrap"; +import ReactVisibilitySensor from "react-visibility-sensor"; + +export const VISIBLE_RESULTS_LIMIT_INITIAL = 10; +const VISIBLE_RESULTS_LIMIT_INCREMENT = 10; +export const SearchResultsTable = ({ + searchResults, + maxLinesPerResult, + fieldToSortBy, + setFieldToSortBy, + numResultsOnServer, + visibleSearchResultsLimit, + setVisibleSearchResultsLimit, +}) => { + const [visibilitySensorVisible, setVisibilitySensorVisible] = useState(false); + + useEffect(() => { + if (true === visibilitySensorVisible && visibleSearchResultsLimit <= numResultsOnServer) { + setVisibleSearchResultsLimit( + visibleSearchResultsLimit + VISIBLE_RESULTS_LIMIT_INCREMENT); + } + }, [visibilitySensorVisible, numResultsOnServer]); + + const getSortIcon = (fieldToSortBy, fieldName) => { + if (fieldToSortBy && fieldName === fieldToSortBy.name) { + return (1 === fieldToSortBy.direction ? faSortDown : faSortUp); + } else { + return faSort; + } + }; + + const toggleSortDirection = (event) => { + const columnName = event.currentTarget.dataset.columnName; + if (null === fieldToSortBy || fieldToSortBy.name !== columnName) { + setFieldToSortBy({ + name: columnName, + direction: 1, + }); + } else if (1 === fieldToSortBy.direction) { + // Switch to descending + setFieldToSortBy({ + name: columnName, + direction: -1, + }); + } else if (-1 === fieldToSortBy.direction) { + // Switch to unsorted + setFieldToSortBy(null); + } + }; + + let rows = []; + + // Construct rows + for (let i = 0; i < searchResults.length; ++i) { + let searchResult = searchResults[i]; + rows.push( + {searchResult.timestamp ? new Date(searchResult.timestamp).toISOString(). + slice(0, 19). + replace("T", " ") : "N/A"} + + 
+                    {searchResult.message}
+
+ + ); + } + + return (
+ + + + + + + + + {rows} + +
+
+ Timestamp +
+ 		+
+ Log message +
+
+ +
+ + Loading +
+ +
); +}; \ No newline at end of file diff --git a/components/webui/imports/ui/SearchView/SearchView.jsx b/components/webui/imports/ui/SearchView/SearchView.jsx new file mode 100644 index 000000000..905abdd4d --- /dev/null +++ b/components/webui/imports/ui/SearchView/SearchView.jsx @@ -0,0 +1,234 @@ +import {faExclamationCircle} from "@fortawesome/free-solid-svg-icons"; +import {FontAwesomeIcon} from "@fortawesome/react-fontawesome"; +import {Meteor} from "meteor/meteor"; +import {useTracker} from "meteor/react-meteor-data"; +import React, {useEffect, useRef, useState} from "react"; +import {ProgressBar} from "react-bootstrap"; + +import {getCollection, SearchResultsMetadataCollection} from "../../api/search/collections"; +import {INVALID_JOB_ID, SearchSignal} from "../../api/search/constants"; + +import "react-datepicker/dist/react-datepicker.css"; +import LOCAL_STORAGE_KEYS from "../constants/LOCAL_STORAGE_KEYS"; +import {changeTimezoneToUTCWithoutChangingTime, computeLast15MinTimeRange} from "./datetime"; +import {SearchControls} from "./SearchControls"; +import {SearchResults} from "./SearchResults"; +import {VISIBLE_RESULTS_LIMIT_INITIAL} from "./SearchResultsTable"; + +const SearchView = () => { + // Query states + const [jobId, setJobId] = useState(INVALID_JOB_ID); + + const [operationErrorMsg, setOperationErrorMsg] = useState(""); + const [localLastSearchSignal, setLocalLastSearchSignal] = useState(SearchSignal.NONE); + + const dbRef = useRef({}); + const localLastSearchSignalRef = useRef(localLastSearchSignal); + + // Query options + const [queryString, setQueryString] = useState(""); + const [timeRange, setTimeRange] = useState(computeLast15MinTimeRange); + const [visibleSearchResultsLimit, setVisibleSearchResultsLimit] = useState( + VISIBLE_RESULTS_LIMIT_INITIAL); + const [fieldToSortBy, setFieldToSortBy] = useState({ + name: "timestamp", + direction: -1, + }); + + // Visuals + const [maxLinesPerResult, setMaxLinesPerResult] = useState( + Number(localStorage.getItem(LOCAL_STORAGE_KEYS.MAX_LINES_PER_RESULT) || 2)); + + // Subscriptions + const resultsMetadata = useTracker(() => { + let result = {lastSignal: localLastSearchSignal}; + + if (INVALID_JOB_ID !== jobId) { + const args = {jobId}; + const subscription = Meteor.subscribe( + Meteor.settings.public.SearchResultsMetadataCollectionName, args); + const doc = SearchResultsMetadataCollection.findOne(); + + const isReady = subscription.ready(); + if (true === isReady) { + result = doc; + } + } + + return result; + }, [jobId, localLastSearchSignal]); + + const searchResults = useTracker(() => { + if (INVALID_JOB_ID === jobId) { + return []; + } + + Meteor.subscribe(Meteor.settings.public.SearchResultsCollectionName, { + jobId: jobId, + fieldToSortBy: fieldToSortBy, + visibleSearchResultsLimit: visibleSearchResultsLimit, + }); + + return getCollection(dbRef.current, jobId.toString()).find().fetch(); + }, [jobId, fieldToSortBy, visibleSearchResultsLimit]); + + // State transitions + useEffect(() => { + localStorage.setItem(LOCAL_STORAGE_KEYS.MAX_LINES_PER_RESULT, maxLinesPerResult.toString()); + }, [maxLinesPerResult]); + + useEffect(() => { + localLastSearchSignalRef.current = localLastSearchSignal; + }, [localLastSearchSignal]); + + // Handlers + const resetVisibleResultSettings = () => { + setVisibleSearchResultsLimit(VISIBLE_RESULTS_LIMIT_INITIAL); + }; + + const submitQuery = () => { + if (INVALID_JOB_ID !== jobId) { + // Clear result caches before starting a new query + handleClearResults(); + } + + setOperationErrorMsg(""); + setLocalLastSearchSignal(SearchSignal.REQ_QUERYING); + resetVisibleResultSettings(); + + const timestampBeginMillis = changeTimezoneToUTCWithoutChangingTime(timeRange.begin). + getTime(); + const timestampEndMillis = changeTimezoneToUTCWithoutChangingTime(timeRange.end).getTime(); + + const args = { + queryString: queryString, + timestampBegin: timestampBeginMillis, + timestampEnd: timestampEndMillis, + }; + Meteor.call("search.submitQuery", args, (error, result) => { + if (error) { + setOperationErrorMsg(error.reason); + } + + setJobId(result["jobId"]); + }); + }; + + const handleClearResults = () => { + delete dbRef.current[jobId.toString()]; + + setJobId(INVALID_JOB_ID); + setOperationErrorMsg(""); + setLocalLastSearchSignal(SearchSignal.REQ_CLEARING); + resetVisibleResultSettings(); + + const args = { + jobId: jobId, + }; + Meteor.call("search.clearResults", args, (error) => { + if (error) { + setOperationErrorMsg(error.reason); + } + + if (SearchSignal.REQ_CLEARING === localLastSearchSignalRef.current) { + // The check prevents clearing localLastSearchSignal=SearchSignal.REQ_QUERYING + // when `handleClearResults` is called by submitQuery. + setLocalLastSearchSignal(SearchSignal.NONE); + } + }); + }; + + const cancelOperation = () => { + setOperationErrorMsg(""); + setLocalLastSearchSignal(SearchSignal.REQ_CANCELLING); + + const args = { + jobId: jobId, + }; + Meteor.call("search.cancelOperation", args, (error) => { + if (error) { + setOperationErrorMsg(error.reason); + } + }); + }; + + const showSearchResults = INVALID_JOB_ID !== jobId; + return (
+
+ + + +
+ + {showSearchResults && } +
); +}; + +const SearchStatus = ({ + resultsMetadata, + errorMsg, +}) => { + if ("" !== errorMsg && null !== errorMsg && undefined !== errorMsg) { + return (
+ + {errorMsg} +
); + } else { + let message; + switch (resultsMetadata["lastSignal"]) { + case SearchSignal.NONE: + message = "Ready"; + break; + case SearchSignal.REQ_CANCELLING: + message = "Cancelling..."; + break; + case SearchSignal.REQ_CLEARING: + message = "Clearing..."; + break; + case SearchSignal.REQ_QUERYING: + case SearchSignal.RSP_SEARCHING: + message = "Searching..."; + break; + default: + message = "Unknown state / No message"; + } + + return <> + + {SearchSignal.RSP_DONE !== resultsMetadata["lastSignal"] && +
{message}
} + ; + } +}; + +export default SearchView; diff --git a/components/webui/imports/ui/SearchView/SearchView.scss b/components/webui/imports/ui/SearchView/SearchView.scss new file mode 100644 index 000000000..37e543070 --- /dev/null +++ b/components/webui/imports/ui/SearchView/SearchView.scss @@ -0,0 +1,94 @@ +.search-results-container { + position: relative; +} + +// NOTE: We use a hierarchical selector to override Bootstrap's hierarchical selector (classes can't override hierarchical selectors without !important) +.search-results thead th { + border: none; + padding: 0; +} + +.search-results-th { + position: sticky; + top: 0; + + &-sortable { + cursor: pointer; + user-select: none; + } +} + +.search-results-table-header { + background-color: #fff; + border-bottom: 1px solid #ccc; + border-top: 1px solid #dee2e6; + padding: 10px; +} + +.search-results-message { + font-size: 0.93rem; + line-height: 1.4rem; + margin: 0; + white-space: pre-wrap; + word-break: break-word; +} + +.search-results-display-settings-container { + max-width: 100%; +} + +.search-results-display-settings { + width: 200px; +} + +.search-results-count { + display: block; + padding: 0.375rem 0; + + color: #999; + font-size: 1rem; + line-height: 1.5; +} + +.search-results-title-bar { + background-color: #fff; + padding: 2px 0; +} + +.search-no-results-status { + color: #ccc; + font-size: 5rem; + line-height: 1; + + padding: 1rem 2rem; +} + +.search-error { + padding: 0.5rem; + + background-color: #ffc0c0; + color: #8e0000; + border-top: 1px solid #ca9b9b; + border-bottom: 1px solid #ecb4b4; +} + +.search-error-icon { + margin-right: 0.5rem; + vertical-align: middle; +} + +.search-progress-bar { + border-top: 1px solid #ccc; + height: 4px; +} + +.timestamp-picker { + border: 1px solid rgb(206, 212, 218); + border-radius: 0; + + font-size: 0.875rem; + line-height: 1.5; + + height: calc(1.5em + 0.5rem + 2px); + padding: 0.25rem 0.5rem; +} diff --git a/components/webui/imports/ui/SearchView/datetime.js b/components/webui/imports/ui/SearchView/datetime.js new file mode 100644 index 000000000..cc09a08f3 --- /dev/null +++ b/components/webui/imports/ui/SearchView/datetime.js @@ -0,0 +1,211 @@ +import {DateTime} from "luxon"; + +export const computeTodayTimeRange = () => { + const endTime = DateTime.utc(); + const beginTime = endTime.minus({days: 1}); + return { + begin: dateTimeToDateWithoutChangingTimestamp(beginTime), + end: dateTimeToDateWithoutChangingTimestamp(endTime), + }; +}; + +export const computeWeekToDateTimeRange = () => { + const endTime = DateTime.utc(); + const beginTime = endTime.startOf("week"); + return { + begin: dateTimeToDateWithoutChangingTimestamp(beginTime), + end: dateTimeToDateWithoutChangingTimestamp(endTime), + }; +}; + +export const computeMonthToDateTimeRange = () => { + const endTime = DateTime.utc(); + const beginTime = endTime.startOf("month"); + return { + begin: dateTimeToDateWithoutChangingTimestamp(beginTime), + end: dateTimeToDateWithoutChangingTimestamp(endTime), + }; +}; + +export const computeYearToDateTimeRange = () => { + const endTime = DateTime.utc(); + const beginTime = endTime.startOf("year"); + return { + begin: dateTimeToDateWithoutChangingTimestamp(beginTime), + end: dateTimeToDateWithoutChangingTimestamp(endTime), + }; +}; + +export const computePrevDayTimeRange = () => { + const endTime = DateTime.utc().minus({days: 1}).endOf("day"); + const beginTime = endTime.startOf("day"); + return { + begin: dateTimeToDateWithoutChangingTimestamp(beginTime), + end: dateTimeToDateWithoutChangingTimestamp(endTime), + }; +}; + +export const computePrevWeekTimeRange = () => { + const endTime = DateTime.utc().minus({weeks: 1}).endOf("week"); + const beginTime = endTime.startOf("week"); + return { + begin: dateTimeToDateWithoutChangingTimestamp(beginTime), + end: dateTimeToDateWithoutChangingTimestamp(endTime), + }; +}; + +export const computePrevMonthTimeRange = () => { + const endTime = DateTime.utc().minus({months: 1}).endOf("month"); + const beginTime = endTime.startOf("month"); + return { + begin: dateTimeToDateWithoutChangingTimestamp(beginTime), + end: dateTimeToDateWithoutChangingTimestamp(endTime), + }; +}; + +export const computePrevYearTimeRange = () => { + const endTime = DateTime.utc().minus({years: 1}).endOf("year"); + const beginTime = endTime.startOf("year"); + return { + begin: dateTimeToDateWithoutChangingTimestamp(beginTime), + end: dateTimeToDateWithoutChangingTimestamp(endTime), + }; +}; + +export const computeLast15MinTimeRange = () => { + const endTime = DateTime.utc(); + const beginTime = endTime.minus({minutes: 15}); + return { + begin: dateTimeToDateWithoutChangingTimestamp(beginTime), + end: dateTimeToDateWithoutChangingTimestamp(endTime), + }; +}; + +export const computeLast60MinTimeRange = () => { + const endTime = DateTime.utc(); + const beginTime = endTime.minus({minutes: 60}); + return { + begin: dateTimeToDateWithoutChangingTimestamp(beginTime), + end: dateTimeToDateWithoutChangingTimestamp(endTime), + }; +}; + +export const computeLast4HourTimeRange = () => { + const endTime = DateTime.utc(); + const beginTime = endTime.minus({hours: 4}); + return { + begin: dateTimeToDateWithoutChangingTimestamp(beginTime), + end: dateTimeToDateWithoutChangingTimestamp(endTime), + }; +}; + +export const computeLast24HourTimeRange = () => { + const endTime = DateTime.utc(); + const beginTime = endTime.minus({hours: 24}); + return { + begin: dateTimeToDateWithoutChangingTimestamp(beginTime), + end: dateTimeToDateWithoutChangingTimestamp(endTime), + }; +}; + +export const computeAllTimeRange = () => { + const endTime = DateTime.utc().plus({years: 1}); + const beginTime = DateTime.fromMillis(0, {zone: "UTC"}); + return { + begin: dateTimeToDateWithoutChangingTimestamp(beginTime), + end: dateTimeToDateWithoutChangingTimestamp(endTime), + }; +}; + +export const cTimePresets = [ + { + key: "last-15-mins", + label: "Last 15 Minutes", + compute: computeLast15MinTimeRange, + }, + { + key: "last-60-mins", + label: "Last 60 Minutes", + compute: computeLast60MinTimeRange, + }, + { + key: "last-4-hours", + label: "Last 4 Hours", + compute: computeLast4HourTimeRange, + }, + { + key: "last-24-hours", + label: "Last 24 Hours", + compute: computeLast24HourTimeRange, + }, + { + key: "prev-day", + label: "Previous Day", + compute: computePrevDayTimeRange, + }, + { + key: "prev-week", + label: "Previous Week", + compute: computePrevWeekTimeRange, + }, + { + key: "prev-month", + label: "Previous Month", + compute: computePrevMonthTimeRange, + }, + { + key: "prev-year", + label: "Previous Year", + compute: computePrevYearTimeRange, + }, + { + key: "today", + label: "Today", + compute: computeTodayTimeRange, + }, + { + key: "week-to-date", + label: "Week to Date", + compute: computeWeekToDateTimeRange, + }, + { + key: "month-to-date", + label: "Month to Date", + compute: computeMonthToDateTimeRange, + }, + { + key: "year-to-date", + label: "Year to Date", + compute: computeYearToDateTimeRange, + }, + { + key: "all-time", + label: "All Time", + compute: computeAllTimeRange, + }, +]; + +export const changeTimezoneToUTCWithoutChangingTime = (date) => { + return new Date(Date.UTC( + date.getFullYear(), + date.getMonth(), + date.getDate(), + date.getHours(), + date.getMinutes(), + date.getSeconds(), + date.getMilliseconds(), + )); +}; + +// TODO Switch date pickers so we don't have to do this hack +export const dateTimeToDateWithoutChangingTimestamp = (dateTime) => { + return dateTime.toLocal().set({ + year: dateTime.year, + month: dateTime.month, + day: dateTime.day, + hour: dateTime.hour, + minute: dateTime.minute, + second: dateTime.second, + millisecond: dateTime.millisecond, + }).toJSDate(); +}; diff --git a/components/webui/imports/ui/Sidebar/Sidebar.jsx b/components/webui/imports/ui/Sidebar/Sidebar.jsx new file mode 100644 index 000000000..5f189517a --- /dev/null +++ b/components/webui/imports/ui/Sidebar/Sidebar.jsx @@ -0,0 +1,48 @@ +import React from "react"; +import {NavLink} from "react-router-dom"; + +import {FontAwesomeIcon} from "@fortawesome/react-fontawesome"; +import { + faAngleDoubleLeft, + faAngleDoubleRight, +} from "@fortawesome/free-solid-svg-icons"; + +const Sidebar = ({isSidebarCollapsed, onSidebarToggle, onSidebarTransitioned, routes}) => { + return ( + + ); +}; + +export default Sidebar; diff --git a/components/webui/imports/ui/Sidebar/Sidebar.scss b/components/webui/imports/ui/Sidebar/Sidebar.scss new file mode 100644 index 000000000..1eb8b2068 --- /dev/null +++ b/components/webui/imports/ui/Sidebar/Sidebar.scss @@ -0,0 +1,153 @@ +#sidebar { + height: 100%; + min-width: 190px; + width: 190px; + + display: flex; + flex-direction: column; + + background: #080808; + color: #dedede; + + transition: min-width 0.5s, width 0.5s; +} + +#sidebar.collapsed { + min-width: 3rem; + width: 3rem; +} + +#sidebar .brand { + align-items: center; /* center element vertically */ + display: flex; + flex: 0 0 3rem; /* cannot grow, cannot shrink, 44px height */ + justify-content: center; /* center element horizontally */ + + border-left: 1px solid #111; + + font-size: 1.25rem; + white-space: nowrap; /* proper animation */ + overflow: hidden; /* proper animation */ +} + +.sidebar-menu { + flex: 1 1 auto; /* can shrink, can grow, auto height */ + display: flex; + flex-direction: column; + overflow-y: auto; +} + +.sidebar-item-icon { + display: inline-block; + margin-left: -1px; + + width: 3rem; + + text-align: center; +} + +.sidebar-item-text { + opacity: inherit; + transition: opacity 0.5s; +} + +#sidebar.collapsed .sidebar-item-text { + opacity: 0; +} + +.sidebar-menu a { + border-left: 1px solid #333; + color: #aaa; + text-decoration: none; + + height: 3rem; + width: 100%; + + display: block; + margin-top: 2px; + overflow: hidden; + + line-height: 3rem; + white-space: nowrap; + + transition: width 0.5s; +} + +.sidebar-menu a:hover, .sidebar-menu .active { + border-left: 1px solid #dedede; + color: #dedede; + text-decoration: none; +} + +.sidebar-menu a:hover { + background-color: #0d5259; +} + +.sidebar-menu .active { + background-color: #004952; +} + +.sidebar-collapse-toggle { + border-left: 1px solid #333; + + line-height: 3rem; + overflow: hidden; + text-align: center; + white-space: nowrap; + + cursor: pointer; + user-select: none; +} + +.sidebar-collapse-icon { + display: inline-block; + + width: 1.5rem; + + text-align: center; + + transition: width 0.5s; +} + +#sidebar.collapsed .sidebar-collapse-icon { + width: 3rem; +} + +.sidebar-collapse-text { + opacity: inherit; + transition: opacity 0.5s; +} + +#sidebar.collapsed .sidebar-collapse-text { + opacity: 0; +} + +.sidebar-collapse-toggle:hover { + background: #0d5259; + border-left: 1px solid #dedede; +} + +.sidebar-menu a.logout:hover { + background: #580000; +} + +#sidebar .sidebar-menu::-webkit-scrollbar { + width: 4px; + border-radius: 2px; +} + +#sidebar .sidebar-menu::-webkit-scrollbar-track { + background: #f1f1f1; + border-right: 1px solid rgb(34, 45, 50); + border-radius: 2px; +} + +#sidebar .sidebar-menu::-webkit-scrollbar-thumb { + background: #888; + border-radius: 2px; +} + +#sidebar .sidebar-menu::-webkit-scrollbar-thumb:hover { + background: #555; + border-radius: 2px; +} diff --git a/components/webui/imports/ui/bootstrap-customized.scss b/components/webui/imports/ui/bootstrap-customized.scss new file mode 100644 index 000000000..e66685621 --- /dev/null +++ b/components/webui/imports/ui/bootstrap-customized.scss @@ -0,0 +1,5 @@ +$primary: #007380; +$secondary: #e9ecef; +$info: #a60058; + +@import "~bootstrap/scss/bootstrap"; diff --git a/components/webui/imports/ui/constants/LOCAL_STORAGE_KEYS.js b/components/webui/imports/ui/constants/LOCAL_STORAGE_KEYS.js new file mode 100644 index 000000000..ff03f723b --- /dev/null +++ b/components/webui/imports/ui/constants/LOCAL_STORAGE_KEYS.js @@ -0,0 +1,7 @@ +const LOCAL_STORAGE_KEYS = Object.freeze({ + IS_SIDEBAR_COLLAPSED: "isSidebarCollapsed", + MAX_LINES_PER_RESULT: "maxLinesPerResult", + SEARCH_CONTROLS_VISIBLE: "searchFilterControlsVisible", +}); + +export default LOCAL_STORAGE_KEYS; diff --git a/components/webui/package-lock.json b/components/webui/package-lock.json new file mode 100644 index 000000000..d7915ab68 --- /dev/null +++ b/components/webui/package-lock.json @@ -0,0 +1,1429 @@ +{ + "name": "webui", + "requires": true, + "lockfileVersion": 1, + "dependencies": { + "@babel/runtime": { + "version": "7.23.1", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.23.1.tgz", + "integrity": "sha512-hC2v6p8ZSI/W0HUzh3V8C5g+NwSKzKPtJwSpTjwl0o297GP9+ZLQSkdvHz46CM3LqyoXxq+5G9komY+eSqSO0g==", + "requires": { + "regenerator-runtime": "^0.14.0" + } + }, + "@fortawesome/fontawesome-common-types": { + "version": "6.4.2", + "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-common-types/-/fontawesome-common-types-6.4.2.tgz", + "integrity": "sha512-1DgP7f+XQIJbLFCTX1V2QnxVmpLdKdzzo2k8EmvDOePfchaIGQ9eCHj2up3/jNEbZuBqel5OxiaOJf37TWauRA==" + }, + "@fortawesome/fontawesome-svg-core": { + "version": "6.4.2", + "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-svg-core/-/fontawesome-svg-core-6.4.2.tgz", + "integrity": "sha512-gjYDSKv3TrM2sLTOKBc5rH9ckje8Wrwgx1CxAPbN5N3Fm4prfi7NsJVWd1jklp7i5uSCVwhZS5qlhMXqLrpAIg==", + "requires": { + "@fortawesome/fontawesome-common-types": "6.4.2" + } + }, + "@fortawesome/free-solid-svg-icons": { + "version": "6.4.2", + "resolved": "https://registry.npmjs.org/@fortawesome/free-solid-svg-icons/-/free-solid-svg-icons-6.4.2.tgz", + "integrity": "sha512-sYwXurXUEQS32fZz9hVCUUv/xu49PEJEyUOsA51l6PU/qVgfbTb2glsTEaJngVVT8VqBATRIdh7XVgV1JF1LkA==", + "requires": { + "@fortawesome/fontawesome-common-types": "6.4.2" + } + }, + "@fortawesome/react-fontawesome": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@fortawesome/react-fontawesome/-/react-fontawesome-0.2.0.tgz", + "integrity": "sha512-uHg75Rb/XORTtVt7OS9WoK8uM276Ufi7gCzshVWkUJbHhh3svsUUeqXerrM96Wm7fRiDzfKRwSoahhMIkGAYHw==", + "requires": { + "prop-types": "^15.8.1" + } + }, + "@msgpack/msgpack": { + "version": "3.0.0-beta2", + "resolved": "https://registry.npmjs.org/@msgpack/msgpack/-/msgpack-3.0.0-beta2.tgz", + "integrity": "sha512-y+l1PNV0XDyY8sM3YtuMLK5vE3/hkfId+Do8pLo/OPxfxuFAUwcGz3oiiUuV46/aBpwTzZ+mRWVMtlSKbradhw==" + }, + "@popperjs/core": { + "version": "2.11.8", + "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.8.tgz", + "integrity": "sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A==" + }, + "@react-aria/ssr": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@react-aria/ssr/-/ssr-3.8.0.tgz", + "integrity": "sha512-Y54xs483rglN5DxbwfCPHxnkvZ+gZ0LbSYmR72LyWPGft8hN/lrl1VRS1EW2SMjnkEWlj+Km2mwvA3kEHDUA0A==", + "requires": { + "@swc/helpers": "^0.5.0" + } + }, + "@restart/hooks": { + "version": "0.4.11", + "resolved": "https://registry.npmjs.org/@restart/hooks/-/hooks-0.4.11.tgz", + "integrity": "sha512-Ft/ncTULZN6ldGHiF/k5qt72O8JyRMOeg0tApvCni8LkoiEahO+z3TNxfXIVGy890YtWVDvJAl662dVJSJXvMw==", + "requires": { + "dequal": "^2.0.3" + } + }, + "@restart/ui": { + "version": "1.6.6", + "resolved": "https://registry.npmjs.org/@restart/ui/-/ui-1.6.6.tgz", + "integrity": "sha512-eC3puKuWE1SRYbojWHXnvCNHGgf3uzHCb6JOhnF4OXPibOIPEkR1sqDSkL643ydigxwh+ruCa1CmYHlzk7ikKA==", + "requires": { + "@babel/runtime": "^7.21.0", + "@popperjs/core": "^2.11.6", + "@react-aria/ssr": "^3.5.0", + "@restart/hooks": "^0.4.9", + "@types/warning": "^3.0.0", + "dequal": "^2.0.3", + "dom-helpers": "^5.2.0", + "uncontrollable": "^8.0.1", + "warning": "^4.0.3" + }, + "dependencies": { + "uncontrollable": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/uncontrollable/-/uncontrollable-8.0.4.tgz", + "integrity": "sha512-ulRWYWHvscPFc0QQXvyJjY6LIXU56f0h8pQFvhxiKk5V1fcI8gp9Ht9leVAhrVjzqMw0BgjspBINx9r6oyJUvQ==" + } + } + }, + "@swc/helpers": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.2.tgz", + "integrity": "sha512-E4KcWTpoLHqwPHLxidpOqQbcrZVgi0rsmmZXUle1jXmJfuIf/UWpczUJ7MZZ5tlxytgJXyp0w4PGkkeLiuIdZw==", + "requires": { + "tslib": "^2.4.0" + } + }, + "@types/prop-types": { + "version": "15.7.8", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.8.tgz", + "integrity": "sha512-kMpQpfZKSCBqltAJwskgePRaYRFukDkm1oItcAbC3gNELR20XIBcN9VRgg4+m8DKsTfkWeA4m4Imp4DDuWy7FQ==" + }, + "@types/react": { + "version": "18.2.24", + "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.24.tgz", + "integrity": "sha512-Ee0Jt4sbJxMu1iDcetZEIKQr99J1Zfb6D4F3qfUWoR1JpInkY1Wdg4WwCyBjL257D0+jGqSl1twBjV8iCaC0Aw==", + "requires": { + "@types/prop-types": "*", + "@types/scheduler": "*", + "csstype": "^3.0.2" + } + }, + "@types/react-transition-group": { + "version": "4.4.7", + "resolved": "https://registry.npmjs.org/@types/react-transition-group/-/react-transition-group-4.4.7.tgz", + "integrity": "sha512-ICCyBl5mvyqYp8Qeq9B5G/fyBSRC0zx3XM3sCC6KkcMsNeAHqXBKkmat4GqdJET5jtYUpZXrxI5flve5qhi2Eg==", + "requires": { + "@types/react": "*" + } + }, + "@types/scheduler": { + "version": "0.16.4", + "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.4.tgz", + "integrity": "sha512-2L9ifAGl7wmXwP4v3pN4p2FLhD0O1qsJpvKmNin5VA8+UvNVb447UDaAEV6UdrkA+m/Xs58U1RFps44x6TFsVQ==" + }, + "@types/warning": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@types/warning/-/warning-3.0.1.tgz", + "integrity": "sha512-ywJmriP+nvjBKNBEMaNZgj2irZHoxcKeYcyMLbqhYKbDVn8yCIULy2Ol/tvIb37O3IBeZj3RU4tXqQTtGwoAMg==" + }, + "bootstrap": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/bootstrap/-/bootstrap-5.3.2.tgz", + "integrity": "sha512-D32nmNWiQHo94BKHLmOrdjlL05q1c8oxbtBphQFb9Z5to6eGRDCm0QgeaZ4zFBHzfg2++rqa2JkqCcxDy0sH0g==" + }, + "classnames": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.3.2.tgz", + "integrity": "sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw==" + }, + "csstype": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.2.tgz", + "integrity": "sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ==" + }, + "date-fns": { + "version": "2.30.0", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz", + "integrity": "sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==", + "requires": { + "@babel/runtime": "^7.21.0" + } + }, + "denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==" + }, + "dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==" + }, + "dom-helpers": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/dom-helpers/-/dom-helpers-5.2.1.tgz", + "integrity": "sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA==", + "requires": { + "@babel/runtime": "^7.8.7", + "csstype": "^3.0.2" + } + }, + "generate-function": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.3.1.tgz", + "integrity": "sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ==", + "requires": { + "is-property": "^1.0.2" + } + }, + "highcharts": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/highcharts/-/highcharts-11.1.0.tgz", + "integrity": "sha512-vhmqq6/frteWMx0GKYWwEFL25g4OYc7+m+9KQJb/notXbNtIb8KVy+ijOF7XAFqF165cq0pdLIePAmyFY5ph3g==" + }, + "highcharts-custom-events": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/highcharts-custom-events/-/highcharts-custom-events-3.0.10.tgz", + "integrity": "sha512-swD0v8qovr9tzW6wNdebJKWQS5xTUclU2XznZw7+U6zMpedzEw7nxMZCog+P3JBDvp2cLO8ySiy7EC29DayfFw==", + "requires": { + "highcharts": ">=5.0.9" + } + }, + "highcharts-react-official": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/highcharts-react-official/-/highcharts-react-official-3.2.1.tgz", + "integrity": "sha512-hyQTX7ezCxl7JqumaWiGsroGWalzh24GedQIgO3vJbkGOZ6ySRAltIYjfxhrq4HszJOySZegotEF7v+haQ75UA==" + }, + "history": { + "version": "4.10.1", + "resolved": "https://registry.npmjs.org/history/-/history-4.10.1.tgz", + "integrity": "sha512-36nwAD620w12kuzPAsyINPWJqlNbij+hpK1k9XRloDtym8mxzGYl2c17LnV6IAGB2Dmg4tEa7G7DlawS0+qjew==", + "requires": { + "@babel/runtime": "^7.1.2", + "loose-envify": "^1.2.0", + "resolve-pathname": "^3.0.0", + "tiny-invariant": "^1.0.2", + "tiny-warning": "^1.0.0", + "value-equal": "^1.0.1" + } + }, + "hoist-non-react-statics": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz", + "integrity": "sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==", + "requires": { + "react-is": "^16.7.0" + } + }, + "iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + } + }, + "invariant": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz", + "integrity": "sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==", + "requires": { + "loose-envify": "^1.0.0" + } + }, + "is-property": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", + "integrity": "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g==" + }, + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==" + }, + "js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" + }, + "long": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", + "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" + }, + "loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "requires": { + "js-tokens": "^3.0.0 || ^4.0.0" + } + }, + "lru-cache": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-8.0.5.tgz", + "integrity": "sha512-MhWWlVnuab1RG5/zMRRcVGXZLCXrZTgfwMikgzCegsPnG62yDQo5JnqKkrK4jO5iKqDAZGItAqN5CtKBCBWRUA==" + }, + "luxon": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.3.tgz", + "integrity": "sha512-tFWBiv3h7z+T/tDaoxA8rqTxy1CHV6gHS//QdaH4pulbq/JuBSGgQspQQqcgnwdAx6pNI7cmvz5Sv/addzHmUg==" + }, + "meteor-node-stubs": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/meteor-node-stubs/-/meteor-node-stubs-1.2.5.tgz", + "integrity": "sha512-FLlOFZx3KnZ5s3yPCK+x58DyX9ewN+oQ12LcpwBXMLtzJ/YyprMQVivd6KIrahZbKJrNenPNUGuDS37WUFg+Mw==", + "requires": { + "assert": "^2.0.0", + "browserify-zlib": "^0.2.0", + "buffer": "^5.7.1", + "console-browserify": "^1.2.0", + "constants-browserify": "^1.0.0", + "crypto-browserify": "^3.12.0", + "domain-browser": "^4.22.0", + "elliptic": "^6.5.4", + "events": "^3.3.0", + "https-browserify": "^1.0.0", + "os-browserify": "^0.3.0", + "path-browserify": "^1.0.0", + "process": "^0.11.10", + "punycode": "^1.4.1", + "querystring-es3": "^0.2.1", + "readable-stream": "^3.6.0", + "stream-browserify": "^3.0.0", + "stream-http": "^3.2.0", + "string_decoder": "^1.3.0", + "timers-browserify": "^2.0.12", + "tty-browserify": "0.0.1", + "url": "^0.11.0", + "util": "^0.12.4", + "vm-browserify": "^1.1.2" + }, + "dependencies": { + "asn1.js": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz", + "integrity": "sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==", + "requires": { + "bn.js": "^4.0.0", + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0", + "safer-buffer": "^2.1.0" + }, + "dependencies": { + "bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + } + } + }, + "assert": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/assert/-/assert-2.0.0.tgz", + "integrity": "sha512-se5Cd+js9dXJnu6Ag2JFc00t+HmHOen+8Q+L7O9zI0PqQXr20uk2J0XQqMxZEeo5U50o8Nvmmx7dZrl+Ufr35A==", + "requires": { + "es6-object-assign": "^1.1.0", + "is-nan": "^1.2.1", + "object-is": "^1.0.1", + "util": "^0.12.0" + } + }, + "available-typed-arrays": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.4.tgz", + "integrity": "sha512-SA5mXJWrId1TaQjfxUYghbqQ/hYioKmLJvPJyDuYRtXXenFNMjj4hSSt1Cf1xsuXSXrtxrVC5Ot4eU6cOtBDdA==" + }, + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, + "bn.js": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.2.0.tgz", + "integrity": "sha512-D7iWRBvnZE8ecXiLj/9wbxH7Tk79fAh8IHaTNq1RWRixsS02W+5qS+iE9yq6RYl0asXx5tw0bLhmT5pIfbSquw==" + }, + "brorand": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz", + "integrity": "sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=" + }, + "browserify-aes": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", + "integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==", + "requires": { + "buffer-xor": "^1.0.3", + "cipher-base": "^1.0.0", + "create-hash": "^1.1.0", + "evp_bytestokey": "^1.0.3", + "inherits": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "browserify-cipher": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/browserify-cipher/-/browserify-cipher-1.0.1.tgz", + "integrity": "sha512-sPhkz0ARKbf4rRQt2hTpAHqn47X3llLkUGn+xEJzLjwY8LRs2p0v7ljvI5EyoRO/mexrNunNECisZs+gw2zz1w==", + "requires": { + "browserify-aes": "^1.0.4", + "browserify-des": "^1.0.0", + "evp_bytestokey": "^1.0.0" + } + }, + "browserify-des": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/browserify-des/-/browserify-des-1.0.2.tgz", + "integrity": "sha512-BioO1xf3hFwz4kc6iBhI3ieDFompMhrMlnDFC4/0/vd5MokpuAc3R+LYbwTA9A5Yc9pq9UYPqffKpW2ObuwX5A==", + "requires": { + "cipher-base": "^1.0.1", + "des.js": "^1.0.0", + "inherits": "^2.0.1", + "safe-buffer": "^5.1.2" + } + }, + "browserify-rsa": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.1.0.tgz", + "integrity": "sha512-AdEER0Hkspgno2aR97SAf6vi0y0k8NuOpGnVH3O99rcA5Q6sh8QxcngtHuJ6uXwnfAXNM4Gn1Gb7/MV1+Ymbog==", + "requires": { + "bn.js": "^5.0.0", + "randombytes": "^2.0.1" + } + }, + "browserify-sign": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.2.1.tgz", + "integrity": "sha512-/vrA5fguVAKKAVTNJjgSm1tRQDHUU6DbwO9IROu/0WAzC8PKhucDSh18J0RMvVeHAn5puMd+QHC2erPRNf8lmg==", + "requires": { + "bn.js": "^5.1.1", + "browserify-rsa": "^4.0.1", + "create-hash": "^1.2.0", + "create-hmac": "^1.1.7", + "elliptic": "^6.5.3", + "inherits": "^2.0.4", + "parse-asn1": "^5.1.5", + "readable-stream": "^3.6.0", + "safe-buffer": "^5.2.0" + } + }, + "browserify-zlib": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/browserify-zlib/-/browserify-zlib-0.2.0.tgz", + "integrity": "sha512-Z942RysHXmJrhqk88FmKBVq/v5tqmSkDz7p54G/MGyjMnCFFnC79XWNbg+Vta8W6Wb2qtSZTSxIGkJrRpCFEiA==", + "requires": { + "pako": "~1.0.5" + } + }, + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "buffer-xor": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz", + "integrity": "sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk=" + }, + "builtin-status-codes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz", + "integrity": "sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug=" + }, + "call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "requires": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + } + }, + "cipher-base": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/cipher-base/-/cipher-base-1.0.4.tgz", + "integrity": "sha512-Kkht5ye6ZGmwv40uUDZztayT2ThLQGfnj/T71N/XzeZeo3nf8foyW7zGTsPYkEya3m5f3cAypH+qe7YOrM1U2Q==", + "requires": { + "inherits": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "console-browserify": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/console-browserify/-/console-browserify-1.2.0.tgz", + "integrity": "sha512-ZMkYO/LkF17QvCPqM0gxw8yUzigAOZOSWSHg91FH6orS7vcEj5dVZTidN2fQ14yBSdg97RqhSNwLUXInd52OTA==" + }, + "constants-browserify": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/constants-browserify/-/constants-browserify-1.0.0.tgz", + "integrity": "sha1-wguW2MYXdIqvHBYCF2DNJ/y4y3U=" + }, + "create-ecdh": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.4.tgz", + "integrity": "sha512-mf+TCx8wWc9VpuxfP2ht0iSISLZnt0JgWlrOKZiNqyUZWnjIaCIVNQArMHnCZKfEYRg6IM7A+NeJoN8gf/Ws0A==", + "requires": { + "bn.js": "^4.1.0", + "elliptic": "^6.5.3" + }, + "dependencies": { + "bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + } + } + }, + "create-hash": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", + "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==", + "requires": { + "cipher-base": "^1.0.1", + "inherits": "^2.0.1", + "md5.js": "^1.3.4", + "ripemd160": "^2.0.1", + "sha.js": "^2.4.0" + } + }, + "create-hmac": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", + "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==", + "requires": { + "cipher-base": "^1.0.3", + "create-hash": "^1.1.0", + "inherits": "^2.0.1", + "ripemd160": "^2.0.0", + "safe-buffer": "^5.0.1", + "sha.js": "^2.4.8" + } + }, + "crypto-browserify": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-3.12.0.tgz", + "integrity": "sha512-fz4spIh+znjO2VjL+IdhEpRJ3YN6sMzITSBijk6FK2UvTqruSQW+/cCZTSNsMiZNvUeq0CqurF+dAbyiGOY6Wg==", + "requires": { + "browserify-cipher": "^1.0.0", + "browserify-sign": "^4.0.0", + "create-ecdh": "^4.0.0", + "create-hash": "^1.1.0", + "create-hmac": "^1.1.0", + "diffie-hellman": "^5.0.0", + "inherits": "^2.0.1", + "pbkdf2": "^3.0.3", + "public-encrypt": "^4.0.0", + "randombytes": "^2.0.0", + "randomfill": "^1.0.3" + } + }, + "define-properties": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", + "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", + "requires": { + "object-keys": "^1.0.12" + } + }, + "des.js": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/des.js/-/des.js-1.0.1.tgz", + "integrity": "sha512-Q0I4pfFrv2VPd34/vfLrFOoRmlYj3OV50i7fskps1jZWK1kApMWWT9G6RRUeYedLcBDIhnSDaUvJMb3AhUlaEA==", + "requires": { + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0" + } + }, + "diffie-hellman": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", + "integrity": "sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==", + "requires": { + "bn.js": "^4.1.0", + "miller-rabin": "^4.0.0", + "randombytes": "^2.0.0" + }, + "dependencies": { + "bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + } + } + }, + "domain-browser": { + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-4.22.0.tgz", + "integrity": "sha512-IGBwjF7tNk3cwypFNH/7bfzBcgSCbaMOD3GsaY1AU/JRrnHnYgEM0+9kQt52iZxjNsjBtJYtao146V+f8jFZNw==" + }, + "elliptic": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.4.tgz", + "integrity": "sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ==", + "requires": { + "bn.js": "^4.11.9", + "brorand": "^1.1.0", + "hash.js": "^1.0.0", + "hmac-drbg": "^1.0.1", + "inherits": "^2.0.4", + "minimalistic-assert": "^1.0.1", + "minimalistic-crypto-utils": "^1.0.1" + }, + "dependencies": { + "bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + } + } + }, + "es-abstract": { + "version": "1.18.3", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.18.3.tgz", + "integrity": "sha512-nQIr12dxV7SSxE6r6f1l3DtAeEYdsGpps13dR0TwJg1S8gyp4ZPgy3FZcHBgbiQqnoqSTb+oC+kO4UQ0C/J8vw==", + "requires": { + "call-bind": "^1.0.2", + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "get-intrinsic": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.2", + "is-callable": "^1.2.3", + "is-negative-zero": "^2.0.1", + "is-regex": "^1.1.3", + "is-string": "^1.0.6", + "object-inspect": "^1.10.3", + "object-keys": "^1.1.1", + "object.assign": "^4.1.2", + "string.prototype.trimend": "^1.0.4", + "string.prototype.trimstart": "^1.0.4", + "unbox-primitive": "^1.0.1" + } + }, + "es-to-primitive": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", + "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "requires": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + } + }, + "es6-object-assign": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/es6-object-assign/-/es6-object-assign-1.1.0.tgz", + "integrity": "sha1-wsNYJlYkfDnqEHyx5mUrb58kUjw=" + }, + "events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==" + }, + "evp_bytestokey": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz", + "integrity": "sha512-/f2Go4TognH/KvCISP7OUsHn85hT9nUkxxA9BEWxFn+Oj9o8ZNLm/40hdlgSLyuOimsrTKLUMEorQexp/aPQeA==", + "requires": { + "md5.js": "^1.3.4", + "safe-buffer": "^5.1.1" + } + }, + "foreach": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.5.tgz", + "integrity": "sha1-C+4AUBiusmDQo6865ljdATbsG5k=" + }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, + "get-intrinsic": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", + "integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==", + "requires": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1" + } + }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "requires": { + "function-bind": "^1.1.1" + } + }, + "has-bigints": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.1.tgz", + "integrity": "sha512-LSBS2LjbNBTf6287JEbEzvJgftkF5qFkmCo9hDRpAzKhUOlJ+hx8dd4USs00SgsUNwc4617J9ki5YtEClM2ffA==" + }, + "has-symbols": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz", + "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==" + }, + "hash-base": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-3.1.0.tgz", + "integrity": "sha512-1nmYp/rhMDiE7AYkDw+lLwlAzz0AntGIe51F3RfFfEqyQ3feY2eI/NcwC6umIQVOASPMsWJLJScWKSSvzL9IVA==", + "requires": { + "inherits": "^2.0.4", + "readable-stream": "^3.6.0", + "safe-buffer": "^5.2.0" + } + }, + "hash.js": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz", + "integrity": "sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA==", + "requires": { + "inherits": "^2.0.3", + "minimalistic-assert": "^1.0.1" + } + }, + "hmac-drbg": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", + "integrity": "sha1-0nRXAQJabHdabFRXk+1QL8DGSaE=", + "requires": { + "hash.js": "^1.0.3", + "minimalistic-assert": "^1.0.0", + "minimalistic-crypto-utils": "^1.0.1" + } + }, + "https-browserify": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz", + "integrity": "sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM=" + }, + "ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "is-arguments": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.0.tgz", + "integrity": "sha512-1Ij4lOMPl/xB5kBDn7I+b2ttPMKa8szhEIrXDuXQD/oe3HJLTLhqhgGspwgyGd6MOywBUqVvYicF72lkgDnIHg==", + "requires": { + "call-bind": "^1.0.0" + } + }, + "is-bigint": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.2.tgz", + "integrity": "sha512-0JV5+SOCQkIdzjBK9buARcV804Ddu7A0Qet6sHi3FimE9ne6m4BGQZfRn+NZiXbBk4F4XmHfDZIipLj9pX8dSA==" + }, + "is-boolean-object": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.1.tgz", + "integrity": "sha512-bXdQWkECBUIAcCkeH1unwJLIpZYaa5VvuygSyS/c2lf719mTKZDU5UdDRlpd01UjADgmW8RfqaP+mRaVPdr/Ng==", + "requires": { + "call-bind": "^1.0.2" + } + }, + "is-callable": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.3.tgz", + "integrity": "sha512-J1DcMe8UYTBSrKezuIUTUwjXsho29693unXM2YhJUTR2txK/eG47bvNa/wipPFmZFgr/N6f1GA66dv0mEyTIyQ==" + }, + "is-date-object": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.4.tgz", + "integrity": "sha512-/b4ZVsG7Z5XVtIxs/h9W8nvfLgSAyKYdtGWQLbqy6jA1icmgjf8WCoTKgeS4wy5tYaPePouzFMANbnj94c2Z+A==" + }, + "is-generator-function": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.9.tgz", + "integrity": "sha512-ZJ34p1uvIfptHCN7sFTjGibB9/oBg17sHqzDLfuwhvmN/qLVvIQXRQ8licZQ35WJ8KuEQt/etnnzQFI9C9Ue/A==" + }, + "is-nan": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/is-nan/-/is-nan-1.3.2.tgz", + "integrity": "sha512-E+zBKpQ2t6MEo1VsonYmluk9NxGrbzpeeLC2xIViuO2EjU2xsXsBPwTr3Ykv9l08UYEVEdWeRZNouaZqF6RN0w==", + "requires": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3" + } + }, + "is-negative-zero": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.1.tgz", + "integrity": "sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w==" + }, + "is-number-object": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.5.tgz", + "integrity": "sha512-RU0lI/n95pMoUKu9v1BZP5MBcZuNSVJkMkAG2dJqC4z2GlkGUNeH68SuHuBKBD/XFe+LHZ+f9BKkLET60Niedw==" + }, + "is-regex": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.3.tgz", + "integrity": "sha512-qSVXFz28HM7y+IWX6vLCsexdlvzT1PJNFSBuaQLQ5o0IEw8UDYW6/2+eCMVyIsbM8CNLX2a/QWmSpyxYEHY7CQ==", + "requires": { + "call-bind": "^1.0.2", + "has-symbols": "^1.0.2" + } + }, + "is-string": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.6.tgz", + "integrity": "sha512-2gdzbKUuqtQ3lYNrUTQYoClPhm7oQu4UdpSZMp1/DGgkHBT8E2Z1l0yMdb6D4zNAxwDiMv8MdulKROJGNl0Q0w==" + }, + "is-symbol": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz", + "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==", + "requires": { + "has-symbols": "^1.0.2" + } + }, + "is-typed-array": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.5.tgz", + "integrity": "sha512-S+GRDgJlR3PyEbsX/Fobd9cqpZBuvUS+8asRqYDMLCb2qMzt1oz5m5oxQCxOgUDxiWsOVNi4yaF+/uvdlHlYug==", + "requires": { + "available-typed-arrays": "^1.0.2", + "call-bind": "^1.0.2", + "es-abstract": "^1.18.0-next.2", + "foreach": "^2.0.5", + "has-symbols": "^1.0.1" + } + }, + "md5.js": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", + "integrity": "sha512-xitP+WxNPcTTOgnTJcrhM0xvdPepipPSf3I8EIpGKeFLjt3PlJLIDG3u8EX53ZIubkb+5U2+3rELYpEhHhzdkg==", + "requires": { + "hash-base": "^3.0.0", + "inherits": "^2.0.1", + "safe-buffer": "^5.1.2" + } + }, + "miller-rabin": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/miller-rabin/-/miller-rabin-4.0.1.tgz", + "integrity": "sha512-115fLhvZVqWwHPbClyntxEVfVDfl9DLLTuJvq3g2O/Oxi8AiNouAHvDSzHS0viUJc+V5vm3eq91Xwqn9dp4jRA==", + "requires": { + "bn.js": "^4.0.0", + "brorand": "^1.0.1" + }, + "dependencies": { + "bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + } + } + }, + "minimalistic-assert": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", + "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==" + }, + "minimalistic-crypto-utils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz", + "integrity": "sha1-9sAMHAsIIkblxNmd+4x8CDsrWCo=" + }, + "object-inspect": { + "version": "1.10.3", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.10.3.tgz", + "integrity": "sha512-e5mCJlSH7poANfC8z8S9s9S2IN5/4Zb3aZ33f5s8YqoazCFzNLloLU8r5VCG+G7WoqLvAAZoVMcy3tp/3X0Plw==" + }, + "object-is": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/object-is/-/object-is-1.1.5.tgz", + "integrity": "sha512-3cyDsyHgtmi7I7DfSSI2LDp6SK2lwvtbg0p0R1e0RvTqF5ceGx+K2dfSjm1bKDMVCFEDAQvy+o8c6a7VujOddw==", + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3" + } + }, + "object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" + }, + "object.assign": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.2.tgz", + "integrity": "sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==", + "requires": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3", + "has-symbols": "^1.0.1", + "object-keys": "^1.1.1" + } + }, + "os-browserify": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/os-browserify/-/os-browserify-0.3.0.tgz", + "integrity": "sha1-hUNzx/XCMVkU/Jv8a9gjj92h7Cc=" + }, + "pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==" + }, + "parse-asn1": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz", + "integrity": "sha512-RnZRo1EPU6JBnra2vGHj0yhp6ebyjBZpmUCLHWiFhxlzvBCCpAuZ7elsBp1PVAbQN0/04VD/19rfzlBSwLstMw==", + "requires": { + "asn1.js": "^5.2.0", + "browserify-aes": "^1.0.0", + "evp_bytestokey": "^1.0.0", + "pbkdf2": "^3.0.3", + "safe-buffer": "^5.1.1" + } + }, + "path-browserify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz", + "integrity": "sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==" + }, + "pbkdf2": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.2.tgz", + "integrity": "sha512-iuh7L6jA7JEGu2WxDwtQP1ddOpaJNC4KlDEFfdQajSGgGPNi4OyDc2R7QnbY2bR9QjBVGwgvTdNJZoE7RaxUMA==", + "requires": { + "create-hash": "^1.1.2", + "create-hmac": "^1.1.4", + "ripemd160": "^2.0.1", + "safe-buffer": "^5.0.1", + "sha.js": "^2.4.8" + } + }, + "process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha1-czIwDoQBYb2j5podHZGn1LwW8YI=" + }, + "public-encrypt": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/public-encrypt/-/public-encrypt-4.0.3.tgz", + "integrity": "sha512-zVpa8oKZSz5bTMTFClc1fQOnyyEzpl5ozpi1B5YcvBrdohMjH2rfsBtyXcuNuwjsDIXmBYlF2N5FlJYhR29t8Q==", + "requires": { + "bn.js": "^4.1.0", + "browserify-rsa": "^4.0.0", + "create-hash": "^1.1.0", + "parse-asn1": "^5.0.0", + "randombytes": "^2.0.1", + "safe-buffer": "^5.1.2" + }, + "dependencies": { + "bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + } + } + }, + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + }, + "querystring-es3": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/querystring-es3/-/querystring-es3-0.2.1.tgz", + "integrity": "sha1-nsYfeQSYdXB9aUFFlv2Qek1xHnM=" + }, + "randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "requires": { + "safe-buffer": "^5.1.0" + } + }, + "randomfill": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/randomfill/-/randomfill-1.0.4.tgz", + "integrity": "sha512-87lcbR8+MhcWcUiQ+9e+Rwx8MyR2P7qnt15ynUlbm3TU/fjbgz4GsvfSUDTemtCCtVCqb4ZcEFlyPNTh9bBTLw==", + "requires": { + "randombytes": "^2.0.5", + "safe-buffer": "^5.1.0" + } + }, + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "ripemd160": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/ripemd160/-/ripemd160-2.0.2.tgz", + "integrity": "sha512-ii4iagi25WusVoiC4B4lq7pbXfAp3D9v5CwfkY33vffw2+pkDjY1D8GaN7spsxvCSx8dkPqOZCEZyfxcmJG2IA==", + "requires": { + "hash-base": "^3.0.0", + "inherits": "^2.0.1" + } + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=" + }, + "sha.js": { + "version": "2.4.11", + "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", + "integrity": "sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==", + "requires": { + "inherits": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "stream-browserify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-3.0.0.tgz", + "integrity": "sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA==", + "requires": { + "inherits": "~2.0.4", + "readable-stream": "^3.5.0" + } + }, + "stream-http": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/stream-http/-/stream-http-3.2.0.tgz", + "integrity": "sha512-Oq1bLqisTyK3TSCXpPbT4sdeYNdmyZJv1LxpEm2vu1ZhK89kSE5YXwZc3cWk0MagGaKriBh9mCFbVGtO+vY29A==", + "requires": { + "builtin-status-codes": "^3.0.0", + "inherits": "^2.0.4", + "readable-stream": "^3.6.0", + "xtend": "^4.0.2" + } + }, + "string.prototype.trimend": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.4.tgz", + "integrity": "sha512-y9xCjw1P23Awk8EvTpcyL2NIr1j7wJ39f+k6lvRnSMz+mz9CGz9NYPelDk42kOz6+ql8xjfK8oYzy3jAP5QU5A==", + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3" + } + }, + "string.prototype.trimstart": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.4.tgz", + "integrity": "sha512-jh6e984OBfvxS50tdY2nRZnoC5/mLFKOREQfw8t5yytkoUsJRNxvI/E39qu1sD0OtWI3OC0XgKSmcWwziwYuZw==", + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3" + } + }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "requires": { + "safe-buffer": "~5.2.0" + } + }, + "timers-browserify": { + "version": "2.0.12", + "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.12.tgz", + "integrity": "sha512-9phl76Cqm6FhSX9Xe1ZUAMLtm1BLkKj2Qd5ApyWkXzsMRaA7dgr81kf4wJmQf/hAvg8EEyJxDo3du/0KlhPiKQ==", + "requires": { + "setimmediate": "^1.0.4" + } + }, + "tty-browserify": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.1.tgz", + "integrity": "sha512-C3TaO7K81YvjCgQH9Q1S3R3P3BtN3RIM8n+OvX4il1K1zgE8ZhI0op7kClgkxtutIE8hQrcrHBXvIheqKUUCxw==" + }, + "unbox-primitive": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.1.tgz", + "integrity": "sha512-tZU/3NqK3dA5gpE1KtyiJUrEB0lxnGkMFHptJ7q6ewdZ8s12QrODwNbhIJStmJkd1QDXa1NRA8aF2A1zk/Ypyw==", + "requires": { + "function-bind": "^1.1.1", + "has-bigints": "^1.0.1", + "has-symbols": "^1.0.2", + "which-boxed-primitive": "^1.0.2" + } + }, + "url": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/url/-/url-0.11.0.tgz", + "integrity": "sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + }, + "dependencies": { + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + } + } + }, + "util": { + "version": "0.12.4", + "resolved": "https://registry.npmjs.org/util/-/util-0.12.4.tgz", + "integrity": "sha512-bxZ9qtSlGUWSOy9Qa9Xgk11kSslpuZwaxCg4sNIDj6FLucDab2JxnHwyNTCpHMtK1MjoQiWQ6DiUMZYbSrO+Sw==", + "requires": { + "inherits": "^2.0.3", + "is-arguments": "^1.0.4", + "is-generator-function": "^1.0.7", + "is-typed-array": "^1.1.3", + "safe-buffer": "^5.1.2", + "which-typed-array": "^1.1.2" + } + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, + "vm-browserify": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-1.1.2.tgz", + "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==" + }, + "which-boxed-primitive": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz", + "integrity": "sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==", + "requires": { + "is-bigint": "^1.0.1", + "is-boolean-object": "^1.1.0", + "is-number-object": "^1.0.4", + "is-string": "^1.0.5", + "is-symbol": "^1.0.3" + } + }, + "which-typed-array": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.4.tgz", + "integrity": "sha512-49E0SpUe90cjpoc7BOJwyPHRqSAd12c10Qm2amdEZrJPCY2NDxaW01zHITrem+rnETY3dwrbH3UUrUwagfCYDA==", + "requires": { + "available-typed-arrays": "^1.0.2", + "call-bind": "^1.0.0", + "es-abstract": "^1.18.0-next.1", + "foreach": "^2.0.5", + "function-bind": "^1.1.1", + "has-symbols": "^1.0.1", + "is-typed-array": "^1.1.3" + } + }, + "xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + } + } + }, + "mysql2": { + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.7.0.tgz", + "integrity": "sha512-c45jA3Jc1X8yJKzrWu1GpplBKGwv/wIV6ITZTlCSY7npF2YfJR+6nMP5e+NTQhUeJPSyOQAbGDCGEHbAl8HN9w==", + "requires": { + "denque": "^2.1.0", + "generate-function": "^2.3.1", + "iconv-lite": "^0.6.3", + "long": "^5.2.1", + "lru-cache": "^8.0.0", + "named-placeholders": "^1.1.3", + "seq-queue": "^0.0.5", + "sqlstring": "^2.3.2" + } + }, + "named-placeholders": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.3.tgz", + "integrity": "sha512-eLoBxg6wE/rZkJPhU/xRX1WTpkFEwDJEN96oxFrTsqBdbT5ec295Q+CoHrL9IT0DipqKhmGcaZmwOt8OON5x1w==", + "requires": { + "lru-cache": "^7.14.1" + }, + "dependencies": { + "lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==" + } + } + }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==" + }, + "path-to-regexp": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.8.0.tgz", + "integrity": "sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==", + "requires": { + "isarray": "0.0.1" + } + }, + "prop-types": { + "version": "15.8.1", + "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", + "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==", + "requires": { + "loose-envify": "^1.4.0", + "object-assign": "^4.1.1", + "react-is": "^16.13.1" + } + }, + "prop-types-extra": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/prop-types-extra/-/prop-types-extra-1.1.1.tgz", + "integrity": "sha512-59+AHNnHYCdiC+vMwY52WmvP5dM3QLeoumYuEyceQDi9aEhtwN9zIQ2ZNo25sMyXnbh32h+P1ezDsUpUH3JAew==", + "requires": { + "react-is": "^16.3.2", + "warning": "^4.0.0" + } + }, + "react": { + "version": "17.0.2", + "resolved": "https://registry.npmjs.org/react/-/react-17.0.2.tgz", + "integrity": "sha512-gnhPt75i/dq/z3/6q/0asP78D0u592D5L1pd7M8P+dck6Fu/jJeL6iVVK23fptSUZj8Vjf++7wXA8UNclGQcbA==", + "requires": { + "loose-envify": "^1.1.0", + "object-assign": "^4.1.1" + } + }, + "react-bootstrap": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/react-bootstrap/-/react-bootstrap-2.9.0.tgz", + "integrity": "sha512-dGh6fGjqR9MBzPOp2KbXJznt1Zy6SWepXYUdxMT18Zu/wJ73HCU8JNZe9dfzjmVssZYsJH9N3HHE4wAtQvNz7g==", + "requires": { + "@babel/runtime": "^7.22.5", + "@restart/hooks": "^0.4.9", + "@restart/ui": "^1.6.6", + "@types/react-transition-group": "^4.4.6", + "classnames": "^2.3.2", + "dom-helpers": "^5.2.1", + "invariant": "^2.2.4", + "prop-types": "^15.8.1", + "prop-types-extra": "^1.1.0", + "react-transition-group": "^4.4.5", + "uncontrollable": "^7.2.1", + "warning": "^4.0.3" + } + }, + "react-datepicker": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/react-datepicker/-/react-datepicker-4.18.0.tgz", + "integrity": "sha512-0MYt3HmLbHVk1sw4v+RCbLAVg5TA3jWP7RyjZbo53PC+SEi+pjdgc92lB53ai/ENZaTOhbXmgni9GzvMrorMAw==", + "requires": { + "@popperjs/core": "^2.11.8", + "classnames": "^2.2.6", + "date-fns": "^2.30.0", + "prop-types": "^15.7.2", + "react-onclickoutside": "^6.13.0", + "react-popper": "^2.3.0" + } + }, + "react-dom": { + "version": "17.0.2", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-17.0.2.tgz", + "integrity": "sha512-s4h96KtLDUQlsENhMn1ar8t2bEa+q/YAtj8pPPdIjPDGBDIVNsrD9aXNWqspUe6AzKCIG0C1HZZLqLV7qpOBGA==", + "requires": { + "loose-envify": "^1.1.0", + "object-assign": "^4.1.1", + "scheduler": "^0.20.2" + } + }, + "react-fast-compare": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/react-fast-compare/-/react-fast-compare-3.2.2.tgz", + "integrity": "sha512-nsO+KSNgo1SbJqJEYRE9ERzo7YtYbou/OqjSQKxV7jcKox7+usiUVZOAC+XnDOABXggQTno0Y1CpVnuWEc1boQ==" + }, + "react-is": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==" + }, + "react-lifecycles-compat": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/react-lifecycles-compat/-/react-lifecycles-compat-3.0.4.tgz", + "integrity": "sha512-fBASbA6LnOU9dOU2eW7aQ8xmYBSXUIWr+UmF9b1efZBazGNO+rcXT/icdKnYm2pTwcRylVUYwW7H1PHfLekVzA==" + }, + "react-onclickoutside": { + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/react-onclickoutside/-/react-onclickoutside-6.13.0.tgz", + "integrity": "sha512-ty8So6tcUpIb+ZE+1HAhbLROvAIJYyJe/1vRrrcmW+jLsaM+/powDRqxzo6hSh9CuRZGSL1Q8mvcF5WRD93a0A==" + }, + "react-popper": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/react-popper/-/react-popper-2.3.0.tgz", + "integrity": "sha512-e1hj8lL3uM+sgSR4Lxzn5h1GxBlpa4CQz0XLF8kx4MDrDRWY0Ena4c97PUeSX9i5W3UAfDP0z0FXCTQkoXUl3Q==", + "requires": { + "react-fast-compare": "^3.0.1", + "warning": "^4.0.2" + } + }, + "react-router": { + "version": "5.3.4", + "resolved": "https://registry.npmjs.org/react-router/-/react-router-5.3.4.tgz", + "integrity": "sha512-Ys9K+ppnJah3QuaRiLxk+jDWOR1MekYQrlytiXxC1RyfbdsZkS5pvKAzCCr031xHixZwpnsYNT5xysdFHQaYsA==", + "requires": { + "@babel/runtime": "^7.12.13", + "history": "^4.9.0", + "hoist-non-react-statics": "^3.1.0", + "loose-envify": "^1.3.1", + "path-to-regexp": "^1.7.0", + "prop-types": "^15.6.2", + "react-is": "^16.6.0", + "tiny-invariant": "^1.0.2", + "tiny-warning": "^1.0.0" + } + }, + "react-router-dom": { + "version": "5.3.4", + "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-5.3.4.tgz", + "integrity": "sha512-m4EqFMHv/Ih4kpcBCONHbkT68KoAeHN4p3lAGoNryfHi0dMy0kCzEZakiKRsvg5wHZ/JLrLW8o8KomWiz/qbYQ==", + "requires": { + "@babel/runtime": "^7.12.13", + "history": "^4.9.0", + "loose-envify": "^1.3.1", + "prop-types": "^15.6.2", + "react-router": "5.3.4", + "tiny-invariant": "^1.0.2", + "tiny-warning": "^1.0.0" + } + }, + "react-transition-group": { + "version": "4.4.5", + "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.5.tgz", + "integrity": "sha512-pZcd1MCJoiKiBR2NRxeCRg13uCXbydPnmB4EOeRrY7480qNWO8IIgQG6zlDkm6uRMsURXPuKq0GWtiM59a5Q6g==", + "requires": { + "@babel/runtime": "^7.5.5", + "dom-helpers": "^5.0.1", + "loose-envify": "^1.4.0", + "prop-types": "^15.6.2" + } + }, + "react-visibility-sensor": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/react-visibility-sensor/-/react-visibility-sensor-5.1.1.tgz", + "integrity": "sha512-cTUHqIK+zDYpeK19rzW6zF9YfT4486TIgizZW53wEZ+/GPBbK7cNS0EHyJVyHYacwFEvvHLEKfgJndbemWhB/w==", + "requires": { + "prop-types": "^15.7.2" + } + }, + "regenerator-runtime": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.0.tgz", + "integrity": "sha512-srw17NI0TUWHuGa5CFGGmhfNIeja30WMBfbslPNhf6JrqQlLN5gcrvig1oqPxiVaXb0oW0XRKtH6Nngs5lKCIA==" + }, + "resolve-pathname": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/resolve-pathname/-/resolve-pathname-3.0.0.tgz", + "integrity": "sha512-C7rARubxI8bXFNB/hqcp/4iUeIXJhJZvFPFPiSPRnhU5UPxzMFIl+2E6yY6c4k9giDJAhtV+enfA+G89N6Csng==" + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "scheduler": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.20.2.tgz", + "integrity": "sha512-2eWfGgAqqWFGqtdMmcL5zCMK1U8KlXv8SQFGglL3CEtd0aDVDWgeF/YoCmvln55m5zSk3J/20hTaSBeSObsQDQ==", + "requires": { + "loose-envify": "^1.1.0", + "object-assign": "^4.1.1" + } + }, + "seq-queue": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/seq-queue/-/seq-queue-0.0.5.tgz", + "integrity": "sha512-hr3Wtp/GZIc/6DAGPDcV4/9WoZhjrkXsi5B/07QgX8tsdc6ilr7BFM6PM6rbdAX1kFSDYeZGLipIZZKyQP0O5Q==" + }, + "sqlstring": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.3.tgz", + "integrity": "sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg==" + }, + "tiny-invariant": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.1.tgz", + "integrity": "sha512-AD5ih2NlSssTCwsMznbvwMZpJ1cbhkGd2uueNxzv2jDlEeZdU04JQfRnggJQ8DrcVBGjAsCKwFBbDlVNtEMlzw==" + }, + "tiny-warning": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/tiny-warning/-/tiny-warning-1.0.3.tgz", + "integrity": "sha512-lBN9zLN/oAf68o3zNXYrdCt1kP8WsiGW8Oo2ka41b2IM5JL/S1CTyX1rW0mb/zSuJun0ZUrDxx4sqvYS2FWzPA==" + }, + "tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" + }, + "uncontrollable": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/uncontrollable/-/uncontrollable-7.2.1.tgz", + "integrity": "sha512-svtcfoTADIB0nT9nltgjujTi7BzVmwjZClOmskKu/E8FW9BXzg9os8OLr4f8Dlnk0rYWJIWr4wv9eKUXiQvQwQ==", + "requires": { + "@babel/runtime": "^7.6.3", + "@types/react": ">=16.9.11", + "invariant": "^2.2.4", + "react-lifecycles-compat": "^3.0.4" + } + }, + "uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==" + }, + "value-equal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/value-equal/-/value-equal-1.0.1.tgz", + "integrity": "sha512-NOJ6JZCAWr0zlxZt+xqCHNTEKOsrks2HQd4MqhP1qy4z1SkbEP467eNx6TgDKXMvUOb+OENfJCZwM+16n7fRfw==" + }, + "warning": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/warning/-/warning-4.0.3.tgz", + "integrity": "sha512-rpJyN222KWIvHJ/F53XSZv0Zl/accqHR8et1kpaMTD/fLCRxtV8iX8czMzY7sVZupTI3zcUTg8eycS2kNF9l6w==", + "requires": { + "loose-envify": "^1.0.0" + } + } + } +} diff --git a/components/webui/package.json b/components/webui/package.json new file mode 100644 index 000000000..9b291b572 --- /dev/null +++ b/components/webui/package.json @@ -0,0 +1,39 @@ +{ + "name": "webui", + "private": true, + "scripts": { + "start": "meteor run", + "test": "meteor test --once --driver-package meteortesting:mocha", + "test-app": "TEST_WATCH=1 meteor test --full-app --driver-package meteortesting:mocha", + "visualize": "meteor --production --extra-packages bundle-visualizer" + }, + "dependencies": { + "@babel/runtime": "^7.15.4", + "@fortawesome/fontawesome-svg-core": "^6.4.2", + "@fortawesome/free-solid-svg-icons": "^6.4.2", + "@fortawesome/react-fontawesome": "^0.2.0", + "@msgpack/msgpack": "^3.0.0-beta2", + "bootstrap": "^5.3.2", + "highcharts": "^11.1.0", + "highcharts-custom-events": "^3.0.10", + "highcharts-react-official": "^3.2.1", + "luxon": "^3.4.3", + "meteor-node-stubs": "^1.1.0", + "mysql2": "^3.7.0", + "react": "^17.0.2", + "react-bootstrap": "^2.9.0", + "react-datepicker": "^4.18.0", + "react-dom": "^17.0.2", + "react-router": "^5.3.4", + "react-router-dom": "^5.3.4", + "react-visibility-sensor": "^5.1.1", + "uuid": "^9.0.1" + }, + "meteor": { + "mainModule": { + "client": "client/main.jsx", + "server": "server/main.js" + }, + "testModule": "tests/main.js" + } +} diff --git a/components/webui/server/main.js b/components/webui/server/main.js new file mode 100644 index 000000000..0be97a0bb --- /dev/null +++ b/components/webui/server/main.js @@ -0,0 +1,18 @@ +import {Meteor} from "meteor/meteor"; + +import "/imports/api/search/server/methods"; +import "/imports/api/search/server/publications"; +import "/imports/api/user/server/methods"; + +import {initSQL, deinitSQL} from "../imports/api/search/sql"; +import {initSearchEventCollection} from "../imports/api/search/collections"; + +Meteor.startup(async () => { + await initSQL() + initSearchEventCollection() +}); + +process.on('exit', async (code) => { + console.log(`Node.js is about to exit with code: ${code}`); + await deinitSQL() +}); diff --git a/components/webui/settings.json b/components/webui/settings.json new file mode 100644 index 000000000..500e657ec --- /dev/null +++ b/components/webui/settings.json @@ -0,0 +1,6 @@ +{ + "public": { + "SearchResultsCollectionName": "results", + "SearchResultsMetadataCollectionName": "results-metadata" + } +} diff --git a/components/webui/tests/main.js b/components/webui/tests/main.js new file mode 100644 index 000000000..7d6ad28a5 --- /dev/null +++ b/components/webui/tests/main.js @@ -0,0 +1,20 @@ +import assert from "assert"; + +describe("webui", function () { + it("package.json has correct name", async function () { + const { name } = await import("../package.json"); + assert.strictEqual(name, "webui"); + }); + + if (Meteor.isClient) { + it("client is not server", function () { + assert.strictEqual(Meteor.isServer, false); + }); + } + + if (Meteor.isServer) { + it("server is not client", function () { + assert.strictEqual(Meteor.isClient, false); + }); + } +}); From 84bb3724bda73f26adaf4099474df298f892e1ed Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 26 Jan 2024 12:51:34 -0500 Subject: [PATCH 02/60] Add docs. --- .../webui/imports/api/search/collections.js | 29 ++++-- .../webui/imports/api/search/constants.js | 16 +++- .../imports/api/search/server/methods.js | 62 +++++++++++-- .../imports/api/search/server/publications.js | 18 ++++ components/webui/imports/api/search/sql.js | 17 ++++ .../webui/imports/api/user/client/methods.js | 90 +++++++++++-------- .../webui/imports/api/user/server/methods.js | 6 ++ components/webui/imports/ui/App.jsx | 2 +- .../imports/ui/SearchView/SearchControls.js | 51 +++++++++-- .../imports/ui/SearchView/SearchResults.jsx | 22 ++++- .../ui/SearchView/SearchResultsHeader.jsx | 11 +++ .../ui/SearchView/SearchResultsTable.jsx | 15 ++++ .../imports/ui/SearchView/SearchView.jsx | 23 +++-- .../webui/imports/ui/Sidebar/Sidebar.jsx | 12 ++- .../ui/constants/LOCAL_STORAGE_KEYS.js | 6 ++ 15 files changed, 312 insertions(+), 68 deletions(-) diff --git a/components/webui/imports/api/search/collections.js b/components/webui/imports/api/search/collections.js index 52e2df43a..901d937d3 100644 --- a/components/webui/imports/api/search/collections.js +++ b/components/webui/imports/api/search/collections.js @@ -1,8 +1,15 @@ import {Mongo} from "meteor/mongo"; import {INVALID_JOB_ID, SearchSignal} from "./constants"; - +/** + * A MongoDB collection for storing metadata about search results. + * + * @constant + */ export const SearchResultsMetadataCollection = new Mongo.Collection(Meteor.settings.public.SearchResultsMetadataCollectionName); +/** + * Initializes the search event collection by inserting a default document if the collection is empty. + */ export const initSearchEventCollection = () => { // create the collection if not exists if (SearchResultsMetadataCollection.find().count() === 0) { @@ -15,11 +22,23 @@ export const initSearchEventCollection = () => { } } -// this should only be accessed by the server; client should use a React referenced-object -export const MY_MONGO_DB = {} +/** + * Object to store references to MongoDB collections. + * This should only be accessed by the server; clients should use a React referenced-object. + * + * @constant + * @type {Object} + */export const MY_MONGO_DB = {} -// Retrieves Mongo Collection object by name if it has been created -// as Meteor.js does not permit creating more than one Collection object with a same name. +/** + * Retrieves a Mongo Collection object by name, creating it if it does not already exist. + * This is to adhere to Meteor.js's restriction against creating more than one Collection object + * with the same name. + * + * @param {Object} dbRef database object where collections are stored + * @param {string} name of the collection to retrieve or create + * @returns {Mongo.Collection} + */ export const getCollection = (dbRef, name) => { if (dbRef[name] === undefined) { dbRef[name] = new Mongo.Collection(name); diff --git a/components/webui/imports/api/search/constants.js b/components/webui/imports/api/search/constants.js index 050bfa3f6..d8128d480 100644 --- a/components/webui/imports/api/search/constants.js +++ b/components/webui/imports/api/search/constants.js @@ -1,4 +1,13 @@ let enumSearchSignal; +/** + * Enum of search-related signals. + * + * This includes request and response signals for various search operations and their respective + * states. + * + * @constant + * @type {Object} + */ export const SearchSignal = Object.freeze({ NONE: (enumSearchSignal=0), REQ_MASK: (enumSearchSignal = 0x10000000), @@ -13,8 +22,13 @@ export const SearchSignal = Object.freeze({ export const isSearchSignalReq = (e) => (0 !== (SearchSignal.REQ_MASK & e)); export const isSearchSignalRsp = (e) => (0 !== (SearchSignal.RSP_MASK & e)); -// below should match `job_orchestration.search_scheduler.common`: class JobStatus let enumJobStatus; +/** + * Enum of job statuses, matching the `JobStatus` class in `job_orchestration.search_scheduler.common`. + * + * @constant + * @type {Object} + */ export const JobStatus = Object.freeze({ PENDING: (enumJobStatus=0), RUNNING: ++enumJobStatus, diff --git a/components/webui/imports/api/search/server/methods.js b/components/webui/imports/api/search/server/methods.js index 02df7b5b3..8933df0e5 100644 --- a/components/webui/imports/api/search/server/methods.js +++ b/components/webui/imports/api/search/server/methods.js @@ -8,9 +8,21 @@ import {getCollection, MY_MONGO_DB, SearchResultsMetadataCollection} from "../co const SEARCH_JOBS_TABLE_NAME = "distributed_search_jobs"; -const sleep = (s) => new Promise(r => setTimeout(r, s * 1000)); - - +/** + * Creates a promise that resolves after a specified number of seconds. + * + * @param {number} seconds to wait before resolving the promise + * @returns {Promise} that resolves after the specified delay + */ +const sleep = (seconds) => new Promise(r => setTimeout(r, seconds * 1000)); + + +/** + * Submits a query job to the SQL database and returns the job ID. + * + * @param {Object} args containing the search configuration. + * @returns {number|null} job ID on successful submission, or null in case of failure + */ const submitQuery = async (args) => { let jobId = null; @@ -25,7 +37,14 @@ const submitQuery = async (args) => { return jobId; }; - +/** + * Waits for a job to finish and retrieves its status from the database. + * + * @param {number} jobId of the job to monitor + * + * @returns {?string} null if the job completes successfully; an error message if the job exits + * in an unexpected status or encounters an error during monitoring + */ const waitTillJobFinishes = async (jobId) => { let errorMsg = null; @@ -58,13 +77,22 @@ const waitTillJobFinishes = async (jobId) => { return errorMsg; }; +/** + * Cancels a job by updating its status to 'CANCELLING' in the database. + * + * @param {number} jobId of the job to be cancelled + */ const cancelQuery = async (jobId) => { const [rows, _] = await SQL_CONNECTION.query(`UPDATE ${SEARCH_JOBS_TABLE_NAME} SET status = ${JobStatus.CANCELLING} WHERE id = (?)`, [jobId]); }; - +/** + * Updates the search event when the specified job finishes. + * + * @param {number} jobId of the job to monitor + */ const updateSearchEventWhenJobFinishes = async (jobId) => { const errorMsg = await waitTillJobFinishes(jobId); const filter = { @@ -81,6 +109,11 @@ const updateSearchEventWhenJobFinishes = async (jobId) => { SearchResultsMetadataCollection.update(filter, modifier); }; +/** + * Creates MongoDB indexes for a specific job's collection. + * + * @param {?number} jobId used to identify the Mongo Collection to add indexes + */ const createMongoIndexes = async (jobId) => { const timestampAscendingIndex = { key: {timestamp: 1, _id: 1}, @@ -100,6 +133,15 @@ const createMongoIndexes = async (jobId) => { }; Meteor.methods({ + /** + * Submits a search query and initiates the search process. + * + * @param {string} queryString + * @param {number} timestampBegin + * @param {number} timestampEnd + * + * @returns {Object} containing {jobId} of the submitted search job + */ async "search.submitQuery"({ queryString, timestampBegin, @@ -132,6 +174,11 @@ Meteor.methods({ return {jobId}; }, + /** + * Clears the results of a search operation identified by jobId. + * + * @param {string} jobId of the search results to clear + */ async "search.clearResults"({ jobId }) @@ -144,6 +191,11 @@ Meteor.methods({ delete MY_MONGO_DB[jobId.toString()]; }, + /** + * Cancels an ongoing search operation identified by jobId. + * + * @param {string} jobId of the search operation to cancel + */ async "search.cancelOperation"({ jobId }) { diff --git a/components/webui/imports/api/search/server/publications.js b/components/webui/imports/api/search/server/publications.js index 162ef5cd6..d71564483 100644 --- a/components/webui/imports/api/search/server/publications.js +++ b/components/webui/imports/api/search/server/publications.js @@ -2,6 +2,14 @@ import {Meteor} from "meteor/meteor"; import {SearchResultsMetadataCollection, getCollection, MY_MONGO_DB} from "../collections"; +/** + * Publishes search results metadata for a specific job. + * + * @param {string} publicationName + * @param {string} jobId of the search operation + * + * @returns {Mongo.Cursor} cursor that provides access to the search results metadata + */ Meteor.publish(Meteor.settings.public.SearchResultsMetadataCollectionName, ({jobId}) => { const filter = { _id: jobId.toString() @@ -10,6 +18,16 @@ Meteor.publish(Meteor.settings.public.SearchResultsMetadataCollectionName, ({job return SearchResultsMetadataCollection.find(filter); }); +/** + * Publishes search results for a specific job with optional sorting and result limit. + * + * @param {string} publicationName + * @param {string} jobId of the search operation + * @param {Object} [fieldToSortBy] used for sorting results + * @param {number} visibleSearchResultsLimit limit of visible search results + * + * @returns {Mongo.Cursor} cursor that provides access to the search results + */ Meteor.publish(Meteor.settings.public.SearchResultsCollectionName, ({ jobId, fieldToSortBy, visibleSearchResultsLimit }) => { diff --git a/components/webui/imports/api/search/sql.js b/components/webui/imports/api/search/sql.js index a000a2a02..19ca7a2b7 100644 --- a/components/webui/imports/api/search/sql.js +++ b/components/webui/imports/api/search/sql.js @@ -3,6 +3,19 @@ import mysql from "mysql2/promise"; // SQL connection for submitting queries to the backend export let SQL_CONNECTION = null; +/** + * Initializes the SQL database connection using environment variables. + * + * This function sets up a connection to a SQL database using environment variables. + * It requires the following environment variables to be defined: + * - CLP_DB_HOST: The host address of the SQL database. + * - CLP_DB_PORT: The port number for the SQL database. + * - CLP_DB_NAME: The name of the SQL database. + * - CLP_DB_USER: The username for authenticating with the SQL database. + * - CLP_DB_PASS: The password for authenticating with the SQL database. + * + * @throws {Error} Throws an error and exits the process if any required environment variables are undefined. + */ export const initSQL = async () => { const CLP_DB_HOST = process.env['CLP_DB_HOST']; const CLP_DB_PORT = process.env['CLP_DB_PORT']; @@ -23,6 +36,10 @@ export const initSQL = async () => { }); await SQL_CONNECTION.connect(); } + +/** + * Deinitializes the SQL database connection if it is initialized. + */ export const deinitSQL = async () => { if (null !== SQL_CONNECTION) { await SQL_CONNECTION.end(); diff --git a/components/webui/imports/api/user/client/methods.js b/components/webui/imports/api/user/client/methods.js index 9d4fc611b..b4fe34d26 100644 --- a/components/webui/imports/api/user/client/methods.js +++ b/components/webui/imports/api/user/client/methods.js @@ -2,54 +2,70 @@ import {Meteor} from "meteor/meteor"; import {v4 as uuidv4} from "uuid"; // TODO: implement a full-fledged registration sys -const LOCAL_STORAGE_KEY_USERNAME = 'username'; -const DUMMY_PASSWORD = 'DummyPassword'; +const LOCAL_STORAGE_KEY_USERNAME = "username"; +const DUMMY_PASSWORD = "DummyPassword"; let LoginRetryCount = 0; const CONST_MAX_LOGIN_RETRY = 3; +/** + * Registers a user with a provided username and then attempts to log in with that username. + * + * @param {string} username to register and log in with + * + * @returns {Promise} true if the registration and login are successful + * false if there's an error during registration or login + */ export const registerAndLoginWithUsername = async (username) => { - try { - await new Promise((resolve, reject) => { - Meteor.call('user.create', {username, password: DUMMY_PASSWORD}, (error) => { - if (error) { - console.log('create user error', error); - reject(error); - } else { - localStorage.setItem(LOCAL_STORAGE_KEY_USERNAME, username); - resolve(true); - } - }); + return new Promise((resolve) => { + Meteor.call("user.create", { + username, + password: DUMMY_PASSWORD, + }, (error) => { + if (error) { + console.log("create user error", error); + resolve(false); + } else { + localStorage.setItem(LOCAL_STORAGE_KEY_USERNAME, username); + resolve(true); + } }); - - return await loginWithUsername(username); - } catch (error) { - return false; - } + }); }; -export const loginWithUsername = async (username) => { - try { - return await new Promise((resolve, reject) => { - Meteor.loginWithPassword(username, DUMMY_PASSWORD, (error) => { - if (!error) { - resolve(true); +/** + * Attempts to log in a user with the provided username using a dummy password. + * + * @param {string} username to register and log in with + * + * @returns {Promise} true if the login is successful + * false if there's an error during login or if the maximum login + * retries are reached + */ +export const loginWithUsername = (username) => { + return new Promise((resolve) => { + Meteor.loginWithPassword(username, DUMMY_PASSWORD, (error) => { + if (!error) { + resolve(true); + } else { + console.log("login error", error, "LOGIN_RETRY_COUNT:", LoginRetryCount); + if (LoginRetryCount < CONST_MAX_LOGIN_RETRY) { + LoginRetryCount++; + resolve(registerAndLoginWithUsername(username)); } else { - console.log('login error', error, 'LOGIN_RETRY_COUNT:', LoginRetryCount); - if (LoginRetryCount < CONST_MAX_LOGIN_RETRY) { - LoginRetryCount++; - resolve(registerAndLoginWithUsername(username)); - } else { - reject(error); - } + resolve(false); } - }); + } }); - } catch (error) { - return false; - } + }); }; +/** + * Attempts to log in a user using a stored username or register a new one if none is found. + * + * @returns {boolean} true if the login is successful + * false if there's an error during login or registration + */ export const login = async () => { let username = localStorage.getItem(LOCAL_STORAGE_KEY_USERNAME); let result; @@ -61,5 +77,5 @@ export const login = async () => { result = await loginWithUsername(username); } - return result -} + return result; +}; diff --git a/components/webui/imports/api/user/server/methods.js b/components/webui/imports/api/user/server/methods.js index 9a429a256..c360ff1e2 100644 --- a/components/webui/imports/api/user/server/methods.js +++ b/components/webui/imports/api/user/server/methods.js @@ -2,6 +2,12 @@ import {Meteor} from "meteor/meteor"; import {Accounts} from 'meteor/accounts-base'; Meteor.methods({ + /** + * Creates a user account with a provided username and password. + * + * @param {string} username for the new user + * @param {string} password for the new user + */ 'user.create'({username, password}) { Accounts.createUser({username, password}); } diff --git a/components/webui/imports/ui/App.jsx b/components/webui/imports/ui/App.jsx index dae67f769..d349d1f82 100644 --- a/components/webui/imports/ui/App.jsx +++ b/components/webui/imports/ui/App.jsx @@ -43,9 +43,9 @@ export const App = () => { return (
diff --git a/components/webui/imports/ui/SearchView/SearchControls.js b/components/webui/imports/ui/SearchView/SearchControls.js index d0d377469..5726669e2 100644 --- a/components/webui/imports/ui/SearchView/SearchControls.js +++ b/components/webui/imports/ui/SearchView/SearchControls.js @@ -10,7 +10,12 @@ import {cTimePresets} from "./datetime"; import LOCAL_STORAGE_KEYS from "../constants/LOCAL_STORAGE_KEYS"; import {isSearchSignalReq, SearchSignal} from "../../api/search/constants"; - +/** + * Renders a date picker control for selecting date and time. + * + * @param {Object} props to be passed to the DatePicker component + * @returns {JSX.Element} + */ const SearchControlsDatePicker = (props) => ( () +/** + * Renders a label for a search filter control. + * + * @param {Object} props to be passed to the Form.Label component + * @returns {JSX.Element} + */ const SearchControlsFilterLabel = (props) => ( () +/** + * Renders the controls for filtering search results by time range, including a date picker and + * preset time range options. + * + * @param {Object} timeRange for filtering. + * @param {function} setTimeRange callback to set timeRange + * @returns {JSX.Element} + */ const SearchFilterControlsDrawer = ({ timeRange, setTimeRange }) => { @@ -70,7 +89,9 @@ const SearchFilterControlsDrawer = ({ // Compute range of end timestamp so that it's after the begin timestamp let timestampEndMin = null; let timestampEndMax = null; - if (timeRange.begin.getFullYear() === timeRange.end.getFullYear() && timeRange.begin.getMonth() === timeRange.end.getMonth() && timeRange.begin.getDate() === timeRange.end.getDate()) { + if (timeRange.begin.getFullYear() === timeRange.end.getFullYear() && + timeRange.begin.getMonth() === timeRange.end.getMonth() && + timeRange.begin.getDate() === timeRange.end.getDate()) { timestampEndMin = new Date(timeRange.begin); // TODO This doesn't handle leap seconds timestampEndMax = new Date(timeRange.end).setHours(23, 59, 59, 999); @@ -121,15 +142,29 @@ const SearchFilterControlsDrawer = ({
); } +/** + * Renders the search controls including query input, filter drawer toggle, and operation buttons + * like submit, clear, and cancel. It also manages the state of the drawer. + * + * @param {string} queryString for matching logs + * @param {function} setQueryString callback to set queryString + * @param {Object} timeRange for filtering + * @param {function} setTimeRange callback to set timeRange + * @param {Object} resultsMetadata which includes last request / response signal + * @param {function} onSubmitQuery callback to submit the search query + * @param {function} onClearResults callback to clear search results + * @param {function} onCancelOperation callback to cancel the ongoing search operation + * @returns {JSX.Element} + */ export const SearchControls = ({ queryString, setQueryString, timeRange, setTimeRange, resultsMetadata, - submitQuery, - handleClearResults, - cancelOperation, + onSubmitQuery, + onClearResults, + onCancelOperation, }) => { const [drawerOpen, setDrawerOpen] = useState("true" === localStorage.getItem(LOCAL_STORAGE_KEYS.SEARCH_CONTROLS_VISIBLE)); const [canceling, setCanceling] = useState(false); @@ -150,12 +185,12 @@ export const SearchControls = ({ e.preventDefault(); setCanceling(false); - submitQuery(); + onSubmitQuery(); } const handleCancelOperation = () => { setCanceling(true); - cancelOperation(); + onCancelOperation(); } return <> @@ -186,7 +221,7 @@ export const SearchControls = ({ (SearchSignal.RSP_DONE === resultsMetadata["lastSignal"]) && + {(0 < numResultsOnServer) ? : <>} From d89c727791249ec2416a21be3937c9685a06383e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 27 Jan 2024 05:49:54 -0500 Subject: [PATCH 05/60] Add log to file support with daily rotation. --- .../imports/api/search/server/methods.js | 21 +- .../imports/api/search/server/publications.js | 20 +- components/webui/imports/api/search/sql.js | 38 ++- .../webui/imports/api/user/server/methods.js | 18 +- components/webui/imports/utils/logger.js | 109 ++++++++ components/webui/package-lock.json | 244 ++++++++++++++++++ components/webui/package.json | 5 +- components/webui/server/main.js | 50 +++- 8 files changed, 458 insertions(+), 47 deletions(-) create mode 100644 components/webui/imports/utils/logger.js diff --git a/components/webui/imports/api/search/server/methods.js b/components/webui/imports/api/search/server/methods.js index 8933df0e5..3c0b27250 100644 --- a/components/webui/imports/api/search/server/methods.js +++ b/components/webui/imports/api/search/server/methods.js @@ -1,5 +1,6 @@ import {Meteor} from "meteor/meteor"; import msgpack from "@msgpack/msgpack"; +import {logger} from "/imports/utils/logger"; import {JOB_STATUS_WAITING_STATES, JobStatus, SearchSignal} from "../constants"; import {SQL_CONNECTION} from "../sql"; @@ -31,7 +32,7 @@ const submitQuery = async (args) => { VALUES (?) `, [Buffer.from(msgpack.encode(args))]); jobId = queryInsertResults["insertId"]; } catch (e) { - console.error("Unable to submit query job to SQL DB", e); + logger.error("Unable to submit query job to SQL DB", e); } return jobId; @@ -55,14 +56,10 @@ const waitTillJobFinishes = async (jobId) => { where id = ${jobId}`); const status = rows[0]["status"]; if (!JOB_STATUS_WAITING_STATES.includes(status)) { - console.debug(`Job ${jobId} exited with status = ${status}`); + logger.info(`Job ${jobId} exited with status = ${status}`); if (JobStatus.SUCCESS !== status) { - if (JobStatus.NO_MATCHING_ARCHIVE === status) { - errorMsg = "No matching archive by query string and time range"; - } else { - errorMsg = `Job exited in an unexpected status=${status}: ${Object.keys(JobStatus)[status]}`; - } + errorMsg = `Job exited in an unexpected status=${status}: ${Object.keys(JobStatus)[status]}`; } break; } @@ -71,7 +68,7 @@ const waitTillJobFinishes = async (jobId) => { } } catch (e) { errorMsg = `Error querying job status, jobId=${jobId}: ${e}`; - console.error(errorMsg); + logger.error(errorMsg); } return errorMsg; @@ -80,7 +77,7 @@ const waitTillJobFinishes = async (jobId) => { /** * Cancels a job by updating its status to 'CANCELLING' in the database. * - * @param {number} jobId of the job to be cancelled + * @param {string} jobId of the job to be cancelled */ const cancelQuery = async (jobId) => { const [rows, _] = await SQL_CONNECTION.query(`UPDATE ${SEARCH_JOBS_TABLE_NAME} @@ -106,6 +103,7 @@ const updateSearchEventWhenJobFinishes = async (jobId) => { } }; + logger.debug("modifier = ", modifier) SearchResultsMetadataCollection.update(filter, modifier); }; @@ -154,6 +152,7 @@ Meteor.methods({ begin_timestamp: timestampBegin, end_timestamp: timestampEnd, }; + logger.info("search.submitQuery args =", args) jobId = await submitQuery(args); if (null !== jobId) { @@ -183,7 +182,7 @@ Meteor.methods({ jobId }) { - console.debug(`Got request to clear results. jobid = ${jobId}`); + logger.info("search.clearResults jobId =", jobId) const resultsCollection = getCollection(MY_MONGO_DB, jobId.toString()); await resultsCollection.dropCollectionAsync(); @@ -199,6 +198,8 @@ Meteor.methods({ async "search.cancelOperation"({ jobId }) { + logger.info("search.cancelOperation jobId =", jobId) + await cancelQuery(jobId); }, }); diff --git a/components/webui/imports/api/search/server/publications.js b/components/webui/imports/api/search/server/publications.js index d71564483..e25e34ca5 100644 --- a/components/webui/imports/api/search/server/publications.js +++ b/components/webui/imports/api/search/server/publications.js @@ -1,4 +1,5 @@ import {Meteor} from "meteor/meteor"; +import {logger} from "/imports/utils/logger"; import {SearchResultsMetadataCollection, getCollection, MY_MONGO_DB} from "../collections"; @@ -11,9 +12,11 @@ import {SearchResultsMetadataCollection, getCollection, MY_MONGO_DB} from "../co * @returns {Mongo.Cursor} cursor that provides access to the search results metadata */ Meteor.publish(Meteor.settings.public.SearchResultsMetadataCollectionName, ({jobId}) => { + logger.debug(`Subscription '${Meteor.settings.public.SearchResultsMetadataCollectionName}'`, + `jobId=${jobId}`); const filter = { - _id: jobId.toString() - } + _id: jobId.toString(), + }; return SearchResultsMetadataCollection.find(filter); }); @@ -29,14 +32,19 @@ Meteor.publish(Meteor.settings.public.SearchResultsMetadataCollectionName, ({job * @returns {Mongo.Cursor} cursor that provides access to the search results */ Meteor.publish(Meteor.settings.public.SearchResultsCollectionName, ({ - jobId, fieldToSortBy, visibleSearchResultsLimit - }) => { - const collection = getCollection(MY_MONGO_DB, jobId.toString()) + jobId, + fieldToSortBy, + visibleSearchResultsLimit, +}) => { + logger.debug(`Subscription '${Meteor.settings.public.SearchResultsCollectionName}'`, + `jobId=${jobId}, fieldToSortBy=${fieldToSortBy}, ` + + `visibleSearchResultsLimit=${visibleSearchResultsLimit}`); + const collection = getCollection(MY_MONGO_DB, jobId.toString()); const findOptions = { limit: visibleSearchResultsLimit, disableOplog: true, - pollingIntervalMs: 250 + pollingIntervalMs: 250, }; if (fieldToSortBy) { const sort = {}; diff --git a/components/webui/imports/api/search/sql.js b/components/webui/imports/api/search/sql.js index 19ca7a2b7..95a806dad 100644 --- a/components/webui/imports/api/search/sql.js +++ b/components/webui/imports/api/search/sql.js @@ -1,4 +1,5 @@ import mysql from "mysql2/promise"; +import {logger} from "/imports/utils/logger"; // SQL connection for submitting queries to the backend export let SQL_CONNECTION = null; @@ -16,33 +17,28 @@ export let SQL_CONNECTION = null; * * @throws {Error} Throws an error and exits the process if any required environment variables are undefined. */ -export const initSQL = async () => { - const CLP_DB_HOST = process.env['CLP_DB_HOST']; - const CLP_DB_PORT = process.env['CLP_DB_PORT']; - const CLP_DB_NAME = process.env['CLP_DB_NAME']; - const CLP_DB_USER = process.env['CLP_DB_USER']; - const CLP_DB_PASS = process.env['CLP_DB_PASS']; - if ([CLP_DB_HOST, CLP_DB_PORT, CLP_DB_NAME, CLP_DB_USER, CLP_DB_PASS].includes(undefined)) { - console.error("Environment variables CLP_DB_URL, CLP_DB_USER and CLP_DB_PASS need to be defined"); - process.exit(1); +export const initSQL = async (host, port, database, user, password) => { + try { + SQL_CONNECTION = await mysql.createConnection({ + host: host, + port: port, + database: database, + user: user, + password: password, + }); + await SQL_CONNECTION.connect(); + } catch (e) { + logger.error(`Unable to create MySQL / mariadb connection with ` + + `host=${host}, port=${port}, database=${database}, user=${user}`); } - - SQL_CONNECTION = await mysql.createConnection({ - host: CLP_DB_HOST, - port: parseInt(CLP_DB_PORT), - database: CLP_DB_NAME, - user: CLP_DB_USER, - password: CLP_DB_PASS, - }); - await SQL_CONNECTION.connect(); -} +}; /** - * Deinitializes the SQL database connection if it is initialized. + * De-initializes the SQL database connection if it is initialized. */ export const deinitSQL = async () => { if (null !== SQL_CONNECTION) { await SQL_CONNECTION.end(); SQL_CONNECTION = null; } -} +}; diff --git a/components/webui/imports/api/user/server/methods.js b/components/webui/imports/api/user/server/methods.js index c360ff1e2..60518e430 100644 --- a/components/webui/imports/api/user/server/methods.js +++ b/components/webui/imports/api/user/server/methods.js @@ -1,5 +1,6 @@ import {Meteor} from "meteor/meteor"; -import {Accounts} from 'meteor/accounts-base'; +import {Accounts} from "meteor/accounts-base"; +import {logger} from "/imports/utils/logger"; Meteor.methods({ /** @@ -8,7 +9,14 @@ Meteor.methods({ * @param {string} username for the new user * @param {string} password for the new user */ - 'user.create'({username, password}) { - Accounts.createUser({username, password}); - } -}) + "user.create"({ + username, + password, + }) { + logger.info("user.create", `username=${username}`); + Accounts.createUser({ + username, + password, + }); + }, +}); diff --git a/components/webui/imports/utils/logger.js b/components/webui/imports/utils/logger.js new file mode 100644 index 000000000..3bff5adca --- /dev/null +++ b/components/webui/imports/utils/logger.js @@ -0,0 +1,109 @@ +import winston from "winston"; +import "winston-daily-rotate-file"; +import JSON5 from "json5"; + +const MAX_LOGS_FILE_SIZE = "100m"; +const MAX_LOGS_RETENTION_DAYS = "30d"; + +const clpLoggingLevelToWinstonMap = { + "DEBUG": "debug", + "INFO": "info", + "WARN": "warn", + "WARNING": "warn", + "ERROR": "error", + "CRITICAL": "error", +}; + +const getStackInfo = () => { + let info = null; + + const stackList = (new Error()).stack.split("\n"); + const stackInfo = stackList[4]; + const stackRegex = /at\s+(.*)\s+\((.*):(\d+):(\d+)\)/i; + const stackMatch = stackRegex.exec(stackInfo); + + if (null !== stackMatch && stackMatch.length === 5) { + info = { + method: stackMatch[1], + filePath: stackMatch[2], + line: stackMatch[3], + }; + } else { + const stackRegex2 = /at\s+(.*):(\d*):(\d*)/i; + const stackMatch2 = stackRegex2.exec(stackInfo); + info = { + method: "", + filePath: stackMatch2[1], + line: stackMatch2[2], + }; + } + + return info; +}; + + +let winstonLogger = null; +const fileLineFuncLog = (level, ...args) => { + const stackInfo = getStackInfo(); + if (null !== stackInfo) { + const logMessage = `[${stackInfo.filePath}:${stackInfo.line}] ` + + `${args.map(a => ("string" === typeof a) ? a : JSON5.stringify(a)).join(" ")}`; + winstonLogger.log({ + level, + message: logMessage, + label: stackInfo.method, + }); + } else { + winstonLogger.log({ + level, + message, + label: "", + }); + } +}; + +export let logger = Object.freeze({ + error: (...args) => (fileLineFuncLog("error", ...args)), + warn: (...args) => (fileLineFuncLog("warn", ...args)), + help: (...args) => (fileLineFuncLog("help", ...args)), + data: (...args) => (fileLineFuncLog("data", ...args)), + info: (...args) => (fileLineFuncLog("info", ...args)), + debug: (...args) => (fileLineFuncLog("debug", ...args)), + prompt: (...args) => (fileLineFuncLog("prompt", ...args)), + verbose: (...args) => (fileLineFuncLog("verbose", ...args)), + input: (...args) => (fileLineFuncLog("input", ...args)), + silly: (...args) => (fileLineFuncLog("silly", ...args)), +}); + +export const initLogger = (logsDir, clpLoggingLevel, isTraceEnabled = false) => { + winstonLogger = winston.createLogger({ + level: clpLoggingLevelToWinstonMap[clpLoggingLevel], + format: winston.format.combine( + winston.format.timestamp(), + winston.format.printf((info) => { + return JSON.stringify({ + timestamp: info.timestamp, + level: info.level, + label: info.label, + message: info.message, + }); + }), + ), + transports: [ + new winston.transports.Console(), + new winston.transports.DailyRotateFile({ + filename: "webui-%DATE%.log", + dirname: logsDir, + datePattern: "YYYY-MM-DD-HH", + maxSize: MAX_LOGS_FILE_SIZE, + maxFiles: MAX_LOGS_RETENTION_DAYS, + }), + ], + }); + + if (false === isTraceEnabled) { + logger = winstonLogger; + } + + logger.info("logger has been initialized"); +}; \ No newline at end of file diff --git a/components/webui/package-lock.json b/components/webui/package-lock.json index d7915ab68..a37f54c2b 100644 --- a/components/webui/package-lock.json +++ b/components/webui/package-lock.json @@ -11,6 +11,21 @@ "regenerator-runtime": "^0.14.0" } }, + "@colors/colors": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", + "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==" + }, + "@dabh/diagnostics": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.3.tgz", + "integrity": "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==", + "requires": { + "colorspace": "1.1.x", + "enabled": "2.0.x", + "kuler": "^2.0.0" + } + }, "@fortawesome/fontawesome-common-types": { "version": "6.4.2", "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-common-types/-/fontawesome-common-types-6.4.2.tgz", @@ -125,11 +140,21 @@ "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.4.tgz", "integrity": "sha512-2L9ifAGl7wmXwP4v3pN4p2FLhD0O1qsJpvKmNin5VA8+UvNVb447UDaAEV6UdrkA+m/Xs58U1RFps44x6TFsVQ==" }, + "@types/triple-beam": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.5.tgz", + "integrity": "sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==" + }, "@types/warning": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/@types/warning/-/warning-3.0.1.tgz", "integrity": "sha512-ywJmriP+nvjBKNBEMaNZgj2irZHoxcKeYcyMLbqhYKbDVn8yCIULy2Ol/tvIb37O3IBeZj3RU4tXqQTtGwoAMg==" }, + "async": { + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.5.tgz", + "integrity": "sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==" + }, "bootstrap": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/bootstrap/-/bootstrap-5.3.2.tgz", @@ -140,6 +165,46 @@ "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.3.2.tgz", "integrity": "sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw==" }, + "color": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz", + "integrity": "sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==", + "requires": { + "color-convert": "^1.9.3", + "color-string": "^1.6.0" + } + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" + }, + "color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "requires": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, + "colorspace": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.4.tgz", + "integrity": "sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==", + "requires": { + "color": "^3.1.3", + "text-hex": "1.0.x" + } + }, "csstype": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.2.tgz", @@ -172,6 +237,29 @@ "csstype": "^3.0.2" } }, + "enabled": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", + "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==" + }, + "fecha": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", + "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==" + }, + "file-stream-rotator": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/file-stream-rotator/-/file-stream-rotator-0.6.1.tgz", + "integrity": "sha512-u+dBid4PvZw17PmDeRcNOtCP9CCK/9lRN2w+r1xIS7yOL9JFrIBKTvrYsxT4P0pGtThYTn++QS5ChHaUov3+zQ==", + "requires": { + "moment": "^2.29.1" + } + }, + "fn.name": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", + "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" + }, "generate-function": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.3.1.tgz", @@ -227,6 +315,11 @@ "safer-buffer": ">= 2.1.2 < 3.0.0" } }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, "invariant": { "version": "2.2.4", "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz", @@ -235,11 +328,21 @@ "loose-envify": "^1.0.0" } }, + "is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" + }, "is-property": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", "integrity": "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g==" }, + "is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==" + }, "isarray": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", @@ -250,6 +353,29 @@ "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" }, + "json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==" + }, + "kuler": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", + "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==" + }, + "logform": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/logform/-/logform-2.6.0.tgz", + "integrity": "sha512-1ulHeNPp6k/LD8H91o7VYFBng5i1BDE7HoKxVbZiGFidS1Rj65qcywLxX+pVfAPoQJEjRdvKcusKwOupHCVOVQ==", + "requires": { + "@colors/colors": "1.6.0", + "@types/triple-beam": "^1.3.2", + "fecha": "^4.2.0", + "ms": "^2.1.1", + "safe-stable-stringify": "^2.3.1", + "triple-beam": "^1.3.0" + } + }, "long": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", @@ -1156,6 +1282,16 @@ } } }, + "moment": { + "version": "2.30.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz", + "integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==" + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, "mysql2": { "version": "3.7.0", "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.7.0.tgz", @@ -1191,6 +1327,19 @@ "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==" }, + "object-hash": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-2.2.0.tgz", + "integrity": "sha512-gScRMn0bS5fH+IuwyIFgnh9zBdo4DV+6GhygmWM9HyNJSgS0hScp1f5vjtm7oIIOiT9trXrShAkLFSc2IqKNgw==" + }, + "one-time": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", + "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", + "requires": { + "fn.name": "1.x.x" + } + }, "path-to-regexp": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.8.0.tgz", @@ -1347,6 +1496,16 @@ "prop-types": "^15.7.2" } }, + "readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, "regenerator-runtime": { "version": "0.14.0", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.0.tgz", @@ -1357,6 +1516,16 @@ "resolved": "https://registry.npmjs.org/resolve-pathname/-/resolve-pathname-3.0.0.tgz", "integrity": "sha512-C7rARubxI8bXFNB/hqcp/4iUeIXJhJZvFPFPiSPRnhU5UPxzMFIl+2E6yY6c4k9giDJAhtV+enfA+G89N6Csng==" }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "safe-stable-stringify": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz", + "integrity": "sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==" + }, "safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -1376,11 +1545,37 @@ "resolved": "https://registry.npmjs.org/seq-queue/-/seq-queue-0.0.5.tgz", "integrity": "sha512-hr3Wtp/GZIc/6DAGPDcV4/9WoZhjrkXsi5B/07QgX8tsdc6ilr7BFM6PM6rbdAX1kFSDYeZGLipIZZKyQP0O5Q==" }, + "simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", + "requires": { + "is-arrayish": "^0.3.1" + } + }, "sqlstring": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.3.tgz", "integrity": "sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg==" }, + "stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==" + }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "requires": { + "safe-buffer": "~5.2.0" + } + }, + "text-hex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", + "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==" + }, "tiny-invariant": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.1.tgz", @@ -1391,6 +1586,11 @@ "resolved": "https://registry.npmjs.org/tiny-warning/-/tiny-warning-1.0.3.tgz", "integrity": "sha512-lBN9zLN/oAf68o3zNXYrdCt1kP8WsiGW8Oo2ka41b2IM5JL/S1CTyX1rW0mb/zSuJun0ZUrDxx4sqvYS2FWzPA==" }, + "triple-beam": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.4.1.tgz", + "integrity": "sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==" + }, "tslib": { "version": "2.6.2", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", @@ -1407,6 +1607,11 @@ "react-lifecycles-compat": "^3.0.4" } }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, "uuid": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", @@ -1424,6 +1629,45 @@ "requires": { "loose-envify": "^1.0.0" } + }, + "winston": { + "version": "3.11.0", + "resolved": "https://registry.npmjs.org/winston/-/winston-3.11.0.tgz", + "integrity": "sha512-L3yR6/MzZAOl0DsysUXHVjOwv8mKZ71TrA/41EIduGpOOV5LQVodqN+QdQ6BS6PJ/RdIshZhq84P/fStEZkk7g==", + "requires": { + "@colors/colors": "^1.6.0", + "@dabh/diagnostics": "^2.0.2", + "async": "^3.2.3", + "is-stream": "^2.0.0", + "logform": "^2.4.0", + "one-time": "^1.0.0", + "readable-stream": "^3.4.0", + "safe-stable-stringify": "^2.3.1", + "stack-trace": "0.0.x", + "triple-beam": "^1.3.0", + "winston-transport": "^4.5.0" + } + }, + "winston-daily-rotate-file": { + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/winston-daily-rotate-file/-/winston-daily-rotate-file-4.7.1.tgz", + "integrity": "sha512-7LGPiYGBPNyGHLn9z33i96zx/bd71pjBn9tqQzO3I4Tayv94WPmBNwKC7CO1wPHdP9uvu+Md/1nr6VSH9h0iaA==", + "requires": { + "file-stream-rotator": "^0.6.1", + "object-hash": "^2.0.1", + "triple-beam": "^1.3.0", + "winston-transport": "^4.4.0" + } + }, + "winston-transport": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.6.0.tgz", + "integrity": "sha512-wbBA9PbPAHxKiygo7ub7BYRiKxms0tpfU2ljtWzb3SjRjv5yl6Ozuy/TkXf00HTAt+Uylo3gSkNwzc4ME0wiIg==", + "requires": { + "logform": "^2.3.2", + "readable-stream": "^3.6.0", + "triple-beam": "^1.3.0" + } } } } diff --git a/components/webui/package.json b/components/webui/package.json index 9b291b572..a8376416f 100644 --- a/components/webui/package.json +++ b/components/webui/package.json @@ -17,6 +17,7 @@ "highcharts": "^11.1.0", "highcharts-custom-events": "^3.0.10", "highcharts-react-official": "^3.2.1", + "json5": "^2.2.3", "luxon": "^3.4.3", "meteor-node-stubs": "^1.1.0", "mysql2": "^3.7.0", @@ -27,7 +28,9 @@ "react-router": "^5.3.4", "react-router-dom": "^5.3.4", "react-visibility-sensor": "^5.1.1", - "uuid": "^9.0.1" + "uuid": "^9.0.1", + "winston": "^3.11.0", + "winston-daily-rotate-file": "^4.7.1" }, "meteor": { "mainModule": { diff --git a/components/webui/server/main.js b/components/webui/server/main.js index 0be97a0bb..c5dd939e8 100644 --- a/components/webui/server/main.js +++ b/components/webui/server/main.js @@ -6,13 +6,55 @@ import "/imports/api/user/server/methods"; import {initSQL, deinitSQL} from "../imports/api/search/sql"; import {initSearchEventCollection} from "../imports/api/search/collections"; +import {initLogger} from "../imports/utils/logger"; + +const DEFAULT_LOGS_DIR = "."; +const DEFAULT_LOGGING_LEVEL = Meteor.isDevelopment ? "DEBUG" : "INFO"; + +function parseArgs() { + const CLP_DB_HOST = process.env["CLP_DB_HOST"]; + const CLP_DB_PORT = process.env["CLP_DB_PORT"]; + const CLP_DB_NAME = process.env["CLP_DB_NAME"]; + const CLP_DB_USER = process.env["CLP_DB_USER"]; + const CLP_DB_PASS = process.env["CLP_DB_PASS"]; + + if ([CLP_DB_HOST, CLP_DB_PORT, CLP_DB_NAME, CLP_DB_USER, CLP_DB_PASS].includes(undefined)) { + console.error( + "Environment variables CLP_DB_URL, CLP_DB_USER and CLP_DB_PASS need to be defined"); + process.exit(1); + } + + const CLP_LOGS_DIR = process.env["CLP_LOGS_DIR"] || DEFAULT_LOGS_DIR; + const CLP_LOGGING_LEVEL = process.env["CLP_LOGGING_LEVEL"] || DEFAULT_LOGGING_LEVEL; + + return { + CLP_DB_HOST, + CLP_DB_PORT, + CLP_DB_NAME, + CLP_DB_USER, + CLP_DB_PASS, + CLP_LOGS_DIR, + CLP_LOGGING_LEVEL, + }; +} Meteor.startup(async () => { - await initSQL() - initSearchEventCollection() + const args = parseArgs(); + + initLogger(args.CLP_LOGS_DIR, args.CLP_LOGGING_LEVEL, Meteor.isDevelopment); + + await initSQL( + args.CLP_DB_HOST, + parseInt(args.CLP_DB_PORT), + args.CLP_DB_NAME, + args.CLP_DB_USER, + args.CLP_DB_PASS, + ); + + initSearchEventCollection(); }); -process.on('exit', async (code) => { +process.on("exit", async (code) => { console.log(`Node.js is about to exit with code: ${code}`); - await deinitSQL() + await deinitSQL(); }); From af4572ba2b6e6fdb3b8142e72f7a05f06558b9ef Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 27 Jan 2024 07:07:43 -0500 Subject: [PATCH 06/60] Support WebUI logging configuration in CLP package. --- .../clp_package_utils/general.py | 24 +++++++++---------- .../clp_package_utils/scripts/start_clp.py | 13 ++++++++-- .../clp-py-utils/clp_py_utils/clp_config.py | 21 ++++++++++++++++ .../package-template/src/etc/clp-config.yml | 1 + components/webui/imports/utils/logger.js | 19 ++++++++------- components/webui/server/main.js | 12 +++++----- 6 files changed, 61 insertions(+), 29 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index 77331db82..82a1e8ce8 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -42,18 +42,12 @@ class DockerMountType(enum.IntEnum): class DockerMount: def __init__(self, type: DockerMountType, src: pathlib.Path, dst: pathlib.Path, is_read_only: bool = False): self.__type = type - self._src = src - self._dst = dst + self.__src = src + self.__dst = dst self.__is_read_only = is_read_only - def get_src(self): - return self._src - - def get_dst(self): - return self._dst - def __str__(self): - mount_str = f"type={DOCKER_MOUNT_TYPE_STRINGS[self.__type]},src={self._src},dst={self._dst}" + mount_str = f"type={DOCKER_MOUNT_TYPE_STRINGS[self.__type]},src={self.__src},dst={self.__dst}" if self.__is_read_only: mount_str += ",readonly" return mount_str @@ -288,8 +282,14 @@ def validate_worker_config(clp_config: CLPConfig): clp_config.validate_archive_output_dir() -def validate_webui_config(clp_config: CLPConfig): - component_name = WEBUI_COMPONENT_NAME +def validate_webui_config(clp_config: CLPConfig, logs_dir: pathlib.Path, settings_json_path: pathlib.Path): + if not settings_json_path.exists(): + raise ValueError(f"{WEBUI_COMPONENT_NAME} {settings_json_path} is not a valid path to Meteor settings.json") + + try: + validate_path_could_be_dir(logs_dir) + except ValueError as ex: + raise ValueError(f"{WEBUI_COMPONENT_NAME} logs directory is invalid: {ex}") - validate_port(f"{component_name}.port", clp_config.webui.host, + validate_port(f"{WEBUI_COMPONENT_NAME}.port", clp_config.webui.host, clp_config.webui.port) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 6ea26eb1a..239404532 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -555,9 +555,15 @@ def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts logger.info(f"{WEBUI_COMPONENT_NAME} already running.") return - validate_webui_config(clp_config) + webui_logs_dir = clp_config.logs_directory / WEBUI_COMPONENT_NAME + settings_json_path = get_clp_home() / 'var' / 'www' / 'settings.json' - settings_json_path = str(mounts.clp_home.get_src() / 'var' / 'www' / 'settings.json') + validate_webui_config(clp_config, webui_logs_dir, settings_json_path) + + # Create directories + webui_logs_dir.mkdir(exist_ok=True, parents=True) + + container_webui_logs_dir = pathlib.Path('/') / 'var' / 'log' / WEBUI_COMPONENT_NAME with open(settings_json_path, 'r') as settings_json_file: settings_json_content = settings_json_file.read() meteor_settings = json.loads(settings_json_content) @@ -578,10 +584,13 @@ def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts '-e', f'CLP_DB_NAME={clp_config.database.name}', '-e', f'CLP_DB_USER={clp_config.database.username}', '-e', f'CLP_DB_PASS={clp_config.database.password}', + '-e', f'WEBUI_LOGS_DIR={container_webui_logs_dir}', + '-e', f'WEBUI_LOGGING_LEVEL={clp_config.webui.logging_level}', '-u', f'{os.getuid()}:{os.getgid()}', ] necessary_mounts = [ mounts.clp_home, + DockerMount(DockerMountType.BIND, webui_logs_dir, container_webui_logs_dir), ] for mount in necessary_mounts: if mount: diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index bb930e8e2..8cb76bc5d 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -206,6 +206,27 @@ def dump_to_primitive_dict(self): class WebUi(BaseModel): host: str = 'localhost' port: int = 4000 + logging_level: str = 'INFO' + + @validator('host') + def validate_host(cls, field): + if '' == field: + raise ValueError(f'{WEBUI_COMPONENT_NAME}.host cannot be empty.') + return field + + @validator('port') + def validate_port(cls, field): + min_valid_port = 0 + max_valid_port = 2 ** 16 - 1 + if min_valid_port > field or max_valid_port < field: + raise ValueError(f'{WEBUI_COMPONENT_NAME}.port is not within valid range ' + f'{min_valid_port}-{max_valid_port}.') + return field + + @validator('logging_level') + def validate_logging_level(cls, field): + _validate_logging_level(cls, field) + return field class CLPConfig(BaseModel): diff --git a/components/package-template/src/etc/clp-config.yml b/components/package-template/src/etc/clp-config.yml index 68803588a..85a66f0dd 100644 --- a/components/package-template/src/etc/clp-config.yml +++ b/components/package-template/src/etc/clp-config.yml @@ -28,6 +28,7 @@ #webui: # host: "localhost" # port: 4000 +# logging_level: "INFO" # #search_scheduler: # jobs_poll_delay: 0.1 # seconds diff --git a/components/webui/imports/utils/logger.js b/components/webui/imports/utils/logger.js index 3bff5adca..ae5750f65 100644 --- a/components/webui/imports/utils/logger.js +++ b/components/webui/imports/utils/logger.js @@ -5,13 +5,14 @@ import JSON5 from "json5"; const MAX_LOGS_FILE_SIZE = "100m"; const MAX_LOGS_RETENTION_DAYS = "30d"; -const clpLoggingLevelToWinstonMap = { - "DEBUG": "debug", - "INFO": "info", - "WARN": "warn", - "WARNING": "warn", - "ERROR": "error", - "CRITICAL": "error", +// attribute names should match clp_py_utils.clp_logging.LOGGING_LEVEL_MAPPING +const webuiLoggingLevelToWinstonMap = { + DEBUG: "debug", + INFO: "info", + WARN: "warn", + WARNING: "warn", + ERROR: "error", + CRITICAL: "error", }; const getStackInfo = () => { @@ -75,9 +76,9 @@ export let logger = Object.freeze({ silly: (...args) => (fileLineFuncLog("silly", ...args)), }); -export const initLogger = (logsDir, clpLoggingLevel, isTraceEnabled = false) => { +export const initLogger = (logsDir, webuiLoggingLevel, isTraceEnabled = false) => { winstonLogger = winston.createLogger({ - level: clpLoggingLevelToWinstonMap[clpLoggingLevel], + level: webuiLoggingLevelToWinstonMap[webuiLoggingLevel], format: winston.format.combine( winston.format.timestamp(), winston.format.printf((info) => { diff --git a/components/webui/server/main.js b/components/webui/server/main.js index c5dd939e8..41699190b 100644 --- a/components/webui/server/main.js +++ b/components/webui/server/main.js @@ -9,7 +9,7 @@ import {initSearchEventCollection} from "../imports/api/search/collections"; import {initLogger} from "../imports/utils/logger"; const DEFAULT_LOGS_DIR = "."; -const DEFAULT_LOGGING_LEVEL = Meteor.isDevelopment ? "DEBUG" : "INFO"; +const DEFAULT_LOGGING_LEVEL = Meteor.isDevelopment ? "debug" : "info"; function parseArgs() { const CLP_DB_HOST = process.env["CLP_DB_HOST"]; @@ -24,8 +24,8 @@ function parseArgs() { process.exit(1); } - const CLP_LOGS_DIR = process.env["CLP_LOGS_DIR"] || DEFAULT_LOGS_DIR; - const CLP_LOGGING_LEVEL = process.env["CLP_LOGGING_LEVEL"] || DEFAULT_LOGGING_LEVEL; + const WEBUI_LOGS_DIR = process.env["WEBUI_LOGS_DIR"] || DEFAULT_LOGS_DIR; + const WEBUI_LOGGING_LEVEL = process.env["WEBUI_LOGGING_LEVEL"] || DEFAULT_LOGGING_LEVEL; return { CLP_DB_HOST, @@ -33,15 +33,15 @@ function parseArgs() { CLP_DB_NAME, CLP_DB_USER, CLP_DB_PASS, - CLP_LOGS_DIR, - CLP_LOGGING_LEVEL, + WEBUI_LOGS_DIR, + WEBUI_LOGGING_LEVEL, }; } Meteor.startup(async () => { const args = parseArgs(); - initLogger(args.CLP_LOGS_DIR, args.CLP_LOGGING_LEVEL, Meteor.isDevelopment); + initLogger(args.WEBUI_LOGS_DIR, args.WEBUI_LOGGING_LEVEL, Meteor.isDevelopment); await initSQL( args.CLP_DB_HOST, From 2364bcedd691e69dede8788f9c32558804e12d58 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 28 Jan 2024 05:55:11 -0500 Subject: [PATCH 07/60] Add WebUI launcher to redirect stderr of Meteor server to logs directory. --- Taskfile.yml | 3 +- .../clp_package_utils/scripts/start_clp.py | 3 + components/webui/launcher.js | 84 +++++++++++++++++++ 3 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 components/webui/launcher.js diff --git a/Taskfile.yml b/Taskfile.yml index ede927919..e3bebf552 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -124,9 +124,10 @@ tasks: - >- rsync -a "{{.WEBUI_BUILD_DIR}}/bundle/" + "$PWD/settings.json" + "$PWD/launcher.js" "{{.WEBUI_BUILD_DIR}}/" - "rm -rf '{{.WEBUI_BUILD_DIR}}/bundle/'" - - "cp $PWD/settings.json {{.WEBUI_BUILD_DIR}}/" - |- cd {{.WEBUI_BUILD_DIR}}/programs/server PATH={{.NODEJS_BIN_DIR}}:$PATH $(readlink -f {{.NODEJS_BIN_DIR}}/npm) install diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 239404532..9a7f19821 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -556,6 +556,7 @@ def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts return webui_logs_dir = clp_config.logs_directory / WEBUI_COMPONENT_NAME + node_path = str(CONTAINER_CLP_HOME / 'var' / 'www' / 'programs'/'server'/'npm'/'node_modules') settings_json_path = get_clp_home() / 'var' / 'www' / 'settings.json' validate_webui_config(clp_config, webui_logs_dir, settings_json_path) @@ -575,6 +576,7 @@ def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts '--network', 'host', '--rm', '--name', container_name, + '-e', f"NODE_PATH={node_path}", '-e', f'MONGO_URL={clp_config.results_cache.get_uri()}', '-e', f'PORT={clp_config.webui.port}', '-e', f'ROOT_URL=http://{clp_config.webui.host}', @@ -600,6 +602,7 @@ def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts node_cmd = [ str(CONTAINER_CLP_HOME / 'bin' / 'node'), + str(CONTAINER_CLP_HOME / 'var' / 'www' / 'launcher.js'), str(CONTAINER_CLP_HOME / 'var' / 'www' / 'main.js') ] cmd = container_cmd + node_cmd diff --git a/components/webui/launcher.js b/components/webui/launcher.js new file mode 100644 index 000000000..ac7d8a9b5 --- /dev/null +++ b/components/webui/launcher.js @@ -0,0 +1,84 @@ +/* Production launcher for CLP WebUI, which redirects Meteor server stderr to rotated error logs + * files in a specified directory for error monitoring. + + * To avoid duplicated installations of dependencies, use the same `node_modules` for the server + * by setting envvar NODE_PATH="./programs/server/npm/node_modules", assumning this script is + * placed under the same directory where bundled `main.js` locates. + * + * This is not intended for development use. For development, please refer to README.md in the + * component root for launching a development server with Meteor-specific error messages print + * to the console. + * + * ENVIRONMENT VARIABLES: + * - NODE_PATH: path to node_modules including "winston" and "winston-daily-rotate-file" + * - WEBUI_LOGS_DIR: path to error logs directory + * SCRIPT USAGE: + * - usage: node /path/to/launcher.js /path/to/main.js + * + */ + +const {spawn} = require("child_process"); +const winston = require("winston"); +require("winston-daily-rotate-file"); + + +const DEFAULT_LOGS_DIR = "."; + +const MAX_LOGS_FILE_SIZE = "100m"; +const MAX_LOGS_RETENTION_DAYS = "30d"; + +const getLogger = (logsDir) => { + return winston.createLogger({ + format: winston.format.combine( + winston.format.timestamp(), + winston.format.printf((info) => { + return JSON.stringify({ + timestamp: info.timestamp, + level: info.level, + label: info.label, + message: info.message, + }); + }), + ), + transports: [ + new winston.transports.DailyRotateFile({ + filename: "webui_error-%DATE%.log", + dirname: logsDir, + datePattern: "YYYY-MM-DD-HH", + maxSize: MAX_LOGS_FILE_SIZE, + maxFiles: MAX_LOGS_RETENTION_DAYS, + }), + ], + }); +}; + +const runScript = (logsDir, scriptPath) => { + const logger = getLogger(logsDir); + const script = spawn(process.argv0, [scriptPath]); + + script.stderr.on("data", (data) => { + logger.error(data.toString()); + }); + + script.on("close", (code) => { + console.log(`Child process exited with code ${code}`); + }); +}; + +const parseArgs = () => { + const WEBUI_LOGS_DIR = process.env["WEBUI_LOGS_DIR"] || DEFAULT_LOGS_DIR; + const scriptPath = process.argv[2]; + + return { + WEBUI_LOGS_DIR, + scriptPath, + }; +}; + +const main = () => { + const args = parseArgs(); + + runScript(args.WEBUI_LOGS_DIR, args.scriptPath); +}; + +main(); \ No newline at end of file From 3b888533314df54fd10da9f05baab792ddc85043 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 28 Jan 2024 06:49:31 -0500 Subject: [PATCH 08/60] In production mode also override winston logging function to allow logging objects. --- components/webui/imports/utils/logger.js | 41 ++++++++++++------------ 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/components/webui/imports/utils/logger.js b/components/webui/imports/utils/logger.js index ae5750f65..544e6de18 100644 --- a/components/webui/imports/utils/logger.js +++ b/components/webui/imports/utils/logger.js @@ -5,6 +5,8 @@ import JSON5 from "json5"; const MAX_LOGS_FILE_SIZE = "100m"; const MAX_LOGS_RETENTION_DAYS = "30d"; +let isTraceEnabled = false; + // attribute names should match clp_py_utils.clp_logging.LOGGING_LEVEL_MAPPING const webuiLoggingLevelToWinstonMap = { DEBUG: "debug", @@ -45,22 +47,23 @@ const getStackInfo = () => { let winstonLogger = null; const fileLineFuncLog = (level, ...args) => { - const stackInfo = getStackInfo(); - if (null !== stackInfo) { - const logMessage = `[${stackInfo.filePath}:${stackInfo.line}] ` + - `${args.map(a => ("string" === typeof a) ? a : JSON5.stringify(a)).join(" ")}`; - winstonLogger.log({ - level, - message: logMessage, - label: stackInfo.method, - }); - } else { - winstonLogger.log({ - level, - message, - label: "", - }); + let logMessage = `${args.map(a => ("string" === typeof a) ? a : JSON5.stringify(a)).join(" ")}`; + let logLabel = ""; + + if (true === isTraceEnabled) { + const stackInfo = getStackInfo(); + + if (null !== stackInfo) { + logMessage = `[${stackInfo.filePath}:${stackInfo.line}] ` + logMessage; + logLabel = stackInfo.method; + } } + + winstonLogger.log({ + level, + message: logMessage, + label: logLabel, + }); }; export let logger = Object.freeze({ @@ -76,7 +79,9 @@ export let logger = Object.freeze({ silly: (...args) => (fileLineFuncLog("silly", ...args)), }); -export const initLogger = (logsDir, webuiLoggingLevel, isTraceEnabled = false) => { +export const initLogger = (logsDir, webuiLoggingLevel, _isTraceEnabled = false) => { + isTraceEnabled = _isTraceEnabled; + winstonLogger = winston.createLogger({ level: webuiLoggingLevelToWinstonMap[webuiLoggingLevel], format: winston.format.combine( @@ -102,9 +107,5 @@ export const initLogger = (logsDir, webuiLoggingLevel, isTraceEnabled = false) = ], }); - if (false === isTraceEnabled) { - logger = winstonLogger; - } - logger.info("logger has been initialized"); }; \ No newline at end of file From d41a4ebbfbe045279a4164318c704edc06f41eca Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 28 Jan 2024 06:58:32 -0500 Subject: [PATCH 09/60] Add docs for logging utilities. --- components/webui/imports/utils/logger.js | 21 ++++++++++++++++++++- components/webui/server/main.js | 12 ++++++++++-- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/components/webui/imports/utils/logger.js b/components/webui/imports/utils/logger.js index 544e6de18..236f24a6f 100644 --- a/components/webui/imports/utils/logger.js +++ b/components/webui/imports/utils/logger.js @@ -5,6 +5,7 @@ import JSON5 from "json5"; const MAX_LOGS_FILE_SIZE = "100m"; const MAX_LOGS_RETENTION_DAYS = "30d"; +let winstonLogger = null; let isTraceEnabled = false; // attribute names should match clp_py_utils.clp_logging.LOGGING_LEVEL_MAPPING @@ -17,6 +18,12 @@ const webuiLoggingLevelToWinstonMap = { CRITICAL: "error", }; +/** + * Retrieves information about the calling function's stack trace. + * + * @returns {Object|null} an object containing method, filePath, and line information, + * or null if the information couldn't be extracted + */ const getStackInfo = () => { let info = null; @@ -45,7 +52,12 @@ const getStackInfo = () => { }; -let winstonLogger = null; +/** + * Logs a message with the specified log level, including optional trace information. + * + * @param {string} level of the log message + * @param {...any} args message or data to be logged + */ const fileLineFuncLog = (level, ...args) => { let logMessage = `${args.map(a => ("string" === typeof a) ? a : JSON5.stringify(a)).join(" ")}`; let logLabel = ""; @@ -79,6 +91,13 @@ export let logger = Object.freeze({ silly: (...args) => (fileLineFuncLog("silly", ...args)), }); +/** + * Initializes winston logger with the specified configuration. + * + * @param {string} logsDir where log files will be stored. + * @param {string} webuiLoggingLevel messages higher than this level will be logged + * @param {boolean} [_isTraceEnabled=false] whether to log function & file names and line numbers + */ export const initLogger = (logsDir, webuiLoggingLevel, _isTraceEnabled = false) => { isTraceEnabled = _isTraceEnabled; diff --git a/components/webui/server/main.js b/components/webui/server/main.js index 41699190b..8c067472d 100644 --- a/components/webui/server/main.js +++ b/components/webui/server/main.js @@ -11,7 +11,15 @@ import {initLogger} from "../imports/utils/logger"; const DEFAULT_LOGS_DIR = "."; const DEFAULT_LOGGING_LEVEL = Meteor.isDevelopment ? "debug" : "info"; -function parseArgs() { +/** + * Parses environment variables and retrieves configuration values for the application. + * + * @returns {Object} object containing configuration values including database host, port, + * name, user, password, logs directory, and logging level + * @throws {Error} if required environment variables are not defined, it exits the process with + * an error + */ +const parseArgs = () => { const CLP_DB_HOST = process.env["CLP_DB_HOST"]; const CLP_DB_PORT = process.env["CLP_DB_PORT"]; const CLP_DB_NAME = process.env["CLP_DB_NAME"]; @@ -36,7 +44,7 @@ function parseArgs() { WEBUI_LOGS_DIR, WEBUI_LOGGING_LEVEL, }; -} +}; Meteor.startup(async () => { const args = parseArgs(); From cd9604db3d158465abc240c412c19fef33656131 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 28 Jan 2024 18:03:06 -0500 Subject: [PATCH 10/60] Add CDN for font "Source Sans Pro" --- components/webui/client/main.html | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/components/webui/client/main.html b/components/webui/client/main.html index e7718bfa7..efc3618f3 100644 --- a/components/webui/client/main.html +++ b/components/webui/client/main.html @@ -1,7 +1,9 @@ - YScope CLP + YScope CLP + -
+
From fda1d2f643439f0d1e93be88805d1c696f58e659 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 28 Jan 2024 19:12:07 -0500 Subject: [PATCH 11/60] Change search progress visualization to prevent elements moving in between searches. --- .../webui/imports/api/search/constants.js | 14 +++++-- .../imports/api/search/server/methods.js | 2 +- .../{SearchControls.js => SearchControls.jsx} | 8 ++-- .../imports/ui/SearchView/SearchResults.jsx | 2 +- .../ui/SearchView/SearchResultsHeader.jsx | 11 +++-- .../imports/ui/SearchView/SearchView.jsx | 40 ++++++++++++++----- 6 files changed, 52 insertions(+), 25 deletions(-) rename components/webui/imports/ui/SearchView/{SearchControls.js => SearchControls.jsx} (96%) diff --git a/components/webui/imports/api/search/constants.js b/components/webui/imports/api/search/constants.js index d8128d480..36d9f5044 100644 --- a/components/webui/imports/api/search/constants.js +++ b/components/webui/imports/api/search/constants.js @@ -10,17 +10,23 @@ let enumSearchSignal; */ export const SearchSignal = Object.freeze({ NONE: (enumSearchSignal=0), + REQ_MASK: (enumSearchSignal = 0x10000000), + REQ_CLEARING: ++enumSearchSignal, REQ_CANCELLING: ++enumSearchSignal, REQ_QUERYING: ++enumSearchSignal, - REQ_CLEARING: ++enumSearchSignal, + RSP_MASK: (enumSearchSignal = 0x20000000), RSP_DONE: ++enumSearchSignal, RSP_ERROR: ++enumSearchSignal, - RSP_SEARCHING: ++enumSearchSignal, + RSP_QUERYING: ++enumSearchSignal, }); -export const isSearchSignalReq = (e) => (0 !== (SearchSignal.REQ_MASK & e)); -export const isSearchSignalRsp = (e) => (0 !== (SearchSignal.RSP_MASK & e)); + +export const isSearchSignalReq = (s) => (0 !== (SearchSignal.REQ_MASK & s)); +export const isSearchSignalRsp = (s) => (0 !== (SearchSignal.RSP_MASK & s)); +export const isSearchSignalQuerying = (s) => ( + [SearchSignal.REQ_QUERYING, SearchSignal.RSP_QUERYING].includes(s) +); let enumJobStatus; /** diff --git a/components/webui/imports/api/search/server/methods.js b/components/webui/imports/api/search/server/methods.js index 3c0b27250..6fd6a9b8c 100644 --- a/components/webui/imports/api/search/server/methods.js +++ b/components/webui/imports/api/search/server/methods.js @@ -158,7 +158,7 @@ Meteor.methods({ if (null !== jobId) { SearchResultsMetadataCollection.insert({ _id: jobId.toString(), - lastSignal: SearchSignal.RSP_SEARCHING, + lastSignal: SearchSignal.RSP_QUERYING, errorMsg: null }); diff --git a/components/webui/imports/ui/SearchView/SearchControls.js b/components/webui/imports/ui/SearchView/SearchControls.jsx similarity index 96% rename from components/webui/imports/ui/SearchView/SearchControls.js rename to components/webui/imports/ui/SearchView/SearchControls.jsx index c8ac0c93b..5793aa687 100644 --- a/components/webui/imports/ui/SearchView/SearchControls.js +++ b/components/webui/imports/ui/SearchView/SearchControls.jsx @@ -1,4 +1,3 @@ -import * as PropTypes from "prop-types"; import React, {useEffect, useState} from "react"; import {Button, Col, Container, Dropdown, DropdownButton, Form, InputGroup, Row} from "react-bootstrap"; @@ -208,7 +207,7 @@ export const SearchControls = ({ - + } { - (SearchSignal.RSP_SEARCHING === resultsMetadata["lastSignal"]) ? + (SearchSignal.RSP_QUERYING === resultsMetadata["lastSignal"]) ? { (SearchSignal.RSP_DONE === resultsMetadata["lastSignal"]) && : : - + (SearchSignal.RESP_QUERYING === resultsMetadata["lastSignal"]) ? + : + } @@ -263,5 +276,5 @@ export const SearchControls = ({ timeRange={timeRange} setTimeRange={setTimeRange} />} - -} + ; +}; diff --git a/components/webui/imports/ui/SearchView/SearchResultsHeader.jsx b/components/webui/imports/ui/SearchView/SearchResultsHeader.jsx index 54d451cfb..dfa91cd17 100644 --- a/components/webui/imports/ui/SearchView/SearchResultsHeader.jsx +++ b/components/webui/imports/ui/SearchView/SearchResultsHeader.jsx @@ -41,7 +41,7 @@ export const SearchResultsHeader = ({ let numResultsText = `Job ID ${jobId}: `; if (0 === numResultsOnServer) { - numResultsText += SearchSignal.RSP_DONE !== resultsMetadata["lastSignal"] ? + numResultsText += SearchSignal.RESP_DONE !== resultsMetadata["lastSignal"] ? "Query is running" : "No results found"; } else if (1 === numResultsOnServer) { @@ -87,4 +87,4 @@ export const SearchResultsHeader = ({ ); -}; \ No newline at end of file +}; diff --git a/components/webui/imports/ui/SearchView/SearchView.jsx b/components/webui/imports/ui/SearchView/SearchView.jsx index 3341c0065..3ad14bc19 100644 --- a/components/webui/imports/ui/SearchView/SearchView.jsx +++ b/components/webui/imports/ui/SearchView/SearchView.jsx @@ -27,7 +27,7 @@ const SearchView = () => { const [jobId, setJobId] = useState(INVALID_JOB_ID); const [operationErrorMsg, setOperationErrorMsg] = useState(""); const [localLastSearchSignal, setLocalLastSearchSignal] = useState(SearchSignal.NONE); - const dbRef = useRef({}); + const dbRef = useRef(new Map()); // gets updated as soon as localLastSearchSignal is updated // to avoid reading old localLastSearchSignal value from Closures const localLastSearchSignalRef = useRef(localLastSearchSignal); diff --git a/components/webui/imports/ui/SearchView/datetime.js b/components/webui/imports/ui/SearchView/datetime.js index afedae002..516145341 100644 --- a/components/webui/imports/ui/SearchView/datetime.js +++ b/components/webui/imports/ui/SearchView/datetime.js @@ -1,190 +1,98 @@ import {DateTime} from "luxon"; -export const computeTodayTimeRange = () => { - const endTime = DateTime.utc(); - const beginTime = endTime.minus({days: 1}); - return { - begin: dateTimeToDateWithoutChangingTimestamp(beginTime), - end: dateTimeToDateWithoutChangingTimestamp(endTime), - }; -}; - -export const computeWeekToDateTimeRange = () => { - const endTime = DateTime.utc(); - const beginTime = endTime.startOf("week"); - return { - begin: dateTimeToDateWithoutChangingTimestamp(beginTime), - end: dateTimeToDateWithoutChangingTimestamp(endTime), - }; -}; - -export const computeMonthToDateTimeRange = () => { - const endTime = DateTime.utc(); - const beginTime = endTime.startOf("month"); - return { - begin: dateTimeToDateWithoutChangingTimestamp(beginTime), - end: dateTimeToDateWithoutChangingTimestamp(endTime), - }; -}; - -export const computeYearToDateTimeRange = () => { - const endTime = DateTime.utc(); - const beginTime = endTime.startOf("year"); - return { - begin: dateTimeToDateWithoutChangingTimestamp(beginTime), - end: dateTimeToDateWithoutChangingTimestamp(endTime), - }; -}; +const TIME_RANGE_UNIT = Object.freeze({ + ALL: "all", + MINUTE: "minute", + HOUR: "hour", + DAY: "day", + WEEK: "week", + MONTH: "month", + YEAR: "year", +}); + +const TIME_RANGE_MODIFIER = Object.freeze({ + NONE: "none", + TODAY: "today", + LAST: "last", + PREV: "prev", + TO_DATE: "to-date", +}); -export const computePrevDayTimeRange = () => { - const endTime = DateTime.utc().minus({days: 1}).endOf("day"); - const beginTime = endTime.startOf("day"); - return { - begin: dateTimeToDateWithoutChangingTimestamp(beginTime), - end: dateTimeToDateWithoutChangingTimestamp(endTime), - }; -}; - -export const computePrevWeekTimeRange = () => { - const endTime = DateTime.utc().minus({weeks: 1}).endOf("week"); - const beginTime = endTime.startOf("week"); - return { - begin: dateTimeToDateWithoutChangingTimestamp(beginTime), - end: dateTimeToDateWithoutChangingTimestamp(endTime), - }; -}; - -export const computePrevMonthTimeRange = () => { - const endTime = DateTime.utc().minus({months: 1}).endOf("month"); - const beginTime = endTime.startOf("month"); - return { - begin: dateTimeToDateWithoutChangingTimestamp(beginTime), - end: dateTimeToDateWithoutChangingTimestamp(endTime), - }; -}; - -export const computePrevYearTimeRange = () => { - const endTime = DateTime.utc().minus({years: 1}).endOf("year"); - const beginTime = endTime.startOf("year"); - return { - begin: dateTimeToDateWithoutChangingTimestamp(beginTime), - end: dateTimeToDateWithoutChangingTimestamp(endTime), - }; -}; - -export const computeLast15MinTimeRange = () => { - const endTime = DateTime.utc(); - const beginTime = endTime.minus({minutes: 15}); - return { - begin: dateTimeToDateWithoutChangingTimestamp(beginTime), - end: dateTimeToDateWithoutChangingTimestamp(endTime), - }; -}; - -export const computeLast60MinTimeRange = () => { - const endTime = DateTime.utc(); - const beginTime = endTime.minus({minutes: 60}); - return { - begin: dateTimeToDateWithoutChangingTimestamp(beginTime), - end: dateTimeToDateWithoutChangingTimestamp(endTime), - }; -}; - -export const computeLast4HourTimeRange = () => { - const endTime = DateTime.utc(); - const beginTime = endTime.minus({hours: 4}); - return { - begin: dateTimeToDateWithoutChangingTimestamp(beginTime), - end: dateTimeToDateWithoutChangingTimestamp(endTime), - }; +// TODO Switch date pickers so we don't have to do this hack +const dateTimeToDateWithoutChangingTimestamp = (dateTime) => { + return dateTime.toLocal().set({ + year: dateTime.year, + month: dateTime.month, + day: dateTime.day, + hour: dateTime.hour, + minute: dateTime.minute, + second: dateTime.second, + millisecond: dateTime.millisecond, + }).toJSDate(); }; -export const computeLast24HourTimeRange = () => { - const endTime = DateTime.utc(); - const beginTime = endTime.minus({hours: 24}); - return { - begin: dateTimeToDateWithoutChangingTimestamp(beginTime), - end: dateTimeToDateWithoutChangingTimestamp(endTime), - }; -}; +export const TIME_RANGE_PRESET_LABEL = Object.freeze({ + [`${TIME_RANGE_UNIT.MINUTE}_${TIME_RANGE_MODIFIER.LAST}_15`]: "Last 15 Minutes", + [`${TIME_RANGE_UNIT.MINUTE}_${TIME_RANGE_MODIFIER.LAST}_60`]: "Last 60 Minutes", + [`${TIME_RANGE_UNIT.HOUR}_${TIME_RANGE_MODIFIER.LAST}_4`]: "Last 4 Hours", + [`${TIME_RANGE_UNIT.HOUR}_${TIME_RANGE_MODIFIER.LAST}_24`]: "Last 24 Hours", + [`${TIME_RANGE_UNIT.DAY}_${TIME_RANGE_MODIFIER.PREV}_1`]: "Previous Day", + [`${TIME_RANGE_UNIT.WEEK}_${TIME_RANGE_MODIFIER.PREV}_1`]: "Previous Week", + [`${TIME_RANGE_UNIT.MONTH}_${TIME_RANGE_MODIFIER.PREV}_1`]: "Previous Month", + [`${TIME_RANGE_UNIT.YEAR}_${TIME_RANGE_MODIFIER.PREV}_1`]: "Previous Year", + [`${TIME_RANGE_UNIT.DAY}_${TIME_RANGE_MODIFIER.TODAY}_0`]: "Today", + [`${TIME_RANGE_UNIT.WEEK}_${TIME_RANGE_MODIFIER.TO_DATE}_0`]: "Week to Date", + [`${TIME_RANGE_UNIT.MONTH}_${TIME_RANGE_MODIFIER.TO_DATE}_0`]: "Month to Date", + [`${TIME_RANGE_UNIT.YEAR}_${TIME_RANGE_MODIFIER.TO_DATE}_0`]: "Year to Date", + [`${TIME_RANGE_UNIT.ALL}_${TIME_RANGE_MODIFIER.NONE}_0`]: "All Time", +}); + +/** + * Computes a time range based on a token. + * + * @param {string} token representing the time range to compute; format: `unit_modifier_amount` + * @returns {Object} containing Date objects representing the computed begin and end time range + */ +export const getRangeComputer = (token) => () => { + const [unit, modifier, amount] = token.split("_"); + let endTime; + let beginTime; + + if (unit === "all") { + endTime = DateTime.utc().plus({years: 1}); + beginTime = DateTime.fromMillis(0, {zone: "UTC"}); + } else { + const isEndingNow = [ + TIME_RANGE_MODIFIER.LAST, + TIME_RANGE_MODIFIER.TODAY, + TIME_RANGE_MODIFIER.TO_DATE, + ].includes(modifier); + const isBeginStartOfUnit = [ + TIME_RANGE_MODIFIER.PREV, + TIME_RANGE_MODIFIER.TODAY, + TIME_RANGE_MODIFIER.TO_DATE, + ].includes(modifier); + + endTime = (true === isEndingNow) ? + DateTime.utc() : + DateTime.utc().minus({[unit]: amount}).endOf(unit); + beginTime = (true === isBeginStartOfUnit) ? + endTime.startOf(unit) : + endTime.minus({[unit]: amount}); + } -export const computeAllTimeRange = () => { - const endTime = DateTime.utc().plus({years: 1}); - const beginTime = DateTime.fromMillis(0, {zone: "UTC"}); return { begin: dateTimeToDateWithoutChangingTimestamp(beginTime), end: dateTimeToDateWithoutChangingTimestamp(endTime), }; }; -export const cTimePresets = [ - { - key: "last-15-mins", - label: "Last 15 Minutes", - compute: computeLast15MinTimeRange, - }, - { - key: "last-60-mins", - label: "Last 60 Minutes", - compute: computeLast60MinTimeRange, - }, - { - key: "last-4-hours", - label: "Last 4 Hours", - compute: computeLast4HourTimeRange, - }, - { - key: "last-24-hours", - label: "Last 24 Hours", - compute: computeLast24HourTimeRange, - }, - { - key: "prev-day", - label: "Previous Day", - compute: computePrevDayTimeRange, - }, - { - key: "prev-week", - label: "Previous Week", - compute: computePrevWeekTimeRange, - }, - { - key: "prev-month", - label: "Previous Month", - compute: computePrevMonthTimeRange, - }, - { - key: "prev-year", - label: "Previous Year", - compute: computePrevYearTimeRange, - }, - { - key: "today", - label: "Today", - compute: computeTodayTimeRange, - }, - { - key: "week-to-date", - label: "Week to Date", - compute: computeWeekToDateTimeRange, - }, - { - key: "month-to-date", - label: "Month to Date", - compute: computeMonthToDateTimeRange, - }, - { - key: "year-to-date", - label: "Year to Date", - compute: computeYearToDateTimeRange, - }, - { - key: "all-time", - label: "All Time", - compute: computeAllTimeRange, - }, -]; - +/** + * Changes the timezone of a given Date object to UTC without changing the time. + * + * @param {Date} date Date object to convert to UTC + * @returns {Date} A new Date object with the same time values in UTC timezone + */ export const changeTimezoneToUTCWithoutChangingTime = (date) => { return new Date(Date.UTC( date.getFullYear(), @@ -197,17 +105,6 @@ export const changeTimezoneToUTCWithoutChangingTime = (date) => { )); }; -// TODO Switch date pickers so we don't have to do this hack -export const dateTimeToDateWithoutChangingTimestamp = (dateTime) => { - return dateTime.toLocal().set({ - year: dateTime.year, - month: dateTime.month, - day: dateTime.day, - hour: dateTime.hour, - minute: dateTime.minute, - second: dateTime.second, - millisecond: dateTime.millisecond, - }).toJSDate(); -}; - -export const DEFAULT_TIME_RANGE_GETTER = computeAllTimeRange; \ No newline at end of file +export const DEFAULT_TIME_RANGE_GETTER = getRangeComputer( + `${TIME_RANGE_UNIT.ALL}_${TIME_RANGE_MODIFIER.NONE}_0`, +); diff --git a/components/webui/imports/ui/Sidebar/Sidebar.jsx b/components/webui/imports/ui/Sidebar/Sidebar.jsx index 6748e3c1f..2d2fd8086 100644 --- a/components/webui/imports/ui/Sidebar/Sidebar.jsx +++ b/components/webui/imports/ui/Sidebar/Sidebar.jsx @@ -14,15 +14,13 @@ import { * @param {boolean} isSidebarCollapsed indicates whether the sidebar is collapsed * @param {Object[]} routes objects for navigation links * @param {function} onSidebarToggle callback to toggle the sidebar's collapsed state - * @param {function} onSidebarTransitioned callback for sidebar transition end event * @returns {JSX.Element} */ -const Sidebar = ({isSidebarCollapsed, routes, onSidebarToggle, onSidebarTransitioned}) => { +const Sidebar = ({isSidebarCollapsed, routes, onSidebarToggle}) => { return ( ); -}; \ No newline at end of file +}; diff --git a/components/webui/imports/utils/logger.js b/components/webui/imports/utils/logger.js index 236f24a6f..bad28aaf1 100644 --- a/components/webui/imports/utils/logger.js +++ b/components/webui/imports/utils/logger.js @@ -127,4 +127,4 @@ export const initLogger = (logsDir, webuiLoggingLevel, _isTraceEnabled = false) }); logger.info("logger has been initialized"); -}; \ No newline at end of file +}; diff --git a/components/webui/imports/utils/misc.js b/components/webui/imports/utils/misc.js index 4c2a15054..58d87a098 100644 --- a/components/webui/imports/utils/misc.js +++ b/components/webui/imports/utils/misc.js @@ -4,4 +4,4 @@ * @param {number} seconds to wait before resolving the promise * @returns {Promise} that resolves after the specified delay */ -export const sleep = (seconds) => new Promise(r => setTimeout(r, seconds * 1000)); \ No newline at end of file +export const sleep = (seconds) => new Promise(r => setTimeout(r, seconds * 1000)); diff --git a/components/webui/launcher.js b/components/webui/launcher.js index ac7d8a9b5..8c906f35e 100644 --- a/components/webui/launcher.js +++ b/components/webui/launcher.js @@ -81,4 +81,4 @@ const main = () => { runScript(args.WEBUI_LOGS_DIR, args.scriptPath); }; -main(); \ No newline at end of file +main(); From c24244f8a5406a0589358bbe752d634243e9fcab Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Mon, 5 Feb 2024 07:00:13 -0500 Subject: [PATCH 27/60] Fix some docstrings. --- components/webui/imports/api/search/collections.js | 3 +-- components/webui/imports/api/search/server/methods.js | 2 +- components/webui/launcher.js | 6 +++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/components/webui/imports/api/search/collections.js b/components/webui/imports/api/search/collections.js index 99fe393ea..c57e22de3 100644 --- a/components/webui/imports/api/search/collections.js +++ b/components/webui/imports/api/search/collections.js @@ -26,8 +26,7 @@ export const initSearchEventCollection = () => { * Object to store references to MongoDB collections. * This should only be accessed by the server; clients should use a React referenced-object. * - * @constant - * @type {Object} + * @type {Map} */ // FIXME: change to use Map before submission export const MY_MONGO_DB = new Map(); diff --git a/components/webui/imports/api/search/server/methods.js b/components/webui/imports/api/search/server/methods.js index c8a2fe716..57d5d5dee 100644 --- a/components/webui/imports/api/search/server/methods.js +++ b/components/webui/imports/api/search/server/methods.js @@ -29,7 +29,7 @@ const updateSearchEventWhenJobFinishes = async (jobId) => { /** * Creates MongoDB indexes for a specific job's collection. * - * @param {?number} jobId used to identify the Mongo Collection to add indexes + * @param {number} jobId used to identify the Mongo Collection to add indexes */ const createMongoIndexes = async (jobId) => { const timestampAscendingIndex = { diff --git a/components/webui/launcher.js b/components/webui/launcher.js index 8c906f35e..fe92dd72e 100644 --- a/components/webui/launcher.js +++ b/components/webui/launcher.js @@ -1,8 +1,9 @@ -/* Production launcher for CLP WebUI, which redirects Meteor server stderr to rotated error logs +/** + * Production launcher for CLP WebUI, which redirects Meteor server stderr to rotated error logs * files in a specified directory for error monitoring. * To avoid duplicated installations of dependencies, use the same `node_modules` for the server - * by setting envvar NODE_PATH="./programs/server/npm/node_modules", assumning this script is + * by setting envvar NODE_PATH="./programs/server/npm/node_modules", assuming this script is * placed under the same directory where bundled `main.js` locates. * * This is not intended for development use. For development, please refer to README.md in the @@ -14,7 +15,6 @@ * - WEBUI_LOGS_DIR: path to error logs directory * SCRIPT USAGE: * - usage: node /path/to/launcher.js /path/to/main.js - * */ const {spawn} = require("child_process"); From f4836a5031c3806b12c55770de5e8d51c7356513 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Mon, 5 Feb 2024 07:46:05 -0500 Subject: [PATCH 28/60] Remove unused RESP_ERROR constant. --- components/webui/imports/api/search/constants.js | 1 - 1 file changed, 1 deletion(-) diff --git a/components/webui/imports/api/search/constants.js b/components/webui/imports/api/search/constants.js index bd5f8d6fc..21e8f47e7 100644 --- a/components/webui/imports/api/search/constants.js +++ b/components/webui/imports/api/search/constants.js @@ -18,7 +18,6 @@ export const SearchSignal = Object.freeze({ RESP_MASK: (enumSearchSignal = 0x20000000), RESP_DONE: ++enumSearchSignal, - RESP_ERROR: ++enumSearchSignal, RESP_QUERYING: ++enumSearchSignal, }); From b585e5602cf527152a1cf7c9c7f37a997c7a7573 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Tue, 6 Feb 2024 02:03:46 -0500 Subject: [PATCH 29/60] Pass database host, port, and name through settings.json along with the search jobs table name. --- .../clp_package_utils/scripts/start_clp.py | 40 ++++++++++++++++--- .../webui/imports/api/search/server/sql.js | 2 +- components/webui/server/main.js | 39 ++++++++---------- 3 files changed, 52 insertions(+), 29 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 4d2e3e836..51b72d353 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -8,6 +8,7 @@ import subprocess import sys import time +import typing import uuid import yaml @@ -19,6 +20,7 @@ QUEUE_COMPONENT_NAME, REDIS_COMPONENT_NAME, RESULTS_CACHE_COMPONENT_NAME, + SEARCH_JOBS_TABLE_NAME, SEARCH_SCHEDULER_COMPONENT_NAME, SEARCH_WORKER_COMPONENT_NAME, WEBUI_COMPONENT_NAME, @@ -625,6 +627,29 @@ def generic_start_worker( logger.info(f"Started {component_name}.") +def update_meteor_settings( + parent_key_prefix: str, + settings: typing.Dict[str, typing.Any], + updates: typing.Dict[str, typing.Any], +): + """ + Recursively updates the given Meteor settings object with the values from `updates`. + + :param parent_key_prefix: The prefix for keys at this level in the settings dictionary. + :param settings: The settings to update. + :param updates: The updates. + :raises ValueError: If a key in `updates` doesn't exist in `settings`. + """ + for key, value in updates.items(): + if key not in settings: + error_msg = f"{parent_key_prefix}{key} is not a valid configuration key for the webui." + raise ValueError(error_msg) + if isinstance(value, dict): + update_meteor_settings(f"{parent_key_prefix}{key}.", settings[key], value) + else: + settings[key] = updates[key] + + def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts): logger.info(f"Starting {WEBUI_COMPONENT_NAME}...") @@ -646,8 +671,16 @@ def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts container_webui_logs_dir = pathlib.Path("/") / "var" / "log" / WEBUI_COMPONENT_NAME with open(settings_json_path, "r") as settings_json_file: - settings_json_content = settings_json_file.read() - meteor_settings = json.loads(settings_json_content) + meteor_settings = json.loads(settings_json_file.read()) + meteor_settings_updates = { + "private": { + "SqlDbHost": clp_config.database.host, + "SqlDbPort": clp_config.database.port, + "SqlDbName": clp_config.database.name, + "SqlDbSearchJobsTableName": SEARCH_JOBS_TABLE_NAME, + } + } + update_meteor_settings("", meteor_settings, meteor_settings_updates) # Start container # fmt: off @@ -662,9 +695,6 @@ def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts "-e", f"PORT={clp_config.webui.port}", "-e", f"ROOT_URL=http://{clp_config.webui.host}", "-e", f"METEOR_SETTINGS={json.dumps(meteor_settings)}", - "-e", f"CLP_DB_HOST={clp_config.database.host}", - "-e", f"CLP_DB_PORT={clp_config.database.port}", - "-e", f"CLP_DB_NAME={clp_config.database.name}", "-e", f"CLP_DB_USER={clp_config.database.username}", "-e", f"CLP_DB_PASS={clp_config.database.password}", "-e", f"WEBUI_LOGS_DIR={container_webui_logs_dir}", diff --git a/components/webui/imports/api/search/server/sql.js b/components/webui/imports/api/search/server/sql.js index bcedadbf1..680cd679e 100644 --- a/components/webui/imports/api/search/server/sql.js +++ b/components/webui/imports/api/search/server/sql.js @@ -5,7 +5,7 @@ import {logger} from "/imports/utils/logger"; import {sleep} from "../../../utils/misc"; import {JOB_STATUS_WAITING_STATES, JobStatus} from "../constants"; -const SEARCH_JOBS_TABLE_NAME = "search_jobs"; +const SEARCH_JOBS_TABLE_NAME = Meteor.settings.private.SqlDbSearchJobsTableName; const SEARCH_JOBS_TABLE_COLUMN_NAMES = { ID: "id", STATUS: "status", diff --git a/components/webui/server/main.js b/components/webui/server/main.js index 6d84033ce..e69fbdaeb 100644 --- a/components/webui/server/main.js +++ b/components/webui/server/main.js @@ -12,33 +12,26 @@ const DEFAULT_LOGS_DIR = "."; const DEFAULT_LOGGING_LEVEL = Meteor.isDevelopment ? "debug" : "info"; /** - * Parses environment variables and retrieves configuration values for the application. + * Parses environment variables into config values for the application. * - * @returns {Object} object containing configuration values including database host, port, - * name, user, password, logs directory, and logging level - * @throws {Error} if required environment variables are not defined, it exits the process with - * an error + * @returns {Object} An object containing config values including the SQL database credentials, + * logs directory, and logging level. + * @throws {Error} if the required environment variables are undefined, it exits the process with an + * error. */ -const parseArgs = () => { - const CLP_DB_HOST = process.env["CLP_DB_HOST"]; - const CLP_DB_PORT = process.env["CLP_DB_PORT"]; - const CLP_DB_NAME = process.env["CLP_DB_NAME"]; +const parseEnvVars = () => { const CLP_DB_USER = process.env["CLP_DB_USER"]; const CLP_DB_PASS = process.env["CLP_DB_PASS"]; - if ([CLP_DB_HOST, CLP_DB_PORT, CLP_DB_NAME, CLP_DB_USER, CLP_DB_PASS].includes(undefined)) { - console.error( - "Environment variables CLP_DB_URL, CLP_DB_USER and CLP_DB_PASS need to be defined"); - process.exit(1); + if ([CLP_DB_USER, CLP_DB_PASS].includes(undefined)) { + console.error("Environment variables CLP_DB_USER and CLP_DB_PASS must be defined"); + process.exit(1) } const WEBUI_LOGS_DIR = process.env["WEBUI_LOGS_DIR"] || DEFAULT_LOGS_DIR; const WEBUI_LOGGING_LEVEL = process.env["WEBUI_LOGGING_LEVEL"] || DEFAULT_LOGGING_LEVEL; return { - CLP_DB_HOST, - CLP_DB_PORT, - CLP_DB_NAME, CLP_DB_USER, CLP_DB_PASS, WEBUI_LOGS_DIR, @@ -47,16 +40,16 @@ const parseArgs = () => { }; Meteor.startup(async () => { - const args = parseArgs(); + const envVars = parseEnvVars(); - initLogger(args.WEBUI_LOGS_DIR, args.WEBUI_LOGGING_LEVEL, Meteor.isDevelopment); + initLogger(envVars.WEBUI_LOGS_DIR, envVars.WEBUI_LOGGING_LEVEL, Meteor.isDevelopment); await initSql( - args.CLP_DB_HOST, - parseInt(args.CLP_DB_PORT), - args.CLP_DB_NAME, - args.CLP_DB_USER, - args.CLP_DB_PASS, + Meteor.settings.private.SqlDbHost, + Meteor.settings.private.SqlDbPort, + Meteor.settings.private.SqlDbName, + envVars.CLP_DB_USER, + envVars.CLP_DB_PASS, ); initSearchEventCollection(); From 5628e03327ee2db3807a00b905c848cb4df5b55e Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Tue, 6 Feb 2024 02:06:51 -0500 Subject: [PATCH 30/60] Remove unused highcharts dependency. --- components/webui/package-lock.json | 18 ------------------ components/webui/package.json | 3 --- 2 files changed, 21 deletions(-) diff --git a/components/webui/package-lock.json b/components/webui/package-lock.json index a37f54c2b..3450bb13e 100644 --- a/components/webui/package-lock.json +++ b/components/webui/package-lock.json @@ -268,24 +268,6 @@ "is-property": "^1.0.2" } }, - "highcharts": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/highcharts/-/highcharts-11.1.0.tgz", - "integrity": "sha512-vhmqq6/frteWMx0GKYWwEFL25g4OYc7+m+9KQJb/notXbNtIb8KVy+ijOF7XAFqF165cq0pdLIePAmyFY5ph3g==" - }, - "highcharts-custom-events": { - "version": "3.0.10", - "resolved": "https://registry.npmjs.org/highcharts-custom-events/-/highcharts-custom-events-3.0.10.tgz", - "integrity": "sha512-swD0v8qovr9tzW6wNdebJKWQS5xTUclU2XznZw7+U6zMpedzEw7nxMZCog+P3JBDvp2cLO8ySiy7EC29DayfFw==", - "requires": { - "highcharts": ">=5.0.9" - } - }, - "highcharts-react-official": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/highcharts-react-official/-/highcharts-react-official-3.2.1.tgz", - "integrity": "sha512-hyQTX7ezCxl7JqumaWiGsroGWalzh24GedQIgO3vJbkGOZ6ySRAltIYjfxhrq4HszJOySZegotEF7v+haQ75UA==" - }, "history": { "version": "4.10.1", "resolved": "https://registry.npmjs.org/history/-/history-4.10.1.tgz", diff --git a/components/webui/package.json b/components/webui/package.json index a8376416f..b036cc261 100644 --- a/components/webui/package.json +++ b/components/webui/package.json @@ -14,9 +14,6 @@ "@fortawesome/react-fontawesome": "^0.2.0", "@msgpack/msgpack": "^3.0.0-beta2", "bootstrap": "^5.3.2", - "highcharts": "^11.1.0", - "highcharts-custom-events": "^3.0.10", - "highcharts-react-official": "^3.2.1", "json5": "^2.2.3", "luxon": "^3.4.3", "meteor-node-stubs": "^1.1.0", From db259dfe4aa6c5ee1d2b33ef0d5aff7c49aa3af6 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 7 Feb 2024 00:51:09 -0500 Subject: [PATCH 31/60] Add new settings missed in previous commit. --- components/webui/settings.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/components/webui/settings.json b/components/webui/settings.json index 500e657ec..69bcdbb04 100644 --- a/components/webui/settings.json +++ b/components/webui/settings.json @@ -1,4 +1,10 @@ { + "private": { + "SqlDbHost": "localhost", + "SqlDbPort": 3306, + "SqlDbName": "clp-db", + "SqlDbSearchJobsTableName": "search_jobs" + }, "public": { "SearchResultsCollectionName": "results", "SearchResultsMetadataCollectionName": "results-metadata" From a9ee0a35ec8d0804a3679669299b41f6d47a302f Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 7 Feb 2024 00:53:03 -0500 Subject: [PATCH 32/60] Replace MongoDB collections map with a class for more robust management. --- .../api/search/SearchJobCollectionsManager.js | 35 +++++++++++++++++++ .../webui/imports/api/search/collections.js | 26 -------------- .../imports/api/search/server/collections.js | 3 ++ .../imports/api/search/server/methods.js | 16 +++++---- .../imports/api/search/server/publications.js | 5 +-- .../imports/ui/SearchView/SearchView.jsx | 15 +++++--- components/webui/server/main.js | 1 + 7 files changed, 62 insertions(+), 39 deletions(-) create mode 100644 components/webui/imports/api/search/SearchJobCollectionsManager.js create mode 100644 components/webui/imports/api/search/server/collections.js diff --git a/components/webui/imports/api/search/SearchJobCollectionsManager.js b/components/webui/imports/api/search/SearchJobCollectionsManager.js new file mode 100644 index 000000000..72184950a --- /dev/null +++ b/components/webui/imports/api/search/SearchJobCollectionsManager.js @@ -0,0 +1,35 @@ +/** + * Class to keep track of MongoDB collections created for search jobs, ensuring all collections have + * unique names. + */ +class SearchJobCollectionsManager { + #collections; + constructor() { + this.#collections = new Map(); + } + + /** + * Gets, or if it doesn't exist, creates a MongoDB collection named with the given job ID. + * + * @param {number} jobId + * @returns {Mongo.Collection} + */ + getOrCreateCollection(jobId) { + const name = jobId.toString(); + if (undefined === this.#collections.get(name)) { + this.#collections.set(name, new Mongo.Collection(name)); + } + return this.#collections.get(name); + } + + /** + * Removes the MongoDB collection with the given job ID. + * + * @param {number} jobId + */ + removeCollection(jobId) { + this.#collections.delete(jobId.toString()); + } +} + +export default SearchJobCollectionsManager; diff --git a/components/webui/imports/api/search/collections.js b/components/webui/imports/api/search/collections.js index c57e22de3..e9bbf0f9b 100644 --- a/components/webui/imports/api/search/collections.js +++ b/components/webui/imports/api/search/collections.js @@ -21,29 +21,3 @@ export const initSearchEventCollection = () => { }); } } - -/** - * Object to store references to MongoDB collections. - * This should only be accessed by the server; clients should use a React referenced-object. - * - * @type {Map} - */ -// FIXME: change to use Map before submission -export const MY_MONGO_DB = new Map(); - -/** - * Retrieves a Mongo Collection object by name, creating it if it does not already exist. - * This is to adhere to Meteor.js's restriction against creating more than one Collection object - * with the same name. - * - * @param {Object} dbRef database object where collections are stored - * @param {string} name of the collection to retrieve or create - * @returns {Mongo.Collection} - */ -export const getCollection = (dbRef, name) => { - if (undefined === dbRef.get(name)) { - dbRef.set(name, new Mongo.Collection(name)); - } - - return dbRef.get(name); -}; diff --git a/components/webui/imports/api/search/server/collections.js b/components/webui/imports/api/search/server/collections.js new file mode 100644 index 000000000..7ca663cb4 --- /dev/null +++ b/components/webui/imports/api/search/server/collections.js @@ -0,0 +1,3 @@ +import SearchJobCollectionsManager from "../SearchJobCollectionsManager"; + +export const searchJobCollectionsManager = new SearchJobCollectionsManager(); diff --git a/components/webui/imports/api/search/server/methods.js b/components/webui/imports/api/search/server/methods.js index 57d5d5dee..300ff83a0 100644 --- a/components/webui/imports/api/search/server/methods.js +++ b/components/webui/imports/api/search/server/methods.js @@ -1,6 +1,7 @@ import {logger} from "/imports/utils/logger"; import {Meteor} from "meteor/meteor"; -import {getCollection, MY_MONGO_DB, SearchResultsMetadataCollection} from "../collections"; +import {SearchResultsMetadataCollection} from "../collections"; +import {searchJobCollectionsManager} from "./collections"; import {SearchSignal} from "../constants"; import {cancelQuery, submitQuery, waitTillJobFinishes} from "./sql"; @@ -18,7 +19,8 @@ const updateSearchEventWhenJobFinishes = async (jobId) => { $set: { lastSignal: SearchSignal.RESP_DONE, errorMsg: errorMsg, - numTotalResults: await getCollection(MY_MONGO_DB, jobId.toString()).countDocuments() + numTotalResults: + await searchJobCollectionsManager.getOrCreateCollection(jobId).countDocuments(), } }; @@ -41,7 +43,7 @@ const createMongoIndexes = async (jobId) => { name: "timestamp-descending" }; - const queryJobCollection = getCollection(MY_MONGO_DB, jobId.toString()); + const queryJobCollection = searchJobCollectionsManager.getOrCreateCollection(jobId); const queryJobRawCollection = queryJobCollection.rawCollection(); await queryJobRawCollection.createIndexes([timestampAscendingIndex, timestampDescendingIndex]); }; @@ -90,7 +92,7 @@ Meteor.methods({ /** * Clears the results of a search operation identified by jobId. * - * @param {string} jobId of the search results to clear + * @param {number} jobId of the search results to clear */ async "search.clearResults"({ jobId @@ -99,10 +101,10 @@ Meteor.methods({ logger.info("search.clearResults jobId =", jobId) try { - const resultsCollection = getCollection(MY_MONGO_DB, jobId.toString()); + const resultsCollection = searchJobCollectionsManager.getOrCreateCollection(jobId); await resultsCollection.dropCollectionAsync(); - delete MY_MONGO_DB[jobId.toString()]; + searchJobCollectionsManager.removeCollection(jobId); } catch (e) { logger.error(`Unable to clear search results for jobId=${jobId}`, e); } @@ -111,7 +113,7 @@ Meteor.methods({ /** * Cancels an ongoing search operation identified by jobId. * - * @param {string} jobId of the search operation to cancel + * @param {number} jobId of the search operation to cancel */ async "search.cancelOperation"({ jobId diff --git a/components/webui/imports/api/search/server/publications.js b/components/webui/imports/api/search/server/publications.js index 032e44335..c2b5c19a8 100644 --- a/components/webui/imports/api/search/server/publications.js +++ b/components/webui/imports/api/search/server/publications.js @@ -1,7 +1,8 @@ import {Meteor} from "meteor/meteor"; import {logger} from "/imports/utils/logger"; -import {SearchResultsMetadataCollection, getCollection, MY_MONGO_DB} from "../collections"; +import {SearchResultsMetadataCollection} from "../collections"; +import {searchJobCollectionsManager} from "./collections"; /** * Publishes search results metadata for a specific job. @@ -41,7 +42,7 @@ Meteor.publish(Meteor.settings.public.SearchResultsCollectionName, ({ `jobId=${jobId}, fieldToSortBy=${fieldToSortBy}, ` + `visibleSearchResultsLimit=${visibleSearchResultsLimit}`); - const collection = getCollection(MY_MONGO_DB, jobId.toString()); + const collection = searchJobCollectionsManager.getOrCreateCollection(jobId); const findOptions = { limit: visibleSearchResultsLimit, disableOplog: true, diff --git a/components/webui/imports/ui/SearchView/SearchView.jsx b/components/webui/imports/ui/SearchView/SearchView.jsx index 2a8e0334f..fb618b616 100644 --- a/components/webui/imports/ui/SearchView/SearchView.jsx +++ b/components/webui/imports/ui/SearchView/SearchView.jsx @@ -5,8 +5,9 @@ import {useTracker} from "meteor/react-meteor-data"; import React, {useEffect, useRef, useState} from "react"; import {ProgressBar} from "react-bootstrap"; -import {getCollection, SearchResultsMetadataCollection} from "../../api/search/collections"; +import {SearchResultsMetadataCollection} from "../../api/search/collections"; import {INVALID_JOB_ID, isSearchSignalQuerying, SearchSignal} from "../../api/search/constants"; +import SearchJobCollectionsManager from "../../api/search/SearchJobCollectionsManager"; import "react-datepicker/dist/react-datepicker.css"; import LOCAL_STORAGE_KEYS from "../constants"; @@ -27,7 +28,7 @@ const SearchView = () => { const [jobId, setJobId] = useState(INVALID_JOB_ID); const [operationErrorMsg, setOperationErrorMsg] = useState(""); const [localLastSearchSignal, setLocalLastSearchSignal] = useState(SearchSignal.NONE); - const dbRef = useRef(new Map()); + const dbRef = useRef(new SearchJobCollectionsManager()); // gets updated as soon as localLastSearchSignal is updated // to avoid reading old localLastSearchSignal value from Closures const localLastSearchSignalRef = useRef(localLastSearchSignal); @@ -76,7 +77,13 @@ const SearchView = () => { visibleSearchResultsLimit: visibleSearchResultsLimit, }); - return getCollection(dbRef.current, jobId.toString()).find().fetch(); + // NOTE: Although we publish and subscribe using the name + // `Meteor.settings.public.SearchResultsCollectionName`, the rows are still returned in the + // job-specific collection (e.g., "1"); this is because on the server, we're returning a + // cursor from the job-specific collection and Meteor creates a collection with the same + // name on the client rather than returning the rows in a collection with the published + // name. + return dbRef.current.getOrCreateCollection(jobId).find().fetch(); }, [jobId, fieldToSortBy, visibleSearchResultsLimit]); // State transitions @@ -122,7 +129,7 @@ const SearchView = () => { }; const handleClearResults = () => { - delete dbRef.current[jobId.toString()]; + dbRef.current.removeCollection(jobId); setJobId(INVALID_JOB_ID); setOperationErrorMsg(""); diff --git a/components/webui/server/main.js b/components/webui/server/main.js index e69fbdaeb..62078fe81 100644 --- a/components/webui/server/main.js +++ b/components/webui/server/main.js @@ -1,5 +1,6 @@ import {Meteor} from "meteor/meteor"; +import "/imports/api/search/server/collections"; import "/imports/api/search/server/methods"; import "/imports/api/search/server/publications"; import "/imports/api/user/server/methods"; From 87b1be4bca299d160b993599ea49ceeb351daaf6 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 7 Feb 2024 00:55:55 -0500 Subject: [PATCH 33/60] Use correct case for logging level. --- components/webui/server/main.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/webui/server/main.js b/components/webui/server/main.js index 62078fe81..74f5e365c 100644 --- a/components/webui/server/main.js +++ b/components/webui/server/main.js @@ -10,7 +10,7 @@ import {initSearchEventCollection} from "/imports/api/search/collections"; import {initLogger} from "/imports/utils/logger"; const DEFAULT_LOGS_DIR = "."; -const DEFAULT_LOGGING_LEVEL = Meteor.isDevelopment ? "debug" : "info"; +const DEFAULT_LOGGING_LEVEL = Meteor.isDevelopment ? "DEBUG" : "INFO"; /** * Parses environment variables into config values for the application. From 82a309df66f3a066d25578783f2718d5c0ac1316 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 7 Feb 2024 04:57:14 -0500 Subject: [PATCH 34/60] Sort search results on both the client and server. --- .../webui/imports/api/search/collections.js | 15 +++++++++++++++ .../imports/api/search/server/publications.js | 11 +++-------- .../webui/imports/ui/SearchView/SearchView.jsx | 10 ++++++++-- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/components/webui/imports/api/search/collections.js b/components/webui/imports/api/search/collections.js index e9bbf0f9b..84a7a6d98 100644 --- a/components/webui/imports/api/search/collections.js +++ b/components/webui/imports/api/search/collections.js @@ -21,3 +21,18 @@ export const initSearchEventCollection = () => { }); } } + +/** + * Adds the given sort to the find options for a MongoDB collection. + * @param {Object|null} fieldToSortBy An object mapping field names to the direction to sort by + * (ASC = 1, DESC = -1). + * @param {Object} findOptions + */ +export const addSortToMongoFindOptions = (fieldToSortBy, findOptions) => { + if (fieldToSortBy) { + findOptions["sort"] = { + [fieldToSortBy.name]: fieldToSortBy.direction, + _id: fieldToSortBy.direction, + }; + } +} diff --git a/components/webui/imports/api/search/server/publications.js b/components/webui/imports/api/search/server/publications.js index c2b5c19a8..da1e1ca32 100644 --- a/components/webui/imports/api/search/server/publications.js +++ b/components/webui/imports/api/search/server/publications.js @@ -1,7 +1,7 @@ import {Meteor} from "meteor/meteor"; import {logger} from "/imports/utils/logger"; -import {SearchResultsMetadataCollection} from "../collections"; +import {addSortToMongoFindOptions, SearchResultsMetadataCollection} from "../collections"; import {searchJobCollectionsManager} from "./collections"; /** @@ -43,18 +43,13 @@ Meteor.publish(Meteor.settings.public.SearchResultsCollectionName, ({ `visibleSearchResultsLimit=${visibleSearchResultsLimit}`); const collection = searchJobCollectionsManager.getOrCreateCollection(jobId); + const findOptions = { limit: visibleSearchResultsLimit, disableOplog: true, pollingIntervalMs: 250, }; - - if (fieldToSortBy) { - const sort = {}; - sort[fieldToSortBy.name] = fieldToSortBy.direction; - sort["_id"] = fieldToSortBy.direction; - findOptions["sort"] = sort; - } + addSortToMongoFindOptions(fieldToSortBy, findOptions); return collection.find({}, findOptions); }); diff --git a/components/webui/imports/ui/SearchView/SearchView.jsx b/components/webui/imports/ui/SearchView/SearchView.jsx index fb618b616..344dbbc2f 100644 --- a/components/webui/imports/ui/SearchView/SearchView.jsx +++ b/components/webui/imports/ui/SearchView/SearchView.jsx @@ -5,7 +5,10 @@ import {useTracker} from "meteor/react-meteor-data"; import React, {useEffect, useRef, useState} from "react"; import {ProgressBar} from "react-bootstrap"; -import {SearchResultsMetadataCollection} from "../../api/search/collections"; +import { + addSortToMongoFindOptions, + SearchResultsMetadataCollection +} from "../../api/search/collections"; import {INVALID_JOB_ID, isSearchSignalQuerying, SearchSignal} from "../../api/search/constants"; import SearchJobCollectionsManager from "../../api/search/SearchJobCollectionsManager"; @@ -77,13 +80,16 @@ const SearchView = () => { visibleSearchResultsLimit: visibleSearchResultsLimit, }); + const findOptions = {}; + addSortToMongoFindOptions(fieldToSortBy, findOptions); + // NOTE: Although we publish and subscribe using the name // `Meteor.settings.public.SearchResultsCollectionName`, the rows are still returned in the // job-specific collection (e.g., "1"); this is because on the server, we're returning a // cursor from the job-specific collection and Meteor creates a collection with the same // name on the client rather than returning the rows in a collection with the published // name. - return dbRef.current.getOrCreateCollection(jobId).find().fetch(); + return dbRef.current.getOrCreateCollection(jobId).find({}, findOptions).fetch(); }, [jobId, fieldToSortBy, visibleSearchResultsLimit]); // State transitions From 5ccc8a7820c395576a33a3229b8ddd6fde3424dc Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 7 Feb 2024 10:38:56 +0000 Subject: [PATCH 35/60] Add docs for webui/launcher.js --- components/webui/launcher.js | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/components/webui/launcher.js b/components/webui/launcher.js index ac7d8a9b5..bac7e97fe 100644 --- a/components/webui/launcher.js +++ b/components/webui/launcher.js @@ -27,6 +27,12 @@ const DEFAULT_LOGS_DIR = "."; const MAX_LOGS_FILE_SIZE = "100m"; const MAX_LOGS_RETENTION_DAYS = "30d"; +/** + * Creates a logger using winston module. + * + * @param {string} logsDir directory where the log files will be saved. + * @returns {object} the logger object + */ const getLogger = (logsDir) => { return winston.createLogger({ format: winston.format.combine( @@ -52,6 +58,13 @@ const getLogger = (logsDir) => { }); }; + +/** + * Runs a script with logging support. + * + * @param {string} logsDir path where the logs will be stored + * @param {string} scriptPath path of the script to be executed + */ const runScript = (logsDir, scriptPath) => { const logger = getLogger(logsDir); const script = spawn(process.argv0, [scriptPath]); @@ -65,6 +78,12 @@ const runScript = (logsDir, scriptPath) => { }); }; +/** + * Parses the command line arguments and retrieves the values for the + * WEBUI_LOGS_DIR and scriptPath variables. + * + * @returns {Object} containing the values for WEBUI_LOGS_DIR and scriptPath + */ const parseArgs = () => { const WEBUI_LOGS_DIR = process.env["WEBUI_LOGS_DIR"] || DEFAULT_LOGS_DIR; const scriptPath = process.argv[2]; @@ -75,10 +94,17 @@ const parseArgs = () => { }; }; +/** + * The main function of the program. + * + * This function is the entry point of the program. + * + * @returns {void} + */ const main = () => { const args = parseArgs(); runScript(args.WEBUI_LOGS_DIR, args.scriptPath); }; -main(); \ No newline at end of file +main(); From d191d531e71d3ecc1814312e27a42dd19d3efb96 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 7 Feb 2024 07:28:26 -0500 Subject: [PATCH 36/60] Replace sql.js with SearchJobsDbManager class for more robust management. --- .../api/search/server/SearchJobsDbManager.js | 140 ++++++++++++++++++ .../imports/api/search/server/methods.js | 92 +++++++++--- .../webui/imports/api/search/server/sql.js | 134 ----------------- components/webui/server/main.js | 22 +-- 4 files changed, 226 insertions(+), 162 deletions(-) create mode 100644 components/webui/imports/api/search/server/SearchJobsDbManager.js delete mode 100644 components/webui/imports/api/search/server/sql.js diff --git a/components/webui/imports/api/search/server/SearchJobsDbManager.js b/components/webui/imports/api/search/server/SearchJobsDbManager.js new file mode 100644 index 000000000..6f7fc3ada --- /dev/null +++ b/components/webui/imports/api/search/server/SearchJobsDbManager.js @@ -0,0 +1,140 @@ +import mysql from "mysql2/promise"; +import msgpack from "@msgpack/msgpack"; +import {JOB_STATUS_WAITING_STATES, JobStatus} from "../constants"; +import {sleep} from "../../../utils/misc"; + +const SEARCH_JOBS_TABLE_COLUMN_NAMES = { + ID: "id", + STATUS: "status", + SEARCH_CONFIG: "search_config", +}; + +/** + * Class for submitting and monitoring search jobs in the database. + */ +class SearchJobsDbManager { + #sqlDbConnection; + #searchJobsTableName; + + /** + * Creates a new instance. + * @param {string} dbHost + * @param {number} dbPort + * @param {string} dbName + * @param {string} dbUser + * @param {string} dbPassword + * @param {string} searchJobsTableName + * @returns {Promise} + * @throws {Error} on error. + */ + static async createNew({dbHost, dbPort, dbName, dbUser, dbPassword, searchJobsTableName}) { + const conn = await mysql.createConnection({ + host: dbHost, + port: dbPort, + database: dbName, + user: dbUser, + password: dbPassword, + }); + return new SearchJobsDbManager(conn, searchJobsTableName); + } + + /** + * @param {mysql.Connection} sqlDbConnection + * @param {string} searchJobsTableName + */ + constructor(sqlDbConnection, searchJobsTableName) { + this.#sqlDbConnection = sqlDbConnection; + this.#searchJobsTableName = searchJobsTableName; + } + + /** + * Connects to the database. + * @returns {Promise} + * @throws {Error} on error. + */ + async connect() { + await this.#sqlDbConnection.connect(); + } + + /** + * Disconnects from the database. + * @returns {Promise} + * @throws {Error} on error. + */ + async disconnect() { + await this.#sqlDbConnection.end(); + } + + /** + * Submits a query job to the database. + * @param {Object} searchConfig The arguments for the query. + * @returns {Promise} The job's ID. + * @throws {Error} on error. + */ + async submitQuery(searchConfig) { + const [queryInsertResults] = await this.#sqlDbConnection.query( + `INSERT INTO ${this.#searchJobsTableName} + (${SEARCH_JOBS_TABLE_COLUMN_NAMES.SEARCH_CONFIG}) + VALUES (?)`, + [Buffer.from(msgpack.encode(searchConfig))], + ); + return queryInsertResults.insertId; + } + + /** + * Submits a query cancellation request to the database. + * @param {number} jobId ID of the job to cancel. + * @returns {Promise} + * @throws {Error} on error. + */ + async submitQueryCancellation(jobId) { + await this.#sqlDbConnection.query( + `UPDATE ${this.#searchJobsTableName} + SET ${SEARCH_JOBS_TABLE_COLUMN_NAMES.STATUS} = ${JobStatus.CANCELLING} + WHERE ${SEARCH_JOBS_TABLE_COLUMN_NAMES.ID} = ?`, + jobId, + ); + } + + /** + * Waits for the job to complete. + * @param {number} jobId + * @returns {Promise} + * @throws {Error} on MySQL error, if the job wasn't found in the database, if the job was + * cancelled, or if the job completed in an unexpected state. + */ + async awaitJobCompletion(jobId) { + while (true) { + let rows; + try { + const [queryRows, _] = await this.#sqlDbConnection.query( + `SELECT ${SEARCH_JOBS_TABLE_COLUMN_NAMES.STATUS} + FROM ${this.#searchJobsTableName} + WHERE ${SEARCH_JOBS_TABLE_COLUMN_NAMES.ID} = ?`, + jobId, + ); + rows = queryRows; + } catch (e) { + throw new Error(`Failed to query status for job ${jobId} - ${e}`); + } + if (rows.length < 1) { + throw new Error(`Job ${jobId} not found in database.`); + } + const status = rows[0][SEARCH_JOBS_TABLE_COLUMN_NAMES.STATUS]; + + if (false === JOB_STATUS_WAITING_STATES.includes(status)) { + if (JobStatus.CANCELLED === status) { + throw new Error(`Job ${jobId} was cancelled.`); + } else if (JobStatus.SUCCESS !== status) { + throw new Error(`Job ${jobId} exited with unexpected status=${status}: ` + `${Object.keys(JobStatus)[status]}.`); + } + break; + } + + await sleep(0.5); + } + } +} + +export default SearchJobsDbManager; diff --git a/components/webui/imports/api/search/server/methods.js b/components/webui/imports/api/search/server/methods.js index 300ff83a0..fa6234335 100644 --- a/components/webui/imports/api/search/server/methods.js +++ b/components/webui/imports/api/search/server/methods.js @@ -3,7 +3,46 @@ import {Meteor} from "meteor/meteor"; import {SearchResultsMetadataCollection} from "../collections"; import {searchJobCollectionsManager} from "./collections"; import {SearchSignal} from "../constants"; -import {cancelQuery, submitQuery, waitTillJobFinishes} from "./sql"; +import SearchJobsDbManager from "./SearchJobsDbManager"; + +/** + * @type {SearchJobsDbManager|null} + */ +let searchJobsDbManager = null; + +/** + * @param {string} dbHost + * @param {number} dbPort + * @param {string} dbName + * @param {string} dbUser + * @param {string} dbPassword + * @param {string} searchJobsTableName + * @returns {Promise} + * @throws {Error} on error. + */ +const initSearchJobsDbManager = async ({dbHost, dbPort, dbName, dbUser, dbPassword, searchJobsTableName}) => { + try { + searchJobsDbManager = await SearchJobsDbManager.createNew({ + dbHost: dbHost, + dbPort: dbPort, + dbName: dbName, + dbUser: dbUser, + dbPassword: dbPassword, + searchJobsTableName: searchJobsTableName, + }); + await searchJobsDbManager.connect(); + } catch (e) { + logger.error("Unable to create MySQL / mariadb connection.", e.toString()); + throw e; + } +}; + +const deinitSearchJobsDbManager = async () => { + if (null !== searchJobsDbManager) { + await searchJobsDbManager.disconnect(); + searchJobsDbManager = null; + } +}; /** * Updates the search event when the specified job finishes. @@ -11,7 +50,12 @@ import {cancelQuery, submitQuery, waitTillJobFinishes} from "./sql"; * @param {number} jobId of the job to monitor */ const updateSearchEventWhenJobFinishes = async (jobId) => { - const errorMsg = await waitTillJobFinishes(jobId); + let errorMsg; + try { + await searchJobsDbManager.awaitJobCompletion(jobId); + } catch (e) { + errorMsg = e.message; + } const filter = { _id: jobId.toString() }; @@ -62,8 +106,6 @@ Meteor.methods({ timestampBegin, timestampEnd, }) { - let jobId = null; - const args = { query_string: queryString, begin_timestamp: timestampBegin, @@ -71,20 +113,26 @@ Meteor.methods({ }; logger.info("search.submitQuery args =", args) - jobId = await submitQuery(args); - if (null !== jobId) { - SearchResultsMetadataCollection.insert({ - _id: jobId.toString(), - lastSignal: SearchSignal.RESP_QUERYING, - errorMsg: null - }); + let jobId; + try { + jobId = await searchJobsDbManager.submitQuery(args); + } catch (e) { + const errorMsg = "Unable to submit search job to the SQL database."; + logger.error(errorMsg, e.toString()); + throw new Meteor.Error("query-submit-error", errorMsg); + } + + SearchResultsMetadataCollection.insert({ + _id: jobId.toString(), + lastSignal: SearchSignal.RESP_QUERYING, + errorMsg: null + }); - Meteor.defer(async () => { - await updateSearchEventWhenJobFinishes(jobId); - }); + Meteor.defer(async () => { + await updateSearchEventWhenJobFinishes(jobId); + }); - await createMongoIndexes(jobId); - } + await createMongoIndexes(jobId); return {jobId}; }, @@ -106,7 +154,9 @@ Meteor.methods({ searchJobCollectionsManager.removeCollection(jobId); } catch (e) { - logger.error(`Unable to clear search results for jobId=${jobId}`, e); + const errorMsg = `Failed to clear search results for jobId ${jobId}.`; + logger.error(errorMsg, e.toString()); + throw new Meteor.Error("clear-results-error", errorMsg); } }, @@ -121,9 +171,13 @@ Meteor.methods({ logger.info("search.cancelOperation jobId =", jobId) try { - await cancelQuery(jobId); + await searchJobsDbManager.submitQueryCancellation(jobId); } catch (e) { - logger.error(`Unable to cancel search operation for jobId=${jobId}`, e); + const errorMsg = `Failed to submit cancel request for job ${jobId}.`; + logger.error(errorMsg, e.toString()); + throw new Meteor.Error("query-cancel-error", errorMsg); } }, }); + +export {deinitSearchJobsDbManager, initSearchJobsDbManager}; diff --git a/components/webui/imports/api/search/server/sql.js b/components/webui/imports/api/search/server/sql.js deleted file mode 100644 index 680cd679e..000000000 --- a/components/webui/imports/api/search/server/sql.js +++ /dev/null @@ -1,134 +0,0 @@ -import msgpack from "@msgpack/msgpack"; -import mysql from "mysql2/promise"; - -import {logger} from "/imports/utils/logger"; -import {sleep} from "../../../utils/misc"; -import {JOB_STATUS_WAITING_STATES, JobStatus} from "../constants"; - -const SEARCH_JOBS_TABLE_NAME = Meteor.settings.private.SqlDbSearchJobsTableName; -const SEARCH_JOBS_TABLE_COLUMN_NAMES = { - ID: "id", - STATUS: "status", - SEARCH_CONFIG: "search_config", -}; - -/** - * SQL connection for submitting queries to the backend - * @type {mysql.Connection} - */ -export let SQL_CONNECTION = null; - -/** - * Initializes the SQL database connection. - * - * @throws {Error} if unable to connect to the SQL database. - */ -export const initSql = async (host, port, database, user, password) => { - try { - SQL_CONNECTION = await mysql.createConnection({ - host: host, - port: port, - database: database, - user: user, - password: password, - }); - await SQL_CONNECTION.connect(); - } catch (e) { - logger.error(`Unable to create MySQL / mariadb connection with ` + - `host=${host}, port=${port}, database=${database}, user=${user}: ` + - `error=${e}`); - throw e; - } -}; - -/** - * De-initializes the SQL database connection if it is initialized. - */ -export const deinitSql = async () => { - if (null !== SQL_CONNECTION) { - await SQL_CONNECTION.end(); - SQL_CONNECTION = null; - } -}; - -/** - * Submits a query job to the SQL database and returns the job ID. - * - * @param {Object} args containing the search configuration. - * @returns {number|null} job ID on successful submission, or null in case of failure - */ -export const submitQuery = async (args) => { - let jobId = null; - - try { - const [queryInsertResults] = await SQL_CONNECTION.query( - `INSERT INTO ${SEARCH_JOBS_TABLE_NAME} - (${SEARCH_JOBS_TABLE_COLUMN_NAMES.SEARCH_CONFIG}) - VALUES (?)`, - [Buffer.from(msgpack.encode(args))], - ); - jobId = queryInsertResults.insertId; - } catch (e) { - logger.error("Unable to submit query job to SQL DB", e); - } - - return jobId; -}; - -/** - * Waits for a job to finish and retrieves its status from the database. - * - * @param {number} jobId of the job to monitor - * - * @returns {?string} null if the job completes successfully; an error message if the job exits - * in an unexpected status or encounters an error during monitoring - */ -export const waitTillJobFinishes = async (jobId) => { - let errorMsg = null; - - try { - while (true) { - const [rows, _] = await SQL_CONNECTION.query( - `SELECT ${SEARCH_JOBS_TABLE_COLUMN_NAMES.STATUS} - FROM ${SEARCH_JOBS_TABLE_NAME} - WHERE ${SEARCH_JOBS_TABLE_COLUMN_NAMES.ID} = ${jobId}` - ); - const status = rows[0][SEARCH_JOBS_TABLE_COLUMN_NAMES.STATUS]; - - if (false === JOB_STATUS_WAITING_STATES.includes(status)) { - logger.info(`Job ${jobId} exited with status=${status}: ${Object.keys( - JobStatus)[status]}.`); - - if (JobStatus.CANCELLED === status) { - errorMsg = `Job was cancelled.`; - } else if (JobStatus.SUCCESS !== status) { - errorMsg = `Job exited with unexpected status=${status}: ${Object.keys( - JobStatus)[status]}.`; - } - - break; - } - - await sleep(0.5); - } - } catch (e) { - errorMsg = `Error querying job status for jobId=${jobId}: ${e}`; - logger.error(errorMsg); - } - - return errorMsg; -}; - -/** - * Cancels a job by updating its status to 'CANCELLING' in the database. - * - * @param {string} jobId of the job to be cancelled - */ -export const cancelQuery = async (jobId) => { - await SQL_CONNECTION.query( - `UPDATE ${SEARCH_JOBS_TABLE_NAME} - SET ${SEARCH_JOBS_TABLE_COLUMN_NAMES.STATUS} = ${JobStatus.CANCELLING} - WHERE ${SEARCH_JOBS_TABLE_COLUMN_NAMES.ID} = (?)`, - [jobId] - ); -}; diff --git a/components/webui/server/main.js b/components/webui/server/main.js index 74f5e365c..2723e15a6 100644 --- a/components/webui/server/main.js +++ b/components/webui/server/main.js @@ -5,8 +5,11 @@ import "/imports/api/search/server/methods"; import "/imports/api/search/server/publications"; import "/imports/api/user/server/methods"; -import {initSql, deinitSql} from "/imports/api/search/server/sql"; import {initSearchEventCollection} from "/imports/api/search/collections"; +import { + deinitSearchJobsDbManager, + initSearchJobsDbManager, +} from "/imports/api/search/server/methods"; import {initLogger} from "/imports/utils/logger"; const DEFAULT_LOGS_DIR = "."; @@ -45,18 +48,19 @@ Meteor.startup(async () => { initLogger(envVars.WEBUI_LOGS_DIR, envVars.WEBUI_LOGGING_LEVEL, Meteor.isDevelopment); - await initSql( - Meteor.settings.private.SqlDbHost, - Meteor.settings.private.SqlDbPort, - Meteor.settings.private.SqlDbName, - envVars.CLP_DB_USER, - envVars.CLP_DB_PASS, - ); + await initSearchJobsDbManager({ + dbHost: Meteor.settings.private.SqlDbHost, + dbPort: Meteor.settings.private.SqlDbPort, + dbName: Meteor.settings.private.SqlDbName, + dbUser: envVars.CLP_DB_USER, + dbPassword: envVars.CLP_DB_PASS, + searchJobsTableName: Meteor.settings.private.SqlDbSearchJobsTableName, + }) initSearchEventCollection(); }); process.on("exit", async (code) => { console.log(`Node.js is about to exit with code: ${code}`); - await deinitSql(); + await deinitSearchJobsDbManager(); }); From bc4665b693e8744f2b867353ff11cf5099041889 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 7 Feb 2024 07:37:43 -0500 Subject: [PATCH 37/60] Minor error handling improvements. --- components/webui/imports/ui/SearchView/SearchView.jsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/webui/imports/ui/SearchView/SearchView.jsx b/components/webui/imports/ui/SearchView/SearchView.jsx index 344dbbc2f..cdd1b45a9 100644 --- a/components/webui/imports/ui/SearchView/SearchView.jsx +++ b/components/webui/imports/ui/SearchView/SearchView.jsx @@ -127,7 +127,9 @@ const SearchView = () => { }; Meteor.call("search.submitQuery", args, (error, result) => { if (error) { + setJobId(INVALID_JOB_ID); setOperationErrorMsg(error.reason); + return; } setJobId(result["jobId"]); @@ -148,6 +150,7 @@ const SearchView = () => { Meteor.call("search.clearResults", args, (error) => { if (error) { setOperationErrorMsg(error.reason); + return; } if (SearchSignal.REQ_CLEARING === localLastSearchSignalRef.current) { From 0ed94339d63c7da5d996c01634ae21c0a3ac75cf Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 7 Feb 2024 07:38:04 -0500 Subject: [PATCH 38/60] Minor docstring fix. --- components/webui/launcher.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/webui/launcher.js b/components/webui/launcher.js index fe92dd72e..997b78329 100644 --- a/components/webui/launcher.js +++ b/components/webui/launcher.js @@ -3,8 +3,8 @@ * files in a specified directory for error monitoring. * To avoid duplicated installations of dependencies, use the same `node_modules` for the server - * by setting envvar NODE_PATH="./programs/server/npm/node_modules", assuming this script is - * placed under the same directory where bundled `main.js` locates. + * by setting envvar NODE_PATH="./programs/server/npm/node_modules", assuming this script is + * placed under the same directory where the bundled `main.js` is located. * * This is not intended for development use. For development, please refer to README.md in the * component root for launching a development server with Meteor-specific error messages print From bb9a67d53da37ff46b52810c9de9fa9dd79d36db Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 7 Feb 2024 08:22:21 -0500 Subject: [PATCH 39/60] Fix reference to SearchJobStatus enum. --- components/webui/imports/api/search/constants.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/webui/imports/api/search/constants.js b/components/webui/imports/api/search/constants.js index 21e8f47e7..9d7b92045 100644 --- a/components/webui/imports/api/search/constants.js +++ b/components/webui/imports/api/search/constants.js @@ -29,7 +29,8 @@ export const isSearchSignalQuerying = (s) => ( let enumJobStatus; /** - * Enum of job statuses, matching the `JobStatus` class in `job_orchestration.search_scheduler.common`. + * Enum of job statuses, matching the `SearchJobStatus` class in + * `job_orchestration.search_scheduler.constants`. * * @constant * @type {Object} From 21cd52a8fcce27ae31fbceb0a6e107a0484b81c4 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 7 Feb 2024 08:22:42 -0500 Subject: [PATCH 40/60] Refactor new tasks. --- Taskfile.yml | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index e34baf15f..4cc5ad52b 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -3,15 +3,15 @@ version: "3" vars: BUILD_DIR: "{{.TASKFILE_DIR}}/build" CORE_COMPONENT_BUILD_DIR: "{{.TASKFILE_DIR}}/build/core" - WEBUI_BUILD_DIR: "{{.TASKFILE_DIR}}/build/webui" - NODEJS_BUILD_DIR: "{{.TASKFILE_DIR}}/build/nodejs" - NODEJS_BIN_DIR: "{{.NODEJS_BUILD_DIR}}/node/bin" LINT_VENV_DIR: "{{.TASKFILE_DIR}}/.lint-venv" + NODEJS_BIN_DIR: "{{.TASKFILE_DIR}}/build/nodejs/node/bin" + NODEJS_BUILD_DIR: "{{.TASKFILE_DIR}}/build/nodejs" PACKAGE_BUILD_DIR: "{{.TASKFILE_DIR}}/build/clp-package" PACKAGE_VENV_DIR: "{{.TASKFILE_DIR}}/build/package-venv" PACKAGE_VERSION: "0.0.3-dev" PYTHON_VERSION: sh: "python3 -c \"import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')\"" + WEBUI_BUILD_DIR: "{{.TASKFILE_DIR}}/build/webui" tasks: default: @@ -115,38 +115,38 @@ tasks: dir: "components/webui" cmds: - "rm -rf '{{.WEBUI_BUILD_DIR}}'" - - "mkdir -p {{.WEBUI_BUILD_DIR}}" + - "mkdir -p '{{.WEBUI_BUILD_DIR}}'" - "meteor npm install --production" - - "meteor build --directory {{.WEBUI_BUILD_DIR}}" + - "meteor build --directory '{{.WEBUI_BUILD_DIR}}'" - >- rsync -a "{{.WEBUI_BUILD_DIR}}/bundle/" - "$PWD/settings.json" - "$PWD/launcher.js" + launcher.js + settings.json "{{.WEBUI_BUILD_DIR}}/" - "rm -rf '{{.WEBUI_BUILD_DIR}}/bundle/'" - |- - cd {{.WEBUI_BUILD_DIR}}/programs/server - PATH={{.NODEJS_BIN_DIR}}:$PATH $(readlink -f {{.NODEJS_BIN_DIR}}/npm) install + cd "{{.WEBUI_BUILD_DIR}}/programs/server" + PATH="{{.NODEJS_BIN_DIR}}":$PATH $(readlink -f "{{.NODEJS_BIN_DIR}}/npm") install sources: - "{{.WEBUI_BUILD_DIR}}/**/*" nodejs: vars: NODEJS_VERSION: "14.21.3" - NODEJS_RELEASE: "v{{.NODEJS_VERSION}}-linux-x64" - TAR_FILENAME: "node-{{.NODEJS_RELEASE}}.tar.xz" - TAR_FILE_PATH: "{{.NODEJS_BUILD_DIR}}/{{.TAR_FILENAME}}" + TAR_FILE_NAME: "node-v14.21.3-linux-x64.tar.xz" + TAR_FILE_PATH: "{{.NODEJS_BUILD_DIR}}/node-v14.21.3-linux-x64.tar.xz" cmds: - "rm -rf '{{.NODEJS_BUILD_DIR}}/node'" - - "mkdir -p {{.NODEJS_BUILD_DIR}}" + - "mkdir -p '{{.NODEJS_BUILD_DIR}}'" - >- curl -fsSL - https://nodejs.org/dist/v{{.NODEJS_VERSION}}/{{.TAR_FILENAME}} - -o {{.TAR_FILE_PATH}} - - "tar xf {{.TAR_FILE_PATH}} -C {{.NODEJS_BUILD_DIR}}" - - "mv {{.NODEJS_BUILD_DIR}}/node-{{.NODEJS_RELEASE}} {{.NODEJS_BUILD_DIR}}/node" - - "rm -f '{{.TAR_FILE_PATH}}'" + "https://nodejs.org/dist/v{{.NODEJS_VERSION}}/{{.TAR_FILE_NAME}}" + -o "{{.NODEJS_BUILD_DIR}}/{{.TAR_FILE_NAME}}" + - "tar xf '{{.NODEJS_BUILD_DIR}}/{{.TAR_FILE_NAME}}' -C '{{.NODEJS_BUILD_DIR}}'" + - >- + mv "{{.NODEJS_BUILD_DIR}}/node-v{{.NODEJS_VERSION}}-linux-x64" "{{.NODEJS_BUILD_DIR}}/node" + - "rm -f '{{.NODEJS_BUILD_DIR}}/{{.TAR_FILE_NAME}}'" sources: - "{{.NODEJS_BUILD_DIR}}/node/**/*" From 4a78012576f95be8e60f5046bbe411893c70b546 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 7 Feb 2024 08:26:39 -0500 Subject: [PATCH 41/60] Use correct path for webui build dir. --- Taskfile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Taskfile.yml b/Taskfile.yml index 4cc5ad52b..13e66cb2e 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -99,8 +99,8 @@ tasks: - "{{.CORE_COMPONENT_BUILD_DIR}}/clo" - "{{.CORE_COMPONENT_BUILD_DIR}}/clp" - "{{.CORE_COMPONENT_BUILD_DIR}}/clp-s" - - "{{.BUILD_DIR}}/webui/built/**/*" - "{{.TASKFILE_DIR}}/Taskfile.yml" + - "{{.WEBUI_BUILD_DIR}}/**/*" - "components/clp-package-utils/dist/*.whl" - "components/clp-py-utils/dist/*.whl" - "components/job-orchestration/dist/*.whl" From 393c1d7fd4ca27fb1310f4bb1f2eff131b3784e5 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 7 Feb 2024 08:28:14 -0500 Subject: [PATCH 42/60] Reorder new tasks. --- Taskfile.yml | 82 ++++++++++++++++++++++++++-------------------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 13e66cb2e..db7fc2f03 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -109,47 +109,6 @@ tasks: - "test -e '{{.PACKAGE_VERSION_FILE}}'" - "test {{.TIMESTAMP | unixEpoch}} -lt $(stat --format %Y '{{.PACKAGE_VERSION_FILE}}')" - webui: - deps: - - "nodejs" - dir: "components/webui" - cmds: - - "rm -rf '{{.WEBUI_BUILD_DIR}}'" - - "mkdir -p '{{.WEBUI_BUILD_DIR}}'" - - "meteor npm install --production" - - "meteor build --directory '{{.WEBUI_BUILD_DIR}}'" - - >- - rsync -a - "{{.WEBUI_BUILD_DIR}}/bundle/" - launcher.js - settings.json - "{{.WEBUI_BUILD_DIR}}/" - - "rm -rf '{{.WEBUI_BUILD_DIR}}/bundle/'" - - |- - cd "{{.WEBUI_BUILD_DIR}}/programs/server" - PATH="{{.NODEJS_BIN_DIR}}":$PATH $(readlink -f "{{.NODEJS_BIN_DIR}}/npm") install - sources: - - "{{.WEBUI_BUILD_DIR}}/**/*" - - nodejs: - vars: - NODEJS_VERSION: "14.21.3" - TAR_FILE_NAME: "node-v14.21.3-linux-x64.tar.xz" - TAR_FILE_PATH: "{{.NODEJS_BUILD_DIR}}/node-v14.21.3-linux-x64.tar.xz" - cmds: - - "rm -rf '{{.NODEJS_BUILD_DIR}}/node'" - - "mkdir -p '{{.NODEJS_BUILD_DIR}}'" - - >- - curl -fsSL - "https://nodejs.org/dist/v{{.NODEJS_VERSION}}/{{.TAR_FILE_NAME}}" - -o "{{.NODEJS_BUILD_DIR}}/{{.TAR_FILE_NAME}}" - - "tar xf '{{.NODEJS_BUILD_DIR}}/{{.TAR_FILE_NAME}}' -C '{{.NODEJS_BUILD_DIR}}'" - - >- - mv "{{.NODEJS_BUILD_DIR}}/node-v{{.NODEJS_VERSION}}-linux-x64" "{{.NODEJS_BUILD_DIR}}/node" - - "rm -f '{{.NODEJS_BUILD_DIR}}/{{.TAR_FILE_NAME}}'" - sources: - - "{{.NODEJS_BUILD_DIR}}/node/**/*" - core: deps: ["core-submodules"] vars: @@ -195,6 +154,47 @@ tasks: vars: COMPONENT: "{{.TASK}}" + nodejs: + vars: + NODEJS_VERSION: "14.21.3" + TAR_FILE_NAME: "node-v14.21.3-linux-x64.tar.xz" + TAR_FILE_PATH: "{{.NODEJS_BUILD_DIR}}/node-v14.21.3-linux-x64.tar.xz" + cmds: + - "rm -rf '{{.NODEJS_BUILD_DIR}}/node'" + - "mkdir -p '{{.NODEJS_BUILD_DIR}}'" + - >- + curl -fsSL + "https://nodejs.org/dist/v{{.NODEJS_VERSION}}/{{.TAR_FILE_NAME}}" + -o "{{.NODEJS_BUILD_DIR}}/{{.TAR_FILE_NAME}}" + - "tar xf '{{.NODEJS_BUILD_DIR}}/{{.TAR_FILE_NAME}}' -C '{{.NODEJS_BUILD_DIR}}'" + - >- + mv "{{.NODEJS_BUILD_DIR}}/node-v{{.NODEJS_VERSION}}-linux-x64" "{{.NODEJS_BUILD_DIR}}/node" + - "rm -f '{{.NODEJS_BUILD_DIR}}/{{.TAR_FILE_NAME}}'" + sources: + - "{{.NODEJS_BUILD_DIR}}/node/**/*" + + webui: + deps: + - "nodejs" + dir: "components/webui" + cmds: + - "rm -rf '{{.WEBUI_BUILD_DIR}}'" + - "mkdir -p '{{.WEBUI_BUILD_DIR}}'" + - "meteor npm install --production" + - "meteor build --directory '{{.WEBUI_BUILD_DIR}}'" + - >- + rsync -a + "{{.WEBUI_BUILD_DIR}}/bundle/" + launcher.js + settings.json + "{{.WEBUI_BUILD_DIR}}/" + - "rm -rf '{{.WEBUI_BUILD_DIR}}/bundle/'" + - |- + cd "{{.WEBUI_BUILD_DIR}}/programs/server" + PATH="{{.NODEJS_BIN_DIR}}":$PATH $(readlink -f "{{.NODEJS_BIN_DIR}}/npm") install + sources: + - "{{.WEBUI_BUILD_DIR}}/**/*" + lint-check: cmds: - task: "core-lint-check" From 7e3a6deb675ea7062bcbeef0564422eb5dd5d81c Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 7 Feb 2024 08:33:33 -0500 Subject: [PATCH 43/60] Update packages. --- components/webui/package-lock.json | 442 +++++++++++------------------ components/webui/package.json | 16 +- 2 files changed, 176 insertions(+), 282 deletions(-) diff --git a/components/webui/package-lock.json b/components/webui/package-lock.json index 3450bb13e..2c1b456c6 100644 --- a/components/webui/package-lock.json +++ b/components/webui/package-lock.json @@ -4,9 +4,9 @@ "lockfileVersion": 1, "dependencies": { "@babel/runtime": { - "version": "7.23.1", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.23.1.tgz", - "integrity": "sha512-hC2v6p8ZSI/W0HUzh3V8C5g+NwSKzKPtJwSpTjwl0o297GP9+ZLQSkdvHz46CM3LqyoXxq+5G9komY+eSqSO0g==", + "version": "7.23.9", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.23.9.tgz", + "integrity": "sha512-0CX6F+BI2s9dkUqr08KFrAIZgNFj75rdBU/DjCyYLIaV/quFjkk6T+EJ2LkZHyZTbEV4L5p97mNkUsHl2wLFAw==", "requires": { "regenerator-runtime": "^0.14.0" } @@ -27,24 +27,24 @@ } }, "@fortawesome/fontawesome-common-types": { - "version": "6.4.2", - "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-common-types/-/fontawesome-common-types-6.4.2.tgz", - "integrity": "sha512-1DgP7f+XQIJbLFCTX1V2QnxVmpLdKdzzo2k8EmvDOePfchaIGQ9eCHj2up3/jNEbZuBqel5OxiaOJf37TWauRA==" + "version": "6.5.1", + "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-common-types/-/fontawesome-common-types-6.5.1.tgz", + "integrity": "sha512-GkWzv+L6d2bI5f/Vk6ikJ9xtl7dfXtoRu3YGE6nq0p/FFqA1ebMOAWg3XgRyb0I6LYyYkiAo+3/KrwuBp8xG7A==" }, "@fortawesome/fontawesome-svg-core": { - "version": "6.4.2", - "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-svg-core/-/fontawesome-svg-core-6.4.2.tgz", - "integrity": "sha512-gjYDSKv3TrM2sLTOKBc5rH9ckje8Wrwgx1CxAPbN5N3Fm4prfi7NsJVWd1jklp7i5uSCVwhZS5qlhMXqLrpAIg==", + "version": "6.5.1", + "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-svg-core/-/fontawesome-svg-core-6.5.1.tgz", + "integrity": "sha512-MfRCYlQPXoLlpem+egxjfkEuP9UQswTrlCOsknus/NcMoblTH2g0jPrapbcIb04KGA7E2GZxbAccGZfWoYgsrQ==", "requires": { - "@fortawesome/fontawesome-common-types": "6.4.2" + "@fortawesome/fontawesome-common-types": "6.5.1" } }, "@fortawesome/free-solid-svg-icons": { - "version": "6.4.2", - "resolved": "https://registry.npmjs.org/@fortawesome/free-solid-svg-icons/-/free-solid-svg-icons-6.4.2.tgz", - "integrity": "sha512-sYwXurXUEQS32fZz9hVCUUv/xu49PEJEyUOsA51l6PU/qVgfbTb2glsTEaJngVVT8VqBATRIdh7XVgV1JF1LkA==", + "version": "6.5.1", + "resolved": "https://registry.npmjs.org/@fortawesome/free-solid-svg-icons/-/free-solid-svg-icons-6.5.1.tgz", + "integrity": "sha512-S1PPfU3mIJa59biTtXJz1oI0+KAXW6bkAb31XKhxdxtuXDiUIFsih4JR1v5BbxY7hVHsD1RKq+jRkVRaf773NQ==", "requires": { - "@fortawesome/fontawesome-common-types": "6.4.2" + "@fortawesome/fontawesome-common-types": "6.5.1" } }, "@fortawesome/react-fontawesome": { @@ -66,17 +66,17 @@ "integrity": "sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A==" }, "@react-aria/ssr": { - "version": "3.8.0", - "resolved": "https://registry.npmjs.org/@react-aria/ssr/-/ssr-3.8.0.tgz", - "integrity": "sha512-Y54xs483rglN5DxbwfCPHxnkvZ+gZ0LbSYmR72LyWPGft8hN/lrl1VRS1EW2SMjnkEWlj+Km2mwvA3kEHDUA0A==", + "version": "3.9.1", + "resolved": "https://registry.npmjs.org/@react-aria/ssr/-/ssr-3.9.1.tgz", + "integrity": "sha512-NqzkLFP8ZVI4GSorS0AYljC13QW2sc8bDqJOkBvkAt3M8gbcAXJWVRGtZBCRscki9RZF+rNlnPdg0G0jYkhJcg==", "requires": { "@swc/helpers": "^0.5.0" } }, "@restart/hooks": { - "version": "0.4.11", - "resolved": "https://registry.npmjs.org/@restart/hooks/-/hooks-0.4.11.tgz", - "integrity": "sha512-Ft/ncTULZN6ldGHiF/k5qt72O8JyRMOeg0tApvCni8LkoiEahO+z3TNxfXIVGy890YtWVDvJAl662dVJSJXvMw==", + "version": "0.4.15", + "resolved": "https://registry.npmjs.org/@restart/hooks/-/hooks-0.4.15.tgz", + "integrity": "sha512-cZFXYTxbpzYcieq/mBwSyXgqnGMHoBVh3J7MU0CCoIB4NRZxV9/TuwTBAaLMqpNhC3zTPMCgkQ5Ey07L02Xmcw==", "requires": { "dequal": "^2.0.3" } @@ -105,22 +105,22 @@ } }, "@swc/helpers": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.2.tgz", - "integrity": "sha512-E4KcWTpoLHqwPHLxidpOqQbcrZVgi0rsmmZXUle1jXmJfuIf/UWpczUJ7MZZ5tlxytgJXyp0w4PGkkeLiuIdZw==", + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.6.tgz", + "integrity": "sha512-aYX01Ke9hunpoCexYAgQucEpARGQ5w/cqHFrIR+e9gdKb1QWTsVJuTJ2ozQzIAxLyRQe/m+2RqzkyOOGiMKRQA==", "requires": { "tslib": "^2.4.0" } }, "@types/prop-types": { - "version": "15.7.8", - "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.8.tgz", - "integrity": "sha512-kMpQpfZKSCBqltAJwskgePRaYRFukDkm1oItcAbC3gNELR20XIBcN9VRgg4+m8DKsTfkWeA4m4Imp4DDuWy7FQ==" + "version": "15.7.11", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.11.tgz", + "integrity": "sha512-ga8y9v9uyeiLdpKddhxYQkxNDrfvuPrlFb0N1qnZZByvcElJaXthF1UhvCh9TLWJBEHeNtdnbysW7Y6Uq8CVng==" }, "@types/react": { - "version": "18.2.24", - "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.24.tgz", - "integrity": "sha512-Ee0Jt4sbJxMu1iDcetZEIKQr99J1Zfb6D4F3qfUWoR1JpInkY1Wdg4WwCyBjL257D0+jGqSl1twBjV8iCaC0Aw==", + "version": "18.2.55", + "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.55.tgz", + "integrity": "sha512-Y2Tz5P4yz23brwm2d7jNon39qoAtMMmalOQv6+fEFt1mT+FcM3D841wDpoUvFXhaYenuROCy3FZYqdTjM7qVyA==", "requires": { "@types/prop-types": "*", "@types/scheduler": "*", @@ -128,17 +128,17 @@ } }, "@types/react-transition-group": { - "version": "4.4.7", - "resolved": "https://registry.npmjs.org/@types/react-transition-group/-/react-transition-group-4.4.7.tgz", - "integrity": "sha512-ICCyBl5mvyqYp8Qeq9B5G/fyBSRC0zx3XM3sCC6KkcMsNeAHqXBKkmat4GqdJET5jtYUpZXrxI5flve5qhi2Eg==", + "version": "4.4.10", + "resolved": "https://registry.npmjs.org/@types/react-transition-group/-/react-transition-group-4.4.10.tgz", + "integrity": "sha512-hT/+s0VQs2ojCX823m60m5f0sL5idt9SO6Tj6Dg+rdphGPIeJbJ6CxvBYkgkGKrYeDjvIpKTR38UzmtHJOGW3Q==", "requires": { "@types/react": "*" } }, "@types/scheduler": { - "version": "0.16.4", - "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.4.tgz", - "integrity": "sha512-2L9ifAGl7wmXwP4v3pN4p2FLhD0O1qsJpvKmNin5VA8+UvNVb447UDaAEV6UdrkA+m/Xs58U1RFps44x6TFsVQ==" + "version": "0.16.8", + "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.8.tgz", + "integrity": "sha512-WZLiwShhwLRmeV6zH+GkbOFT6Z6VklCItrDioxUnv+u4Ll+8vKeFySoFyK/0ctcRpOmwAicELfmys1sDc/Rw+A==" }, "@types/triple-beam": { "version": "1.3.5", @@ -146,9 +146,9 @@ "integrity": "sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==" }, "@types/warning": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@types/warning/-/warning-3.0.1.tgz", - "integrity": "sha512-ywJmriP+nvjBKNBEMaNZgj2irZHoxcKeYcyMLbqhYKbDVn8yCIULy2Ol/tvIb37O3IBeZj3RU4tXqQTtGwoAMg==" + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/warning/-/warning-3.0.3.tgz", + "integrity": "sha512-D1XC7WK8K+zZEveUPY+cf4+kgauk8N4eHr/XIHXGlGYkHLud6hK9lYfZk1ry1TNh798cZUCgb6MqGEG8DkJt6Q==" }, "async": { "version": "3.2.5", @@ -161,9 +161,9 @@ "integrity": "sha512-D32nmNWiQHo94BKHLmOrdjlL05q1c8oxbtBphQFb9Z5to6eGRDCm0QgeaZ4zFBHzfg2++rqa2JkqCcxDy0sH0g==" }, "classnames": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.3.2.tgz", - "integrity": "sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw==" + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.5.1.tgz", + "integrity": "sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow==" }, "color": { "version": "3.2.1", @@ -206,9 +206,9 @@ } }, "csstype": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.2.tgz", - "integrity": "sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ==" + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz", + "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==" }, "date-fns": { "version": "2.30.0", @@ -377,14 +377,14 @@ "integrity": "sha512-MhWWlVnuab1RG5/zMRRcVGXZLCXrZTgfwMikgzCegsPnG62yDQo5JnqKkrK4jO5iKqDAZGItAqN5CtKBCBWRUA==" }, "luxon": { - "version": "3.4.3", - "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.3.tgz", - "integrity": "sha512-tFWBiv3h7z+T/tDaoxA8rqTxy1CHV6gHS//QdaH4pulbq/JuBSGgQspQQqcgnwdAx6pNI7cmvz5Sv/addzHmUg==" + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz", + "integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==" }, "meteor-node-stubs": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/meteor-node-stubs/-/meteor-node-stubs-1.2.5.tgz", - "integrity": "sha512-FLlOFZx3KnZ5s3yPCK+x58DyX9ewN+oQ12LcpwBXMLtzJ/YyprMQVivd6KIrahZbKJrNenPNUGuDS37WUFg+Mw==", + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/meteor-node-stubs/-/meteor-node-stubs-1.2.7.tgz", + "integrity": "sha512-20bAFUhEIOD/Cos2nmvhqf2NOKpTf63WVQ+nwuaX2OFj31sU6GL4KkNylkWum8McwsH0LsMr/F+UHhduTX7KRg==", "requires": { "assert": "^2.0.0", "browserify-zlib": "^0.2.0", @@ -414,8 +414,7 @@ "dependencies": { "asn1.js": { "version": "5.4.1", - "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz", - "integrity": "sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==", + "bundled": true, "requires": { "bn.js": "^4.0.0", "inherits": "^2.0.1", @@ -425,15 +424,13 @@ "dependencies": { "bn.js": { "version": "4.12.0", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", - "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + "bundled": true } } }, "assert": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/assert/-/assert-2.0.0.tgz", - "integrity": "sha512-se5Cd+js9dXJnu6Ag2JFc00t+HmHOen+8Q+L7O9zI0PqQXr20uk2J0XQqMxZEeo5U50o8Nvmmx7dZrl+Ufr35A==", + "bundled": true, "requires": { "es6-object-assign": "^1.1.0", "is-nan": "^1.2.1", @@ -443,28 +440,23 @@ }, "available-typed-arrays": { "version": "1.0.4", - "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.4.tgz", - "integrity": "sha512-SA5mXJWrId1TaQjfxUYghbqQ/hYioKmLJvPJyDuYRtXXenFNMjj4hSSt1Cf1xsuXSXrtxrVC5Ot4eU6cOtBDdA==" + "bundled": true }, "base64-js": { "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + "bundled": true }, "bn.js": { "version": "5.2.0", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.2.0.tgz", - "integrity": "sha512-D7iWRBvnZE8ecXiLj/9wbxH7Tk79fAh8IHaTNq1RWRixsS02W+5qS+iE9yq6RYl0asXx5tw0bLhmT5pIfbSquw==" + "bundled": true }, "brorand": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz", - "integrity": "sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=" + "bundled": true }, "browserify-aes": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", - "integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==", + "bundled": true, "requires": { "buffer-xor": "^1.0.3", "cipher-base": "^1.0.0", @@ -476,8 +468,7 @@ }, "browserify-cipher": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/browserify-cipher/-/browserify-cipher-1.0.1.tgz", - "integrity": "sha512-sPhkz0ARKbf4rRQt2hTpAHqn47X3llLkUGn+xEJzLjwY8LRs2p0v7ljvI5EyoRO/mexrNunNECisZs+gw2zz1w==", + "bundled": true, "requires": { "browserify-aes": "^1.0.4", "browserify-des": "^1.0.0", @@ -486,8 +477,7 @@ }, "browserify-des": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/browserify-des/-/browserify-des-1.0.2.tgz", - "integrity": "sha512-BioO1xf3hFwz4kc6iBhI3ieDFompMhrMlnDFC4/0/vd5MokpuAc3R+LYbwTA9A5Yc9pq9UYPqffKpW2ObuwX5A==", + "bundled": true, "requires": { "cipher-base": "^1.0.1", "des.js": "^1.0.0", @@ -497,8 +487,7 @@ }, "browserify-rsa": { "version": "4.1.0", - "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.1.0.tgz", - "integrity": "sha512-AdEER0Hkspgno2aR97SAf6vi0y0k8NuOpGnVH3O99rcA5Q6sh8QxcngtHuJ6uXwnfAXNM4Gn1Gb7/MV1+Ymbog==", + "bundled": true, "requires": { "bn.js": "^5.0.0", "randombytes": "^2.0.1" @@ -506,8 +495,7 @@ }, "browserify-sign": { "version": "4.2.1", - "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.2.1.tgz", - "integrity": "sha512-/vrA5fguVAKKAVTNJjgSm1tRQDHUU6DbwO9IROu/0WAzC8PKhucDSh18J0RMvVeHAn5puMd+QHC2erPRNf8lmg==", + "bundled": true, "requires": { "bn.js": "^5.1.1", "browserify-rsa": "^4.0.1", @@ -522,16 +510,14 @@ }, "browserify-zlib": { "version": "0.2.0", - "resolved": "https://registry.npmjs.org/browserify-zlib/-/browserify-zlib-0.2.0.tgz", - "integrity": "sha512-Z942RysHXmJrhqk88FmKBVq/v5tqmSkDz7p54G/MGyjMnCFFnC79XWNbg+Vta8W6Wb2qtSZTSxIGkJrRpCFEiA==", + "bundled": true, "requires": { "pako": "~1.0.5" } }, "buffer": { "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "bundled": true, "requires": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" @@ -539,18 +525,15 @@ }, "buffer-xor": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz", - "integrity": "sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk=" + "bundled": true }, "builtin-status-codes": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz", - "integrity": "sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug=" + "bundled": true }, "call-bind": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", - "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "bundled": true, "requires": { "function-bind": "^1.1.1", "get-intrinsic": "^1.0.2" @@ -558,8 +541,7 @@ }, "cipher-base": { "version": "1.0.4", - "resolved": "https://registry.npmjs.org/cipher-base/-/cipher-base-1.0.4.tgz", - "integrity": "sha512-Kkht5ye6ZGmwv40uUDZztayT2ThLQGfnj/T71N/XzeZeo3nf8foyW7zGTsPYkEya3m5f3cAypH+qe7YOrM1U2Q==", + "bundled": true, "requires": { "inherits": "^2.0.1", "safe-buffer": "^5.0.1" @@ -567,18 +549,15 @@ }, "console-browserify": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/console-browserify/-/console-browserify-1.2.0.tgz", - "integrity": "sha512-ZMkYO/LkF17QvCPqM0gxw8yUzigAOZOSWSHg91FH6orS7vcEj5dVZTidN2fQ14yBSdg97RqhSNwLUXInd52OTA==" + "bundled": true }, "constants-browserify": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/constants-browserify/-/constants-browserify-1.0.0.tgz", - "integrity": "sha1-wguW2MYXdIqvHBYCF2DNJ/y4y3U=" + "bundled": true }, "create-ecdh": { "version": "4.0.4", - "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.4.tgz", - "integrity": "sha512-mf+TCx8wWc9VpuxfP2ht0iSISLZnt0JgWlrOKZiNqyUZWnjIaCIVNQArMHnCZKfEYRg6IM7A+NeJoN8gf/Ws0A==", + "bundled": true, "requires": { "bn.js": "^4.1.0", "elliptic": "^6.5.3" @@ -586,15 +565,13 @@ "dependencies": { "bn.js": { "version": "4.12.0", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", - "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + "bundled": true } } }, "create-hash": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", - "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==", + "bundled": true, "requires": { "cipher-base": "^1.0.1", "inherits": "^2.0.1", @@ -605,8 +582,7 @@ }, "create-hmac": { "version": "1.1.7", - "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", - "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==", + "bundled": true, "requires": { "cipher-base": "^1.0.3", "create-hash": "^1.1.0", @@ -618,8 +594,7 @@ }, "crypto-browserify": { "version": "3.12.0", - "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-3.12.0.tgz", - "integrity": "sha512-fz4spIh+znjO2VjL+IdhEpRJ3YN6sMzITSBijk6FK2UvTqruSQW+/cCZTSNsMiZNvUeq0CqurF+dAbyiGOY6Wg==", + "bundled": true, "requires": { "browserify-cipher": "^1.0.0", "browserify-sign": "^4.0.0", @@ -636,16 +611,14 @@ }, "define-properties": { "version": "1.1.3", - "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", - "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", + "bundled": true, "requires": { "object-keys": "^1.0.12" } }, "des.js": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/des.js/-/des.js-1.0.1.tgz", - "integrity": "sha512-Q0I4pfFrv2VPd34/vfLrFOoRmlYj3OV50i7fskps1jZWK1kApMWWT9G6RRUeYedLcBDIhnSDaUvJMb3AhUlaEA==", + "bundled": true, "requires": { "inherits": "^2.0.1", "minimalistic-assert": "^1.0.0" @@ -653,8 +626,7 @@ }, "diffie-hellman": { "version": "5.0.3", - "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", - "integrity": "sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==", + "bundled": true, "requires": { "bn.js": "^4.1.0", "miller-rabin": "^4.0.0", @@ -663,20 +635,17 @@ "dependencies": { "bn.js": { "version": "4.12.0", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", - "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + "bundled": true } } }, "domain-browser": { "version": "4.22.0", - "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-4.22.0.tgz", - "integrity": "sha512-IGBwjF7tNk3cwypFNH/7bfzBcgSCbaMOD3GsaY1AU/JRrnHnYgEM0+9kQt52iZxjNsjBtJYtao146V+f8jFZNw==" + "bundled": true }, "elliptic": { "version": "6.5.4", - "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.4.tgz", - "integrity": "sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ==", + "bundled": true, "requires": { "bn.js": "^4.11.9", "brorand": "^1.1.0", @@ -689,15 +658,13 @@ "dependencies": { "bn.js": { "version": "4.12.0", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", - "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + "bundled": true } } }, "es-abstract": { "version": "1.18.3", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.18.3.tgz", - "integrity": "sha512-nQIr12dxV7SSxE6r6f1l3DtAeEYdsGpps13dR0TwJg1S8gyp4ZPgy3FZcHBgbiQqnoqSTb+oC+kO4UQ0C/J8vw==", + "bundled": true, "requires": { "call-bind": "^1.0.2", "es-to-primitive": "^1.2.1", @@ -719,8 +686,7 @@ }, "es-to-primitive": { "version": "1.2.1", - "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", - "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "bundled": true, "requires": { "is-callable": "^1.1.4", "is-date-object": "^1.0.1", @@ -729,18 +695,15 @@ }, "es6-object-assign": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/es6-object-assign/-/es6-object-assign-1.1.0.tgz", - "integrity": "sha1-wsNYJlYkfDnqEHyx5mUrb58kUjw=" + "bundled": true }, "events": { "version": "3.3.0", - "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", - "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==" + "bundled": true }, "evp_bytestokey": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz", - "integrity": "sha512-/f2Go4TognH/KvCISP7OUsHn85hT9nUkxxA9BEWxFn+Oj9o8ZNLm/40hdlgSLyuOimsrTKLUMEorQexp/aPQeA==", + "bundled": true, "requires": { "md5.js": "^1.3.4", "safe-buffer": "^5.1.1" @@ -748,18 +711,15 @@ }, "foreach": { "version": "2.0.5", - "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.5.tgz", - "integrity": "sha1-C+4AUBiusmDQo6865ljdATbsG5k=" + "bundled": true }, "function-bind": { "version": "1.1.1", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + "bundled": true }, "get-intrinsic": { "version": "1.1.1", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", - "integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==", + "bundled": true, "requires": { "function-bind": "^1.1.1", "has": "^1.0.3", @@ -768,26 +728,22 @@ }, "has": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", - "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "bundled": true, "requires": { "function-bind": "^1.1.1" } }, "has-bigints": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.1.tgz", - "integrity": "sha512-LSBS2LjbNBTf6287JEbEzvJgftkF5qFkmCo9hDRpAzKhUOlJ+hx8dd4USs00SgsUNwc4617J9ki5YtEClM2ffA==" + "bundled": true }, "has-symbols": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz", - "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==" + "bundled": true }, "hash-base": { "version": "3.1.0", - "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-3.1.0.tgz", - "integrity": "sha512-1nmYp/rhMDiE7AYkDw+lLwlAzz0AntGIe51F3RfFfEqyQ3feY2eI/NcwC6umIQVOASPMsWJLJScWKSSvzL9IVA==", + "bundled": true, "requires": { "inherits": "^2.0.4", "readable-stream": "^3.6.0", @@ -796,8 +752,7 @@ }, "hash.js": { "version": "1.1.7", - "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz", - "integrity": "sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA==", + "bundled": true, "requires": { "inherits": "^2.0.3", "minimalistic-assert": "^1.0.1" @@ -805,8 +760,7 @@ }, "hmac-drbg": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", - "integrity": "sha1-0nRXAQJabHdabFRXk+1QL8DGSaE=", + "bundled": true, "requires": { "hash.js": "^1.0.3", "minimalistic-assert": "^1.0.0", @@ -815,59 +769,49 @@ }, "https-browserify": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz", - "integrity": "sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM=" + "bundled": true }, "ieee754": { "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + "bundled": true }, "inherits": { "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + "bundled": true }, "is-arguments": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.0.tgz", - "integrity": "sha512-1Ij4lOMPl/xB5kBDn7I+b2ttPMKa8szhEIrXDuXQD/oe3HJLTLhqhgGspwgyGd6MOywBUqVvYicF72lkgDnIHg==", + "bundled": true, "requires": { "call-bind": "^1.0.0" } }, "is-bigint": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.2.tgz", - "integrity": "sha512-0JV5+SOCQkIdzjBK9buARcV804Ddu7A0Qet6sHi3FimE9ne6m4BGQZfRn+NZiXbBk4F4XmHfDZIipLj9pX8dSA==" + "bundled": true }, "is-boolean-object": { "version": "1.1.1", - "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.1.tgz", - "integrity": "sha512-bXdQWkECBUIAcCkeH1unwJLIpZYaa5VvuygSyS/c2lf719mTKZDU5UdDRlpd01UjADgmW8RfqaP+mRaVPdr/Ng==", + "bundled": true, "requires": { "call-bind": "^1.0.2" } }, "is-callable": { "version": "1.2.3", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.3.tgz", - "integrity": "sha512-J1DcMe8UYTBSrKezuIUTUwjXsho29693unXM2YhJUTR2txK/eG47bvNa/wipPFmZFgr/N6f1GA66dv0mEyTIyQ==" + "bundled": true }, "is-date-object": { "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.4.tgz", - "integrity": "sha512-/b4ZVsG7Z5XVtIxs/h9W8nvfLgSAyKYdtGWQLbqy6jA1icmgjf8WCoTKgeS4wy5tYaPePouzFMANbnj94c2Z+A==" + "bundled": true }, "is-generator-function": { "version": "1.0.9", - "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.9.tgz", - "integrity": "sha512-ZJ34p1uvIfptHCN7sFTjGibB9/oBg17sHqzDLfuwhvmN/qLVvIQXRQ8licZQ35WJ8KuEQt/etnnzQFI9C9Ue/A==" + "bundled": true }, "is-nan": { "version": "1.3.2", - "resolved": "https://registry.npmjs.org/is-nan/-/is-nan-1.3.2.tgz", - "integrity": "sha512-E+zBKpQ2t6MEo1VsonYmluk9NxGrbzpeeLC2xIViuO2EjU2xsXsBPwTr3Ykv9l08UYEVEdWeRZNouaZqF6RN0w==", + "bundled": true, "requires": { "call-bind": "^1.0.0", "define-properties": "^1.1.3" @@ -875,18 +819,15 @@ }, "is-negative-zero": { "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.1.tgz", - "integrity": "sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w==" + "bundled": true }, "is-number-object": { "version": "1.0.5", - "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.5.tgz", - "integrity": "sha512-RU0lI/n95pMoUKu9v1BZP5MBcZuNSVJkMkAG2dJqC4z2GlkGUNeH68SuHuBKBD/XFe+LHZ+f9BKkLET60Niedw==" + "bundled": true }, "is-regex": { "version": "1.1.3", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.3.tgz", - "integrity": "sha512-qSVXFz28HM7y+IWX6vLCsexdlvzT1PJNFSBuaQLQ5o0IEw8UDYW6/2+eCMVyIsbM8CNLX2a/QWmSpyxYEHY7CQ==", + "bundled": true, "requires": { "call-bind": "^1.0.2", "has-symbols": "^1.0.2" @@ -894,21 +835,18 @@ }, "is-string": { "version": "1.0.6", - "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.6.tgz", - "integrity": "sha512-2gdzbKUuqtQ3lYNrUTQYoClPhm7oQu4UdpSZMp1/DGgkHBT8E2Z1l0yMdb6D4zNAxwDiMv8MdulKROJGNl0Q0w==" + "bundled": true }, "is-symbol": { "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz", - "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==", + "bundled": true, "requires": { "has-symbols": "^1.0.2" } }, "is-typed-array": { "version": "1.1.5", - "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.5.tgz", - "integrity": "sha512-S+GRDgJlR3PyEbsX/Fobd9cqpZBuvUS+8asRqYDMLCb2qMzt1oz5m5oxQCxOgUDxiWsOVNi4yaF+/uvdlHlYug==", + "bundled": true, "requires": { "available-typed-arrays": "^1.0.2", "call-bind": "^1.0.2", @@ -919,8 +857,7 @@ }, "md5.js": { "version": "1.3.5", - "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", - "integrity": "sha512-xitP+WxNPcTTOgnTJcrhM0xvdPepipPSf3I8EIpGKeFLjt3PlJLIDG3u8EX53ZIubkb+5U2+3rELYpEhHhzdkg==", + "bundled": true, "requires": { "hash-base": "^3.0.0", "inherits": "^2.0.1", @@ -929,8 +866,7 @@ }, "miller-rabin": { "version": "4.0.1", - "resolved": "https://registry.npmjs.org/miller-rabin/-/miller-rabin-4.0.1.tgz", - "integrity": "sha512-115fLhvZVqWwHPbClyntxEVfVDfl9DLLTuJvq3g2O/Oxi8AiNouAHvDSzHS0viUJc+V5vm3eq91Xwqn9dp4jRA==", + "bundled": true, "requires": { "bn.js": "^4.0.0", "brorand": "^1.0.1" @@ -938,30 +874,25 @@ "dependencies": { "bn.js": { "version": "4.12.0", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", - "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + "bundled": true } } }, "minimalistic-assert": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", - "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==" + "bundled": true }, "minimalistic-crypto-utils": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz", - "integrity": "sha1-9sAMHAsIIkblxNmd+4x8CDsrWCo=" + "bundled": true }, "object-inspect": { "version": "1.10.3", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.10.3.tgz", - "integrity": "sha512-e5mCJlSH7poANfC8z8S9s9S2IN5/4Zb3aZ33f5s8YqoazCFzNLloLU8r5VCG+G7WoqLvAAZoVMcy3tp/3X0Plw==" + "bundled": true }, "object-is": { "version": "1.1.5", - "resolved": "https://registry.npmjs.org/object-is/-/object-is-1.1.5.tgz", - "integrity": "sha512-3cyDsyHgtmi7I7DfSSI2LDp6SK2lwvtbg0p0R1e0RvTqF5ceGx+K2dfSjm1bKDMVCFEDAQvy+o8c6a7VujOddw==", + "bundled": true, "requires": { "call-bind": "^1.0.2", "define-properties": "^1.1.3" @@ -969,13 +900,11 @@ }, "object-keys": { "version": "1.1.1", - "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", - "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" + "bundled": true }, "object.assign": { "version": "4.1.2", - "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.2.tgz", - "integrity": "sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==", + "bundled": true, "requires": { "call-bind": "^1.0.0", "define-properties": "^1.1.3", @@ -985,18 +914,15 @@ }, "os-browserify": { "version": "0.3.0", - "resolved": "https://registry.npmjs.org/os-browserify/-/os-browserify-0.3.0.tgz", - "integrity": "sha1-hUNzx/XCMVkU/Jv8a9gjj92h7Cc=" + "bundled": true }, "pako": { "version": "1.0.11", - "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", - "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==" + "bundled": true }, "parse-asn1": { "version": "5.1.6", - "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz", - "integrity": "sha512-RnZRo1EPU6JBnra2vGHj0yhp6ebyjBZpmUCLHWiFhxlzvBCCpAuZ7elsBp1PVAbQN0/04VD/19rfzlBSwLstMw==", + "bundled": true, "requires": { "asn1.js": "^5.2.0", "browserify-aes": "^1.0.0", @@ -1007,13 +933,11 @@ }, "path-browserify": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz", - "integrity": "sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==" + "bundled": true }, "pbkdf2": { "version": "3.1.2", - "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.2.tgz", - "integrity": "sha512-iuh7L6jA7JEGu2WxDwtQP1ddOpaJNC4KlDEFfdQajSGgGPNi4OyDc2R7QnbY2bR9QjBVGwgvTdNJZoE7RaxUMA==", + "bundled": true, "requires": { "create-hash": "^1.1.2", "create-hmac": "^1.1.4", @@ -1024,13 +948,11 @@ }, "process": { "version": "0.11.10", - "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", - "integrity": "sha1-czIwDoQBYb2j5podHZGn1LwW8YI=" + "bundled": true }, "public-encrypt": { "version": "4.0.3", - "resolved": "https://registry.npmjs.org/public-encrypt/-/public-encrypt-4.0.3.tgz", - "integrity": "sha512-zVpa8oKZSz5bTMTFClc1fQOnyyEzpl5ozpi1B5YcvBrdohMjH2rfsBtyXcuNuwjsDIXmBYlF2N5FlJYhR29t8Q==", + "bundled": true, "requires": { "bn.js": "^4.1.0", "browserify-rsa": "^4.0.0", @@ -1042,38 +964,32 @@ "dependencies": { "bn.js": { "version": "4.12.0", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", - "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + "bundled": true } } }, "punycode": { "version": "1.4.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + "bundled": true }, "querystring": { "version": "0.2.0", - "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", - "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + "bundled": true }, "querystring-es3": { "version": "0.2.1", - "resolved": "https://registry.npmjs.org/querystring-es3/-/querystring-es3-0.2.1.tgz", - "integrity": "sha1-nsYfeQSYdXB9aUFFlv2Qek1xHnM=" + "bundled": true }, "randombytes": { "version": "2.1.0", - "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", - "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "bundled": true, "requires": { "safe-buffer": "^5.1.0" } }, "randomfill": { "version": "1.0.4", - "resolved": "https://registry.npmjs.org/randomfill/-/randomfill-1.0.4.tgz", - "integrity": "sha512-87lcbR8+MhcWcUiQ+9e+Rwx8MyR2P7qnt15ynUlbm3TU/fjbgz4GsvfSUDTemtCCtVCqb4ZcEFlyPNTh9bBTLw==", + "bundled": true, "requires": { "randombytes": "^2.0.5", "safe-buffer": "^5.1.0" @@ -1081,8 +997,7 @@ }, "readable-stream": { "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "bundled": true, "requires": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -1091,8 +1006,7 @@ }, "ripemd160": { "version": "2.0.2", - "resolved": "https://registry.npmjs.org/ripemd160/-/ripemd160-2.0.2.tgz", - "integrity": "sha512-ii4iagi25WusVoiC4B4lq7pbXfAp3D9v5CwfkY33vffw2+pkDjY1D8GaN7spsxvCSx8dkPqOZCEZyfxcmJG2IA==", + "bundled": true, "requires": { "hash-base": "^3.0.0", "inherits": "^2.0.1" @@ -1100,23 +1014,19 @@ }, "safe-buffer": { "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + "bundled": true }, "safer-buffer": { "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + "bundled": true }, "setimmediate": { "version": "1.0.5", - "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", - "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=" + "bundled": true }, "sha.js": { "version": "2.4.11", - "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", - "integrity": "sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==", + "bundled": true, "requires": { "inherits": "^2.0.1", "safe-buffer": "^5.0.1" @@ -1124,8 +1034,7 @@ }, "stream-browserify": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-3.0.0.tgz", - "integrity": "sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA==", + "bundled": true, "requires": { "inherits": "~2.0.4", "readable-stream": "^3.5.0" @@ -1133,8 +1042,7 @@ }, "stream-http": { "version": "3.2.0", - "resolved": "https://registry.npmjs.org/stream-http/-/stream-http-3.2.0.tgz", - "integrity": "sha512-Oq1bLqisTyK3TSCXpPbT4sdeYNdmyZJv1LxpEm2vu1ZhK89kSE5YXwZc3cWk0MagGaKriBh9mCFbVGtO+vY29A==", + "bundled": true, "requires": { "builtin-status-codes": "^3.0.0", "inherits": "^2.0.4", @@ -1144,8 +1052,7 @@ }, "string.prototype.trimend": { "version": "1.0.4", - "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.4.tgz", - "integrity": "sha512-y9xCjw1P23Awk8EvTpcyL2NIr1j7wJ39f+k6lvRnSMz+mz9CGz9NYPelDk42kOz6+ql8xjfK8oYzy3jAP5QU5A==", + "bundled": true, "requires": { "call-bind": "^1.0.2", "define-properties": "^1.1.3" @@ -1153,8 +1060,7 @@ }, "string.prototype.trimstart": { "version": "1.0.4", - "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.4.tgz", - "integrity": "sha512-jh6e984OBfvxS50tdY2nRZnoC5/mLFKOREQfw8t5yytkoUsJRNxvI/E39qu1sD0OtWI3OC0XgKSmcWwziwYuZw==", + "bundled": true, "requires": { "call-bind": "^1.0.2", "define-properties": "^1.1.3" @@ -1162,29 +1068,25 @@ }, "string_decoder": { "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "bundled": true, "requires": { "safe-buffer": "~5.2.0" } }, "timers-browserify": { "version": "2.0.12", - "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.12.tgz", - "integrity": "sha512-9phl76Cqm6FhSX9Xe1ZUAMLtm1BLkKj2Qd5ApyWkXzsMRaA7dgr81kf4wJmQf/hAvg8EEyJxDo3du/0KlhPiKQ==", + "bundled": true, "requires": { "setimmediate": "^1.0.4" } }, "tty-browserify": { "version": "0.0.1", - "resolved": "https://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.1.tgz", - "integrity": "sha512-C3TaO7K81YvjCgQH9Q1S3R3P3BtN3RIM8n+OvX4il1K1zgE8ZhI0op7kClgkxtutIE8hQrcrHBXvIheqKUUCxw==" + "bundled": true }, "unbox-primitive": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.1.tgz", - "integrity": "sha512-tZU/3NqK3dA5gpE1KtyiJUrEB0lxnGkMFHptJ7q6ewdZ8s12QrODwNbhIJStmJkd1QDXa1NRA8aF2A1zk/Ypyw==", + "bundled": true, "requires": { "function-bind": "^1.1.1", "has-bigints": "^1.0.1", @@ -1194,8 +1096,7 @@ }, "url": { "version": "0.11.0", - "resolved": "https://registry.npmjs.org/url/-/url-0.11.0.tgz", - "integrity": "sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE=", + "bundled": true, "requires": { "punycode": "1.3.2", "querystring": "0.2.0" @@ -1203,15 +1104,13 @@ "dependencies": { "punycode": { "version": "1.3.2", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + "bundled": true } } }, "util": { "version": "0.12.4", - "resolved": "https://registry.npmjs.org/util/-/util-0.12.4.tgz", - "integrity": "sha512-bxZ9qtSlGUWSOy9Qa9Xgk11kSslpuZwaxCg4sNIDj6FLucDab2JxnHwyNTCpHMtK1MjoQiWQ6DiUMZYbSrO+Sw==", + "bundled": true, "requires": { "inherits": "^2.0.3", "is-arguments": "^1.0.4", @@ -1223,18 +1122,15 @@ }, "util-deprecate": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + "bundled": true }, "vm-browserify": { "version": "1.1.2", - "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-1.1.2.tgz", - "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==" + "bundled": true }, "which-boxed-primitive": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz", - "integrity": "sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==", + "bundled": true, "requires": { "is-bigint": "^1.0.1", "is-boolean-object": "^1.1.0", @@ -1245,8 +1141,7 @@ }, "which-typed-array": { "version": "1.1.4", - "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.4.tgz", - "integrity": "sha512-49E0SpUe90cjpoc7BOJwyPHRqSAd12c10Qm2amdEZrJPCY2NDxaW01zHITrem+rnETY3dwrbH3UUrUwagfCYDA==", + "bundled": true, "requires": { "available-typed-arrays": "^1.0.2", "call-bind": "^1.0.0", @@ -1259,8 +1154,7 @@ }, "xtend": { "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + "bundled": true } } }, @@ -1275,9 +1169,9 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "mysql2": { - "version": "3.7.0", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.7.0.tgz", - "integrity": "sha512-c45jA3Jc1X8yJKzrWu1GpplBKGwv/wIV6ITZTlCSY7npF2YfJR+6nMP5e+NTQhUeJPSyOQAbGDCGEHbAl8HN9w==", + "version": "3.9.1", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.9.1.tgz", + "integrity": "sha512-3njoWAAhGBYy0tWBabqUQcLtczZUxrmmtc2vszQUekg3kTJyZ5/IeLC3Fo04u6y6Iy5Sba7pIIa2P/gs8D3ZeQ==", "requires": { "denque": "^2.1.0", "generate-function": "^2.3.1", @@ -1359,9 +1253,9 @@ } }, "react-bootstrap": { - "version": "2.9.0", - "resolved": "https://registry.npmjs.org/react-bootstrap/-/react-bootstrap-2.9.0.tgz", - "integrity": "sha512-dGh6fGjqR9MBzPOp2KbXJznt1Zy6SWepXYUdxMT18Zu/wJ73HCU8JNZe9dfzjmVssZYsJH9N3HHE4wAtQvNz7g==", + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/react-bootstrap/-/react-bootstrap-2.10.0.tgz", + "integrity": "sha512-87gRP69VAfeU2yKgp8RI3HvzhPNrnYIV2QNranYXataz3ef+k7OhvKGGdxQLQfUsQ2RTmlY66tn4pdFrZ94hNg==", "requires": { "@babel/runtime": "^7.22.5", "@restart/hooks": "^0.4.9", @@ -1378,9 +1272,9 @@ } }, "react-datepicker": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/react-datepicker/-/react-datepicker-4.18.0.tgz", - "integrity": "sha512-0MYt3HmLbHVk1sw4v+RCbLAVg5TA3jWP7RyjZbo53PC+SEi+pjdgc92lB53ai/ENZaTOhbXmgni9GzvMrorMAw==", + "version": "4.25.0", + "resolved": "https://registry.npmjs.org/react-datepicker/-/react-datepicker-4.25.0.tgz", + "integrity": "sha512-zB7CSi44SJ0sqo8hUQ3BF1saE/knn7u25qEMTO1CQGofY1VAKahO8k9drZtp0cfW1DMfoYLR3uSY1/uMvbEzbg==", "requires": { "@popperjs/core": "^2.11.8", "classnames": "^2.2.6", @@ -1489,9 +1383,9 @@ } }, "regenerator-runtime": { - "version": "0.14.0", - "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.0.tgz", - "integrity": "sha512-srw17NI0TUWHuGa5CFGGmhfNIeja30WMBfbslPNhf6JrqQlLN5gcrvig1oqPxiVaXb0oW0XRKtH6Nngs5lKCIA==" + "version": "0.14.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", + "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==" }, "resolve-pathname": { "version": "3.0.0", diff --git a/components/webui/package.json b/components/webui/package.json index b036cc261..49c589ce4 100644 --- a/components/webui/package.json +++ b/components/webui/package.json @@ -8,19 +8,19 @@ "visualize": "meteor --production --extra-packages bundle-visualizer" }, "dependencies": { - "@babel/runtime": "^7.15.4", - "@fortawesome/fontawesome-svg-core": "^6.4.2", - "@fortawesome/free-solid-svg-icons": "^6.4.2", + "@babel/runtime": "^7.23.9", + "@fortawesome/fontawesome-svg-core": "^6.5.1", + "@fortawesome/free-solid-svg-icons": "^6.5.1", "@fortawesome/react-fontawesome": "^0.2.0", "@msgpack/msgpack": "^3.0.0-beta2", "bootstrap": "^5.3.2", "json5": "^2.2.3", - "luxon": "^3.4.3", - "meteor-node-stubs": "^1.1.0", - "mysql2": "^3.7.0", + "luxon": "^3.4.4", + "meteor-node-stubs": "^1.2.7", + "mysql2": "^3.9.1", "react": "^17.0.2", - "react-bootstrap": "^2.9.0", - "react-datepicker": "^4.18.0", + "react-bootstrap": "^2.10.0", + "react-datepicker": "^4.25.0", "react-dom": "^17.0.2", "react-router": "^5.3.4", "react-router-dom": "^5.3.4", From df019064684a16badc29f1b5c9992f8ccdce3c96 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 8 Feb 2024 02:48:43 +0800 Subject: [PATCH 44/60] Add IngestionView to display compression stats. --- .../imports/api/ingestion/collections.js | 3 + .../api/ingestion/server/StatsDbManager.js | 65 ++++++ .../api/ingestion/server/publications.js | 59 ++++++ .../api/search/server/SearchJobsDbManager.js | 63 ++---- .../imports/api/search/server/methods.js | 84 +++----- .../webui/imports/api/user/client/methods.js | 2 +- components/webui/imports/ui/App.jsx | 53 +++-- components/webui/imports/ui/App.scss | 1 + .../imports/ui/IngestView/IngestView.jsx | 191 ++++++++++++++++++ .../imports/ui/IngestView/IngestView.scss | 157 ++++++++++++++ .../imports/ui/SearchView/SearchControls.jsx | 2 +- .../imports/ui/SearchView/SearchView.jsx | 4 +- .../webui/imports/ui/Sidebar/Sidebar.jsx | 7 +- components/webui/imports/ui/constants.js | 2 +- components/webui/imports/utils/DbManager.js | 79 ++++++++ components/webui/imports/utils/misc.js | 21 +- components/webui/server/main.js | 36 ++-- components/webui/settings.json | 7 +- 18 files changed, 685 insertions(+), 151 deletions(-) create mode 100644 components/webui/imports/api/ingestion/collections.js create mode 100644 components/webui/imports/api/ingestion/server/StatsDbManager.js create mode 100644 components/webui/imports/api/ingestion/server/publications.js create mode 100644 components/webui/imports/ui/IngestView/IngestView.jsx create mode 100644 components/webui/imports/ui/IngestView/IngestView.scss create mode 100644 components/webui/imports/utils/DbManager.js diff --git a/components/webui/imports/api/ingestion/collections.js b/components/webui/imports/api/ingestion/collections.js new file mode 100644 index 000000000..29bc16ec9 --- /dev/null +++ b/components/webui/imports/api/ingestion/collections.js @@ -0,0 +1,3 @@ +import {Mongo} from "meteor/mongo"; + +export const StatsCollection = new Mongo.Collection(Meteor.settings.public.StatsCollectionName); diff --git a/components/webui/imports/api/ingestion/server/StatsDbManager.js b/components/webui/imports/api/ingestion/server/StatsDbManager.js new file mode 100644 index 000000000..31ddf842f --- /dev/null +++ b/components/webui/imports/api/ingestion/server/StatsDbManager.js @@ -0,0 +1,65 @@ +const CLP_ARCHIVES_TABLE_COLUMN_NAMES = { + BEGIN_TIMESTAMP: "begin_timestamp", + END_TIMESTAMP: "end_timestamp", + UNCOMPRESSED_SIZE: "uncompressed_size", + SIZE: "size", +}; + +const CLP_FILES_TABLE_COLUMN_NAMES = { + ORIG_FILE_ID: "orig_file_id", + NUM_MESSAGES: "num_messages", +}; + +/** + * Class for retrieving compression stats in the database. + */ +class StatsDbManager { + #sqlDbConnection; + #clpFilesTableName; + #clpArchivesTableName; + + /** + * @param {mysql.Connection} sqlDbConnection + * @param {object} tableNames + * @param {string} tableNames.clpArchivesTableName + * @param {string} tableNames.clpFilesTableName + */ + constructor(sqlDbConnection, { + clpArchivesTableName, + clpFilesTableName, + }) { + this.#sqlDbConnection = sqlDbConnection; + + this.#clpArchivesTableName = clpArchivesTableName; + this.#clpFilesTableName = clpFilesTableName; + } + + /** + * Queries compression stats. + * @returns {Promise} stats + * @throws {Error} on error. + */ + async getCompressionStats() { + const [queryStats] = await this.#sqlDbConnection.query( + `SELECT a.begin_timestamp AS begin_timestamp, + a.end_timestamp AS end_timestamp, + a.total_uncompressed_size AS total_uncompressed_size, + a.total_compressed_size AS total_compressed_size, + b.num_files AS num_files, + b.num_messages AS num_messages + FROM (SELECT MIN(${CLP_ARCHIVES_TABLE_COLUMN_NAMES.BEGIN_TIMESTAMP}) AS begin_timestamp, + MAX(${CLP_ARCHIVES_TABLE_COLUMN_NAMES.END_TIMESTAMP}) AS end_timestamp, + SUM(${CLP_ARCHIVES_TABLE_COLUMN_NAMES.UNCOMPRESSED_SIZE}) AS total_uncompressed_size, + SUM(${CLP_ARCHIVES_TABLE_COLUMN_NAMES.SIZE}) AS total_compressed_size + FROM ${this.#clpArchivesTableName}) a, + (SELECT NULLIF(COUNT(DISTINCT ${CLP_FILES_TABLE_COLUMN_NAMES.ORIG_FILE_ID}), 0) AS num_files, + SUM(${CLP_FILES_TABLE_COLUMN_NAMES.NUM_MESSAGES}) AS num_messages + FROM ${this.#clpFilesTableName}) b;`, + ); + + + return queryStats[0]; + } +} + +export default StatsDbManager; diff --git a/components/webui/imports/api/ingestion/server/publications.js b/components/webui/imports/api/ingestion/server/publications.js new file mode 100644 index 000000000..168cc55a2 --- /dev/null +++ b/components/webui/imports/api/ingestion/server/publications.js @@ -0,0 +1,59 @@ +import {Meteor} from "meteor/meteor"; +import {logger} from "../../../utils/logger"; + +import {StatsCollection} from "../collections"; + +import StatsDbManager from "./StatsDbManager"; + +/** + * @type {StatsDbManager|null} + */ +let statsDbManager = null; + +/** + * @param {mysql.Connection} sqlDbConnection + * @param {object} tableNames + * @param {string} tableNames.clpArchivesTableName + * @param {string} tableNames.clpFilesTableName + * @param {string} clpArchivesTableName + * @param {string} clpFilesTableName + * @returns {void} + * @throws {Error} on error. + */ +const initStatsDbManager = (sqlDbConnection, { + clpArchivesTableName, + clpFilesTableName, +}) => { + statsDbManager = new StatsDbManager(sqlDbConnection, { + clpArchivesTableName, + clpFilesTableName, + }); +}; + +/** + * Updates and Publishes compression statistics. + * + * @param {string} publicationName + * + * @returns {Mongo.Cursor} + */ +Meteor.publish(Meteor.settings.public.StatsCollectionName, async () => { + // logger.debug(`Subscription '${Meteor.settings.public.StatsCollectionName}'`); + + const stats = await statsDbManager.getCompressionStats(); + + const filter = { + id: "compression_stats", + }; + const modifier = { + $set: stats, + }; + const options = { + upsert: true, + }; + await StatsCollection.updateAsync(filter, modifier, options); + + return StatsCollection.find(filter); +}); + +export {initStatsDbManager}; diff --git a/components/webui/imports/api/search/server/SearchJobsDbManager.js b/components/webui/imports/api/search/server/SearchJobsDbManager.js index 6f7fc3ada..b6c779e92 100644 --- a/components/webui/imports/api/search/server/SearchJobsDbManager.js +++ b/components/webui/imports/api/search/server/SearchJobsDbManager.js @@ -1,7 +1,7 @@ -import mysql from "mysql2/promise"; import msgpack from "@msgpack/msgpack"; -import {JOB_STATUS_WAITING_STATES, JobStatus} from "../constants"; + import {sleep} from "../../../utils/misc"; +import {JOB_STATUS_WAITING_STATES, JobStatus} from "../constants"; const SEARCH_JOBS_TABLE_COLUMN_NAMES = { ID: "id", @@ -16,55 +16,16 @@ class SearchJobsDbManager { #sqlDbConnection; #searchJobsTableName; - /** - * Creates a new instance. - * @param {string} dbHost - * @param {number} dbPort - * @param {string} dbName - * @param {string} dbUser - * @param {string} dbPassword - * @param {string} searchJobsTableName - * @returns {Promise} - * @throws {Error} on error. - */ - static async createNew({dbHost, dbPort, dbName, dbUser, dbPassword, searchJobsTableName}) { - const conn = await mysql.createConnection({ - host: dbHost, - port: dbPort, - database: dbName, - user: dbUser, - password: dbPassword, - }); - return new SearchJobsDbManager(conn, searchJobsTableName); - } - /** * @param {mysql.Connection} sqlDbConnection - * @param {string} searchJobsTableName + * @param {object} tableNames + * @param {string} tableNames.searchJobsTableName */ - constructor(sqlDbConnection, searchJobsTableName) { + constructor(sqlDbConnection, {searchJobsTableName}) { this.#sqlDbConnection = sqlDbConnection; this.#searchJobsTableName = searchJobsTableName; } - /** - * Connects to the database. - * @returns {Promise} - * @throws {Error} on error. - */ - async connect() { - await this.#sqlDbConnection.connect(); - } - - /** - * Disconnects from the database. - * @returns {Promise} - * @throws {Error} on error. - */ - async disconnect() { - await this.#sqlDbConnection.end(); - } - /** * Submits a query job to the database. * @param {Object} searchConfig The arguments for the query. @@ -74,8 +35,8 @@ class SearchJobsDbManager { async submitQuery(searchConfig) { const [queryInsertResults] = await this.#sqlDbConnection.query( `INSERT INTO ${this.#searchJobsTableName} - (${SEARCH_JOBS_TABLE_COLUMN_NAMES.SEARCH_CONFIG}) - VALUES (?)`, + (${SEARCH_JOBS_TABLE_COLUMN_NAMES.SEARCH_CONFIG}) + VALUES (?)`, [Buffer.from(msgpack.encode(searchConfig))], ); return queryInsertResults.insertId; @@ -90,8 +51,8 @@ class SearchJobsDbManager { async submitQueryCancellation(jobId) { await this.#sqlDbConnection.query( `UPDATE ${this.#searchJobsTableName} - SET ${SEARCH_JOBS_TABLE_COLUMN_NAMES.STATUS} = ${JobStatus.CANCELLING} - WHERE ${SEARCH_JOBS_TABLE_COLUMN_NAMES.ID} = ?`, + SET ${SEARCH_JOBS_TABLE_COLUMN_NAMES.STATUS} = ${JobStatus.CANCELLING} + WHERE ${SEARCH_JOBS_TABLE_COLUMN_NAMES.ID} = ?`, jobId, ); } @@ -109,8 +70,8 @@ class SearchJobsDbManager { try { const [queryRows, _] = await this.#sqlDbConnection.query( `SELECT ${SEARCH_JOBS_TABLE_COLUMN_NAMES.STATUS} - FROM ${this.#searchJobsTableName} - WHERE ${SEARCH_JOBS_TABLE_COLUMN_NAMES.ID} = ?`, + FROM ${this.#searchJobsTableName} + WHERE ${SEARCH_JOBS_TABLE_COLUMN_NAMES.ID} = ?`, jobId, ); rows = queryRows; @@ -127,7 +88,7 @@ class SearchJobsDbManager { throw new Error(`Job ${jobId} was cancelled.`); } else if (JobStatus.SUCCESS !== status) { throw new Error(`Job ${jobId} exited with unexpected status=${status}: ` - `${Object.keys(JobStatus)[status]}.`); + `${Object.keys(JobStatus)[status]}.`); } break; } diff --git a/components/webui/imports/api/search/server/methods.js b/components/webui/imports/api/search/server/methods.js index fa6234335..7584dab12 100644 --- a/components/webui/imports/api/search/server/methods.js +++ b/components/webui/imports/api/search/server/methods.js @@ -1,8 +1,8 @@ import {logger} from "/imports/utils/logger"; import {Meteor} from "meteor/meteor"; import {SearchResultsMetadataCollection} from "../collections"; -import {searchJobCollectionsManager} from "./collections"; import {SearchSignal} from "../constants"; +import {searchJobCollectionsManager} from "./collections"; import SearchJobsDbManager from "./SearchJobsDbManager"; /** @@ -11,37 +11,14 @@ import SearchJobsDbManager from "./SearchJobsDbManager"; let searchJobsDbManager = null; /** - * @param {string} dbHost - * @param {number} dbPort - * @param {string} dbName - * @param {string} dbUser - * @param {string} dbPassword - * @param {string} searchJobsTableName - * @returns {Promise} + * @param {mysql.Connection} sqlDbConnection + * @param {object} tableNames + * @param {string} tableNames.searchJobsTableName + * @returns {void} * @throws {Error} on error. */ -const initSearchJobsDbManager = async ({dbHost, dbPort, dbName, dbUser, dbPassword, searchJobsTableName}) => { - try { - searchJobsDbManager = await SearchJobsDbManager.createNew({ - dbHost: dbHost, - dbPort: dbPort, - dbName: dbName, - dbUser: dbUser, - dbPassword: dbPassword, - searchJobsTableName: searchJobsTableName, - }); - await searchJobsDbManager.connect(); - } catch (e) { - logger.error("Unable to create MySQL / mariadb connection.", e.toString()); - throw e; - } -}; - -const deinitSearchJobsDbManager = async () => { - if (null !== searchJobsDbManager) { - await searchJobsDbManager.disconnect(); - searchJobsDbManager = null; - } +const initSearchJobsDbManager = (sqlDbConnection, {searchJobsTableName}) => { + searchJobsDbManager = new SearchJobsDbManager(sqlDbConnection, {searchJobsTableName}); }; /** @@ -57,7 +34,7 @@ const updateSearchEventWhenJobFinishes = async (jobId) => { errorMsg = e.message; } const filter = { - _id: jobId.toString() + _id: jobId.toString(), }; const modifier = { $set: { @@ -65,10 +42,10 @@ const updateSearchEventWhenJobFinishes = async (jobId) => { errorMsg: errorMsg, numTotalResults: await searchJobCollectionsManager.getOrCreateCollection(jobId).countDocuments(), - } + }, }; - logger.debug("modifier = ", modifier) + logger.debug("modifier = ", modifier); SearchResultsMetadataCollection.update(filter, modifier); }; @@ -79,12 +56,18 @@ const updateSearchEventWhenJobFinishes = async (jobId) => { */ const createMongoIndexes = async (jobId) => { const timestampAscendingIndex = { - key: {timestamp: 1, _id: 1}, - name: "timestamp-ascending" + key: { + timestamp: 1, + _id: 1, + }, + name: "timestamp-ascending", }; const timestampDescendingIndex = { - key: {timestamp: -1, _id: -1}, - name: "timestamp-descending" + key: { + timestamp: -1, + _id: -1, + }, + name: "timestamp-descending", }; const queryJobCollection = searchJobCollectionsManager.getOrCreateCollection(jobId); @@ -102,16 +85,16 @@ Meteor.methods({ * @returns {Object} containing {jobId} of the submitted search job */ async "search.submitQuery"({ - queryString, - timestampBegin, - timestampEnd, - }) { + queryString, + timestampBegin, + timestampEnd, + }) { const args = { query_string: queryString, begin_timestamp: timestampBegin, end_timestamp: timestampEnd, }; - logger.info("search.submitQuery args =", args) + logger.info("search.submitQuery args =", args); let jobId; try { @@ -125,7 +108,7 @@ Meteor.methods({ SearchResultsMetadataCollection.insert({ _id: jobId.toString(), lastSignal: SearchSignal.RESP_QUERYING, - errorMsg: null + errorMsg: null, }); Meteor.defer(async () => { @@ -143,10 +126,9 @@ Meteor.methods({ * @param {number} jobId of the search results to clear */ async "search.clearResults"({ - jobId - }) - { - logger.info("search.clearResults jobId =", jobId) + jobId, + }) { + logger.info("search.clearResults jobId =", jobId); try { const resultsCollection = searchJobCollectionsManager.getOrCreateCollection(jobId); @@ -166,9 +148,9 @@ Meteor.methods({ * @param {number} jobId of the search operation to cancel */ async "search.cancelOperation"({ - jobId - }) { - logger.info("search.cancelOperation jobId =", jobId) + jobId, + }) { + logger.info("search.cancelOperation jobId =", jobId); try { await searchJobsDbManager.submitQueryCancellation(jobId); @@ -180,4 +162,4 @@ Meteor.methods({ }, }); -export {deinitSearchJobsDbManager, initSearchJobsDbManager}; +export {initSearchJobsDbManager}; diff --git a/components/webui/imports/api/user/client/methods.js b/components/webui/imports/api/user/client/methods.js index b4fe34d26..d14040df7 100644 --- a/components/webui/imports/api/user/client/methods.js +++ b/components/webui/imports/api/user/client/methods.js @@ -63,7 +63,7 @@ export const loginWithUsername = (username) => { /** * Attempts to log in a user using a stored username or register a new one if none is found. * - * @returns {boolean} true if the login is successful + * @returns {Promise} true if the login is successful * false if there's an error during login or registration */ export const login = async () => { diff --git a/components/webui/imports/ui/App.jsx b/components/webui/imports/ui/App.jsx index f4d390478..14d744431 100644 --- a/components/webui/imports/ui/App.jsx +++ b/components/webui/imports/ui/App.jsx @@ -1,22 +1,25 @@ import React from "react"; import {Redirect, Route, Switch} from "react-router"; -import {faSearch} from "@fortawesome/free-solid-svg-icons"; +import {faSearch, faFileUpload} from "@fortawesome/free-solid-svg-icons"; +import IngestView from './IngestView/IngestView.jsx'; import SearchView from "./SearchView/SearchView.jsx"; import Sidebar from "./Sidebar/Sidebar.jsx"; import {login} from "../api/user/client/methods"; +import {LOCAL_STORAGE_KEYS} from "./constants"; + import "./App.scss"; -import LOCAL_STORAGE_KEYS from "./constants"; const ROUTES = [ + {path: "/ingest", label: "Ingest", icon: faFileUpload, component: IngestView}, {path: "/search", label: "Search", icon: faSearch, component: SearchView}, ]; export const App = () => { const [loggedIn, setLoggedIn] = React.useState(false); - const [isSidebarStateCollapsed, setSidebarStateCollapsed] = React.useState( + const [isSidebarCollapsed, setSidebarStateCollapsed] = React.useState( "true" === localStorage.getItem(LOCAL_STORAGE_KEYS.IS_SIDEBAR_COLLAPSED) ); @@ -26,35 +29,43 @@ export const App = () => { }, []); React.useEffect(() => { - localStorage.setItem(LOCAL_STORAGE_KEYS.IS_SIDEBAR_COLLAPSED, isSidebarStateCollapsed.toString()); - }, [isSidebarStateCollapsed]); + localStorage.setItem(LOCAL_STORAGE_KEYS.IS_SIDEBAR_COLLAPSED, isSidebarCollapsed.toString()); + }, [isSidebarCollapsed]); const handleSidebarToggle = () => { - setSidebarStateCollapsed(!isSidebarStateCollapsed); + setSidebarStateCollapsed(!isSidebarCollapsed); } + const Spinner = () =>
+
+
+ Loading... +
+
+
; + + const Routes = () => + + + + + + + + + + ; + return (
- {!loggedIn ?
-
-
- Loading... -
-
-
: - - - - - - - } + {!loggedIn ? : }
); diff --git a/components/webui/imports/ui/App.scss b/components/webui/imports/ui/App.scss index e4230d432..ebec5adff 100644 --- a/components/webui/imports/ui/App.scss +++ b/components/webui/imports/ui/App.scss @@ -11,5 +11,6 @@ body { height: 100%; } +@import "IngestView/IngestView.scss"; @import "SearchView/SearchView.scss"; @import "Sidebar/Sidebar.scss"; diff --git a/components/webui/imports/ui/IngestView/IngestView.jsx b/components/webui/imports/ui/IngestView/IngestView.jsx new file mode 100644 index 000000000..699bd81e3 --- /dev/null +++ b/components/webui/imports/ui/IngestView/IngestView.jsx @@ -0,0 +1,191 @@ +import {faChartBar, faClock, faEnvelope, faFileAlt, faHdd} from "@fortawesome/free-solid-svg-icons"; + +import {FontAwesomeIcon} from "@fortawesome/react-fontawesome"; +import {DateTime} from "luxon"; +import {Meteor} from "meteor/meteor"; +import {useTracker} from "meteor/react-meteor-data"; +import React, {useEffect} from "react"; +import {Col, Container, ProgressBar, Row} from "react-bootstrap"; + +import {StatsCollection} from "../../api/ingestion/collections"; +import {computeHumanSize} from "../../utils/misc"; + + +/** + * Presents compression statistics. + * @returns {JSX.Element} + */ +const IngestView = () => { + const stats = useTracker(() => { + return StatsCollection.findOne(); + }, []); + + useEffect(() => { + let subscription = Meteor.subscribe(Meteor.settings.public.StatsCollectionName); + const interval = Meteor.setInterval(() => { + const oldSubscription = subscription; + subscription = Meteor.subscribe(Meteor.settings.public.StatsCollectionName); + oldSubscription.stop(); + }, 5000); + + return () => { + Meteor.clearInterval(interval); + }; + }, []); + + return ( + + + + {stats ? ( + + + + +
+ + ) : (<>)} + + + + ); +}; + +/** + * Presents space savings from the given statistics. + * + * @param {Object} stats + * @param {string} stats.total_uncompressed_size + * @param {string} stats.total_compressed_size + * @returns {JSX.Element} + */ +const SpaceSavings = ({stats}) => { + const logsUncompressedSize = parseInt(stats.total_uncompressed_size) || 0; + const logsCompressedSize = parseInt(stats.total_compressed_size) || 0; + const spaceSavings = logsUncompressedSize > 0 ? 100 * (1 - logsCompressedSize / logsUncompressedSize) : 0; + + return ( +
+ +

Space Savings

+ + + + + + {spaceSavings.toFixed(2) + "%"} + + + + + +
+ {computeHumanSize(logsUncompressedSize)} before compression + +
+ + + + +
+ {computeHumanSize(logsCompressedSize)} after compression + +
+ + +
+ ); +}; + +/** + * Presents details from the given statistics. + * + * @param {Object|null} stats - The statistics object. + * @param {number|null} stats.begin_timestamp + * @param {number|null} stats.end_timestamp + * @param {number|null} stats.num_files + * @param {number|null} stats.num_messages + * @returns {JSX.Element} + */ +const Details = ({stats}) => { + const { + begin_timestamp: beginTimeStamp, + end_timestamp: endTimeStamp, + num_files: numFiles, + num_messages: numMessages + } = stats; + + let timeRangeRow = null; + if (null !== endTimeStamp) { + let timestamp_format = "kkkk-MMM-dd HH:mm"; + timeRangeRow = ( +
+
+ +
+
+ + {DateTime.fromMillis(beginTimeStamp).toFormat(timestamp_format)} + to + {DateTime.fromMillis(endTimeStamp).toFormat(timestamp_format)} + + time range +
+
+ ); + } + + let num_files_row = null; + if (null !== numFiles) { + num_files_row = ( +
+
+ +
+
+ {numFiles.toLocaleString()} + files +
+
+ ); + } + + let num_messages_row = null; + if (null !== numMessages) { + num_messages_row = ( +
+
+ +
+
+ {numMessages.toLocaleString()} + messages +
+
+ ); + } + + if (!(timeRangeRow || num_files_row || num_messages_row)) { + // No details to display + return (<>); + } + + return ( +
+ +

Details

+ + + + + {timeRangeRow} + {num_files_row} + {num_messages_row} + + +
+ ); +}; + +export default IngestView; diff --git a/components/webui/imports/ui/IngestView/IngestView.scss b/components/webui/imports/ui/IngestView/IngestView.scss new file mode 100644 index 000000000..9516d9c82 --- /dev/null +++ b/components/webui/imports/ui/IngestView/IngestView.scss @@ -0,0 +1,157 @@ +.ingest-container { + // Background highlight + background: linear-gradient(to bottom, #004850, #004850 160px, transparent 160px); + padding: 30px; +} + +.panel { + background-color: white; + border: 1px solid #ddd; + border-radius: 3px; + box-shadow: 0 1px 11px 0 rgba(0, 0, 0, 0.1); + padding: 15px; + margin: 10px 0; +} + +.panel-h1 { + font-size: 1.5rem; + line-height: 1.5rem; + margin: 0 0 15px 0; +} + +.panel-icon { + color: #004850; + font-size: 1.5rem; + line-height: 1.5rem; +} + +.ingest { + &-stats { + &-details { + &-icon-container { + width: 40px; + + flex: 0 0 auto; + margin-right: 10px; + + color: #ad1869; + font-size: 40px; + line-height: 40px; + text-align: center; + } + + &-text-container { + flex-basis: 0; + flex-grow: 1; + max-width: 100%; + } + + &-row { + border-top: 1px solid #eee; + display: flex; + flex-wrap: wrap; + padding: 15px 0; + } + } + + &-detail { + display: block; + font-size: 1.5rem; + line-height: 1; + } + } +} + +.ingest-stat-bar { + height: 4px; + margin-bottom: 0.25rem; +} + +.ingest-stat-bar > .progress-bar { + background-color: $info; +} + +.ingest-form-group { + padding-bottom: 15px; + padding-top: 15px; +} + +.ingest-form-group-h2 { + font-family: "Source Sans Pro", sans-serif; + font-size: 1.25rem; + line-height: 1.25rem; +} + +.ingest-form-group-desc { + font-size: 1rem; + font-style: italic; + line-height: 1.5rem; +} + +.ingest-form-group-h2, .ingest-form-group-desc, .ingest-desc-text { + color: #777; +} + +.ingest-form-label { + color: #333; + font-size: 1rem; + line-height: 1rem; + padding-bottom: calc(0.25rem + 3px); + padding-top: calc(0.25rem + 3px); + text-align: right; +} + +.ingest-form-control { + height: calc(1rem + 0.5rem + 6px); + font-size: 1rem; + line-height: 1rem; +} + +.ingest-form-error-msg { + color: $danger; +} + +.job-status-icon, .job-log-download-link, .job-action { + font-size: 1rem; + line-height: 1rem; + text-align: center; + + padding: 0.375rem 0; + + height: 1.75rem; + width: 1.75rem; +} + +.job-log-download-link { + background-color: $info; + border-radius: 0.25rem; + color: #fff; + + display: block; +} + +.job-log-download-link:hover { + color: #fff; + background-color: darken($info, 7.5%) +} + +.job-status-icon { + border-radius: 0.25rem; + + display: block; + + &-danger { + background-color: lighten($danger, 40%); + color: $danger; + } + + &-info { + background-color: lighten($info, 50%); + color: $info; + } + + &-success { + background-color: lighten($success, 50%); + color: $success; + } +} diff --git a/components/webui/imports/ui/SearchView/SearchControls.jsx b/components/webui/imports/ui/SearchView/SearchControls.jsx index 5e07e2c25..9e5703690 100644 --- a/components/webui/imports/ui/SearchView/SearchControls.jsx +++ b/components/webui/imports/ui/SearchView/SearchControls.jsx @@ -15,7 +15,7 @@ import {FontAwesomeIcon} from "@fortawesome/react-fontawesome"; import {faBars, faSearch, faTimes, faTrash} from "@fortawesome/free-solid-svg-icons"; import {computeTimeRange, TIME_RANGE_PRESET_LABEL} from "./datetime"; -import LOCAL_STORAGE_KEYS from "../constants"; +import {LOCAL_STORAGE_KEYS} from "../constants"; import {isSearchSignalQuerying, isSearchSignalReq, SearchSignal} from "../../api/search/constants"; import "./SearchControls.scss"; diff --git a/components/webui/imports/ui/SearchView/SearchView.jsx b/components/webui/imports/ui/SearchView/SearchView.jsx index cdd1b45a9..46f6f0bc0 100644 --- a/components/webui/imports/ui/SearchView/SearchView.jsx +++ b/components/webui/imports/ui/SearchView/SearchView.jsx @@ -7,13 +7,13 @@ import {ProgressBar} from "react-bootstrap"; import { addSortToMongoFindOptions, - SearchResultsMetadataCollection + SearchResultsMetadataCollection, } from "../../api/search/collections"; import {INVALID_JOB_ID, isSearchSignalQuerying, SearchSignal} from "../../api/search/constants"; import SearchJobCollectionsManager from "../../api/search/SearchJobCollectionsManager"; import "react-datepicker/dist/react-datepicker.css"; -import LOCAL_STORAGE_KEYS from "../constants"; +import {LOCAL_STORAGE_KEYS} from "../constants"; import {changeTimezoneToUtcWithoutChangingTime, DEFAULT_TIME_RANGE_GETTER} from "./datetime"; import {SearchControls} from "./SearchControls.jsx"; import {SearchResults} from "./SearchResults.jsx"; diff --git a/components/webui/imports/ui/Sidebar/Sidebar.jsx b/components/webui/imports/ui/Sidebar/Sidebar.jsx index 2d2fd8086..945bd993b 100644 --- a/components/webui/imports/ui/Sidebar/Sidebar.jsx +++ b/components/webui/imports/ui/Sidebar/Sidebar.jsx @@ -31,9 +31,10 @@ const Sidebar = ({isSidebarCollapsed, routes, onSidebarToggle}) => { {routes.map((route, i) => (false === (route["hide"] ?? false)) && ( -
+
+
{route["label"]} diff --git a/components/webui/imports/ui/constants.js b/components/webui/imports/ui/constants.js index 934250350..7cc7b527f 100644 --- a/components/webui/imports/ui/constants.js +++ b/components/webui/imports/ui/constants.js @@ -10,4 +10,4 @@ const LOCAL_STORAGE_KEYS = Object.freeze({ SEARCH_CONTROLS_VISIBLE: "searchFilterControlsVisible", }); -export default LOCAL_STORAGE_KEYS; +export {LOCAL_STORAGE_KEYS}; diff --git a/components/webui/imports/utils/DbManager.js b/components/webui/imports/utils/DbManager.js new file mode 100644 index 000000000..dfddb09ba --- /dev/null +++ b/components/webui/imports/utils/DbManager.js @@ -0,0 +1,79 @@ +import mysql from "mysql2/promise"; +import {initStatsDbManager} from "../api/ingestion/server/publications"; +import {initSearchJobsDbManager} from "../api/search/server/methods"; +import {logger} from "./logger"; + + +/** + * @type {mysql.Connection|null} + */ +let dbConnection = null; + +/** + * Creates a new database connection and initializes DB managers with it. + * + * @param {object} dbConfig + * @param {string} dbConfig.dbHost + * @param {number} dbConfig.dbPort + * @param {string} dbConfig.dbName + * @param {string} dbConfig.dbUser + * @param {string} dbConfig.dbPassword + * + * @param {object} tableNames + * @param {string} tableNames.searchJobsTableName + * @param {string} tableNames.clpArchivesTableName + * @param {string} tableNames.clpFilesTableName + * + * @returns {Promise} + * @throws {Error} on error. + */ +const initDbManagers = async ({ + dbHost, + dbPort, + dbName, + dbUser, + dbPassword, +}, { + searchJobsTableName, + clpArchivesTableName, + clpFilesTableName +}) => { + if (null !== dbConnection) { + logger.error("This method should not be called twice. "); + return; + } + + try { + dbConnection = await mysql.createConnection({ + host: dbHost, + port: dbPort, + database: dbName, + user: dbUser, + password: dbPassword, + }); + await dbConnection.connect(); + + initSearchJobsDbManager(dbConnection, { + searchJobsTableName + }); + initStatsDbManager(dbConnection, { + clpArchivesTableName, + clpFilesTableName, + }); + } catch (e) { + logger.error("Unable to create MySQL / mariadb connection.", + e.toString()); + throw e; + } +}; + +/** + * De-initialize database managers. + * @returns {Promise} + * @throws {Error} on error. + */ +const deinitDbManagers = async () => { + await dbConnection.end(); +}; + +export {initDbManagers, deinitDbManagers}; diff --git a/components/webui/imports/utils/misc.js b/components/webui/imports/utils/misc.js index 58d87a098..4876f3076 100644 --- a/components/webui/imports/utils/misc.js +++ b/components/webui/imports/utils/misc.js @@ -4,4 +4,23 @@ * @param {number} seconds to wait before resolving the promise * @returns {Promise} that resolves after the specified delay */ -export const sleep = (seconds) => new Promise(r => setTimeout(r, seconds * 1000)); +const sleep = (seconds) => new Promise(r => setTimeout(r, seconds * 1000)); + +/** + * Computes a human-readable representation of a size in bytes. + * + * @param {number} num + * @returns {string} + */ +const computeHumanSize = (num) => { + const si_prefixes = ["", "K", "M", "G", "T", "P", "E", "Z"]; + for (let i = 0; i < si_prefixes.length; ++i) { + if (Math.abs(num) < 1024.0) { + return "" + Math.round(num) + " " + si_prefixes[i] + "B"; + } + num /= 1024.0; + } + return Math.round(num) + " B"; +}; + +export {sleep, computeHumanSize}; \ No newline at end of file diff --git a/components/webui/server/main.js b/components/webui/server/main.js index 2723e15a6..d9a0a3dc7 100644 --- a/components/webui/server/main.js +++ b/components/webui/server/main.js @@ -1,16 +1,14 @@ +import {initSearchEventCollection} from "/imports/api/search/collections"; +import {initLogger} from "/imports/utils/logger"; import {Meteor} from "meteor/meteor"; +import "/imports/api/ingestion/collections"; +import "/imports/api/ingestion/server/publications"; import "/imports/api/search/server/collections"; import "/imports/api/search/server/methods"; import "/imports/api/search/server/publications"; import "/imports/api/user/server/methods"; - -import {initSearchEventCollection} from "/imports/api/search/collections"; -import { - deinitSearchJobsDbManager, - initSearchJobsDbManager, -} from "/imports/api/search/server/methods"; -import {initLogger} from "/imports/utils/logger"; +import {deinitDbManagers, initDbManagers} from "../imports/utils/DbManager"; const DEFAULT_LOGS_DIR = "."; const DEFAULT_LOGGING_LEVEL = Meteor.isDevelopment ? "DEBUG" : "INFO"; @@ -29,7 +27,7 @@ const parseEnvVars = () => { if ([CLP_DB_USER, CLP_DB_PASS].includes(undefined)) { console.error("Environment variables CLP_DB_USER and CLP_DB_PASS must be defined"); - process.exit(1) + process.exit(1); } const WEBUI_LOGS_DIR = process.env["WEBUI_LOGS_DIR"] || DEFAULT_LOGS_DIR; @@ -48,19 +46,23 @@ Meteor.startup(async () => { initLogger(envVars.WEBUI_LOGS_DIR, envVars.WEBUI_LOGGING_LEVEL, Meteor.isDevelopment); - await initSearchJobsDbManager({ - dbHost: Meteor.settings.private.SqlDbHost, - dbPort: Meteor.settings.private.SqlDbPort, - dbName: Meteor.settings.private.SqlDbName, - dbUser: envVars.CLP_DB_USER, - dbPassword: envVars.CLP_DB_PASS, - searchJobsTableName: Meteor.settings.private.SqlDbSearchJobsTableName, - }) + await initDbManagers({ + dbHost: Meteor.settings.private.SqlDbHost, + dbPort: Meteor.settings.private.SqlDbPort, + dbName: Meteor.settings.private.SqlDbName, + dbUser: envVars.CLP_DB_USER, + dbPassword: envVars.CLP_DB_PASS, + }, { + searchJobsTableName: Meteor.settings.private.SqlDbSearchJobsTableName, + clpArchivesTableName: Meteor.settings.private.SqlDbArchivesTableName, + clpFilesTableName: Meteor.settings.private.SqlDbClpFilesTableName, + }, + ); initSearchEventCollection(); }); process.on("exit", async (code) => { console.log(`Node.js is about to exit with code: ${code}`); - await deinitSearchJobsDbManager(); + await deinitDbManagers(); }); diff --git a/components/webui/settings.json b/components/webui/settings.json index 69bcdbb04..f6e96bacd 100644 --- a/components/webui/settings.json +++ b/components/webui/settings.json @@ -3,10 +3,13 @@ "SqlDbHost": "localhost", "SqlDbPort": 3306, "SqlDbName": "clp-db", - "SqlDbSearchJobsTableName": "search_jobs" + "SqlDbSearchJobsTableName": "search_jobs", + "SqlDbArchivesTableName": "clp_archives", + "SqlDbClpFilesTableName": "clp_files" }, "public": { "SearchResultsCollectionName": "results", - "SearchResultsMetadataCollectionName": "results-metadata" + "SearchResultsMetadataCollectionName": "results-metadata", + "StatsCollectionName": "stats" } } From c3c23c169c8ef7ec333885d418f55a709b86d257 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 7 Feb 2024 19:02:30 -0500 Subject: [PATCH 45/60] Remove unused styles. --- .../imports/ui/IngestView/IngestView.scss | 85 ------------------- 1 file changed, 85 deletions(-) diff --git a/components/webui/imports/ui/IngestView/IngestView.scss b/components/webui/imports/ui/IngestView/IngestView.scss index 9516d9c82..fffe6808d 100644 --- a/components/webui/imports/ui/IngestView/IngestView.scss +++ b/components/webui/imports/ui/IngestView/IngestView.scss @@ -70,88 +70,3 @@ .ingest-stat-bar > .progress-bar { background-color: $info; } - -.ingest-form-group { - padding-bottom: 15px; - padding-top: 15px; -} - -.ingest-form-group-h2 { - font-family: "Source Sans Pro", sans-serif; - font-size: 1.25rem; - line-height: 1.25rem; -} - -.ingest-form-group-desc { - font-size: 1rem; - font-style: italic; - line-height: 1.5rem; -} - -.ingest-form-group-h2, .ingest-form-group-desc, .ingest-desc-text { - color: #777; -} - -.ingest-form-label { - color: #333; - font-size: 1rem; - line-height: 1rem; - padding-bottom: calc(0.25rem + 3px); - padding-top: calc(0.25rem + 3px); - text-align: right; -} - -.ingest-form-control { - height: calc(1rem + 0.5rem + 6px); - font-size: 1rem; - line-height: 1rem; -} - -.ingest-form-error-msg { - color: $danger; -} - -.job-status-icon, .job-log-download-link, .job-action { - font-size: 1rem; - line-height: 1rem; - text-align: center; - - padding: 0.375rem 0; - - height: 1.75rem; - width: 1.75rem; -} - -.job-log-download-link { - background-color: $info; - border-radius: 0.25rem; - color: #fff; - - display: block; -} - -.job-log-download-link:hover { - color: #fff; - background-color: darken($info, 7.5%) -} - -.job-status-icon { - border-radius: 0.25rem; - - display: block; - - &-danger { - background-color: lighten($danger, 40%); - color: $danger; - } - - &-info { - background-color: lighten($info, 50%); - color: $info; - } - - &-success { - background-color: lighten($success, 50%); - color: $success; - } -} From a6dd9cac418cee2d6402b0c0d99b766f1eaf1881 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 8 Feb 2024 11:36:56 +0800 Subject: [PATCH 46/60] Move compression stats periodic refreshing logics from every subscription to the global scope. --- .../imports/api/ingestion/collections.js | 6 +- .../api/ingestion/server/publications.js | 64 +++++++++++++++---- .../imports/ui/IngestView/IngestView.jsx | 24 ++----- components/webui/imports/utils/DbManager.js | 4 +- 4 files changed, 67 insertions(+), 31 deletions(-) diff --git a/components/webui/imports/api/ingestion/collections.js b/components/webui/imports/api/ingestion/collections.js index 29bc16ec9..fd0c8f4ba 100644 --- a/components/webui/imports/api/ingestion/collections.js +++ b/components/webui/imports/api/ingestion/collections.js @@ -1,3 +1,7 @@ import {Mongo} from "meteor/mongo"; -export const StatsCollection = new Mongo.Collection(Meteor.settings.public.StatsCollectionName); +const StatsCollection = new Mongo.Collection(Meteor.settings.public.StatsCollectionName); + +const STATS_COLLECTION_ID_COMPRESSION = "compression_stats"; + +export {StatsCollection, STATS_COLLECTION_ID_COMPRESSION}; diff --git a/components/webui/imports/api/ingestion/server/publications.js b/components/webui/imports/api/ingestion/server/publications.js index 168cc55a2..0cca38780 100644 --- a/components/webui/imports/api/ingestion/server/publications.js +++ b/components/webui/imports/api/ingestion/server/publications.js @@ -1,15 +1,50 @@ import {Meteor} from "meteor/meteor"; -import {logger} from "../../../utils/logger"; -import {StatsCollection} from "../collections"; +import {STATS_COLLECTION_ID_COMPRESSION, StatsCollection} from "../collections"; import StatsDbManager from "./StatsDbManager"; + +/** + * @type {number} + */ +const STATS_REFRESH_INTERVAL_MS = 5000; + /** * @type {StatsDbManager|null} */ let statsDbManager = null; +/** + * @type {number|null} + */ +let refreshMeteorInterval = null; + +/** + * Updates the compression statistics in the StatsCollection. + * + * @async + * @returns {Promise} + */ +const refreshCompressionStats = async () => { + if (Meteor.server.stream_server.all_sockets().length === 0) { + return; + } + + const stats = await statsDbManager.getCompressionStats(); + const filter = { + id: STATS_COLLECTION_ID_COMPRESSION, + }; + const modifier = { + $set: stats, + }; + const options = { + upsert: true, + }; + + await StatsCollection.updateAsync(filter, modifier, options); +}; + /** * @param {mysql.Connection} sqlDbConnection * @param {object} tableNames @@ -28,6 +63,18 @@ const initStatsDbManager = (sqlDbConnection, { clpArchivesTableName, clpFilesTableName, }); + + refreshMeteorInterval = Meteor.setInterval(refreshCompressionStats, STATS_REFRESH_INTERVAL_MS); +}; + +/** + * @returns {void} + */ +const deinitStatsDbManager = () => { + if (null !== refreshMeteorInterval) { + Meteor.clearInterval(refreshMeteorInterval); + refreshMeteorInterval = null; + } }; /** @@ -40,20 +87,13 @@ const initStatsDbManager = (sqlDbConnection, { Meteor.publish(Meteor.settings.public.StatsCollectionName, async () => { // logger.debug(`Subscription '${Meteor.settings.public.StatsCollectionName}'`); - const stats = await statsDbManager.getCompressionStats(); + await refreshCompressionStats(); const filter = { - id: "compression_stats", - }; - const modifier = { - $set: stats, + id: STATS_COLLECTION_ID_COMPRESSION, }; - const options = { - upsert: true, - }; - await StatsCollection.updateAsync(filter, modifier, options); return StatsCollection.find(filter); }); -export {initStatsDbManager}; +export {initStatsDbManager, deinitStatsDbManager}; diff --git a/components/webui/imports/ui/IngestView/IngestView.jsx b/components/webui/imports/ui/IngestView/IngestView.jsx index 699bd81e3..41f1f732f 100644 --- a/components/webui/imports/ui/IngestView/IngestView.jsx +++ b/components/webui/imports/ui/IngestView/IngestView.jsx @@ -2,9 +2,8 @@ import {faChartBar, faClock, faEnvelope, faFileAlt, faHdd} from "@fortawesome/fr import {FontAwesomeIcon} from "@fortawesome/react-fontawesome"; import {DateTime} from "luxon"; -import {Meteor} from "meteor/meteor"; import {useTracker} from "meteor/react-meteor-data"; -import React, {useEffect} from "react"; +import React from "react"; import {Col, Container, ProgressBar, Row} from "react-bootstrap"; import {StatsCollection} from "../../api/ingestion/collections"; @@ -17,20 +16,9 @@ import {computeHumanSize} from "../../utils/misc"; */ const IngestView = () => { const stats = useTracker(() => { - return StatsCollection.findOne(); - }, []); + Meteor.subscribe(Meteor.settings.public.StatsCollectionName); - useEffect(() => { - let subscription = Meteor.subscribe(Meteor.settings.public.StatsCollectionName); - const interval = Meteor.setInterval(() => { - const oldSubscription = subscription; - subscription = Meteor.subscribe(Meteor.settings.public.StatsCollectionName); - oldSubscription.stop(); - }, 5000); - - return () => { - Meteor.clearInterval(interval); - }; + return StatsCollection.findOne(); }, []); return ( @@ -62,7 +50,9 @@ const IngestView = () => { const SpaceSavings = ({stats}) => { const logsUncompressedSize = parseInt(stats.total_uncompressed_size) || 0; const logsCompressedSize = parseInt(stats.total_compressed_size) || 0; - const spaceSavings = logsUncompressedSize > 0 ? 100 * (1 - logsCompressedSize / logsUncompressedSize) : 0; + const spaceSavings = logsUncompressedSize > 0 ? + 100 * (1 - logsCompressedSize / logsUncompressedSize) : + 0; return (
@@ -113,7 +103,7 @@ const Details = ({stats}) => { begin_timestamp: beginTimeStamp, end_timestamp: endTimeStamp, num_files: numFiles, - num_messages: numMessages + num_messages: numMessages, } = stats; let timeRangeRow = null; diff --git a/components/webui/imports/utils/DbManager.js b/components/webui/imports/utils/DbManager.js index dfddb09ba..1e2de8b14 100644 --- a/components/webui/imports/utils/DbManager.js +++ b/components/webui/imports/utils/DbManager.js @@ -1,5 +1,5 @@ import mysql from "mysql2/promise"; -import {initStatsDbManager} from "../api/ingestion/server/publications"; +import {deinitStatsDbManager, initStatsDbManager} from "../api/ingestion/server/publications"; import {initSearchJobsDbManager} from "../api/search/server/methods"; import {logger} from "./logger"; @@ -73,6 +73,8 @@ const initDbManagers = async ({ * @throws {Error} on error. */ const deinitDbManagers = async () => { + deinitStatsDbManager(); + await dbConnection.end(); }; From 65e3a08b5352db78981aa4e7b200620f214d2b36 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Thu, 8 Feb 2024 12:03:23 +0800 Subject: [PATCH 47/60] Add jsdoc as a dev dependency and enable documentation generation. --- components/webui/.gitignore | 1 + .../api/ingestion/server/publications.js | 1 - components/webui/package-lock.json | 202 ++++++++++++++++++ components/webui/package.json | 4 + 4 files changed, 207 insertions(+), 1 deletion(-) diff --git a/components/webui/.gitignore b/components/webui/.gitignore index c2658d7d1..3465ead02 100644 --- a/components/webui/.gitignore +++ b/components/webui/.gitignore @@ -1 +1,2 @@ node_modules/ +docs/ diff --git a/components/webui/imports/api/ingestion/server/publications.js b/components/webui/imports/api/ingestion/server/publications.js index 0cca38780..4ae04d285 100644 --- a/components/webui/imports/api/ingestion/server/publications.js +++ b/components/webui/imports/api/ingestion/server/publications.js @@ -23,7 +23,6 @@ let refreshMeteorInterval = null; /** * Updates the compression statistics in the StatsCollection. * - * @async * @returns {Promise} */ const refreshCompressionStats = async () => { diff --git a/components/webui/package-lock.json b/components/webui/package-lock.json index 2c1b456c6..9b65d6b99 100644 --- a/components/webui/package-lock.json +++ b/components/webui/package-lock.json @@ -3,6 +3,12 @@ "requires": true, "lockfileVersion": 1, "dependencies": { + "@babel/parser": { + "version": "7.23.9", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.23.9.tgz", + "integrity": "sha512-9tcKgqKbs3xGJ+NtKF2ndOBBLVwPjl1SHxPQkd36r3Dlirw3xWUeGaTbqr7uGZcTaxkVNwc+03SVP7aCdWrTlA==", + "dev": true + }, "@babel/runtime": { "version": "7.23.9", "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.23.9.tgz", @@ -55,6 +61,15 @@ "prop-types": "^15.8.1" } }, + "@jsdoc/salty": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/@jsdoc/salty/-/salty-0.2.7.tgz", + "integrity": "sha512-mh8LbS9d4Jq84KLw8pzho7XC2q2/IJGiJss3xwRoLD1A+EE16SjN4PfaG4jRCzKegTFLlN0Zd8SdUPE6XdoPFg==", + "dev": true, + "requires": { + "lodash": "^4.17.21" + } + }, "@msgpack/msgpack": { "version": "3.0.0-beta2", "resolved": "https://registry.npmjs.org/@msgpack/msgpack/-/msgpack-3.0.0-beta2.tgz", @@ -112,6 +127,28 @@ "tslib": "^2.4.0" } }, + "@types/linkify-it": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@types/linkify-it/-/linkify-it-3.0.5.tgz", + "integrity": "sha512-yg6E+u0/+Zjva+buc3EIb+29XEg4wltq7cSmd4Uc2EE/1nUVmxyzpX6gUXD0V8jIrG0r7YeOGVIbYRkxeooCtw==", + "dev": true + }, + "@types/markdown-it": { + "version": "12.2.3", + "resolved": "https://registry.npmjs.org/@types/markdown-it/-/markdown-it-12.2.3.tgz", + "integrity": "sha512-GKMHFfv3458yYy+v/N8gjufHO6MSZKCOXpZc5GXIWWy8uldwfmPn98vp81gZ5f9SVw8YYBctgfJ22a2d7AOMeQ==", + "dev": true, + "requires": { + "@types/linkify-it": "*", + "@types/mdurl": "*" + } + }, + "@types/mdurl": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@types/mdurl/-/mdurl-1.0.5.tgz", + "integrity": "sha512-6L6VymKTzYSrEf4Nev4Xa1LCHKrlTlYCBMTlQKFuddo1CvQcE52I0mwfOJayueUC7MJuXOeHTcIU683lzd0cUA==", + "dev": true + }, "@types/prop-types": { "version": "15.7.11", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.11.tgz", @@ -150,16 +187,37 @@ "resolved": "https://registry.npmjs.org/@types/warning/-/warning-3.0.3.tgz", "integrity": "sha512-D1XC7WK8K+zZEveUPY+cf4+kgauk8N4eHr/XIHXGlGYkHLud6hK9lYfZk1ry1TNh798cZUCgb6MqGEG8DkJt6Q==" }, + "argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true + }, "async": { "version": "3.2.5", "resolved": "https://registry.npmjs.org/async/-/async-3.2.5.tgz", "integrity": "sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==" }, + "bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==", + "dev": true + }, "bootstrap": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/bootstrap/-/bootstrap-5.3.2.tgz", "integrity": "sha512-D32nmNWiQHo94BKHLmOrdjlL05q1c8oxbtBphQFb9Z5to6eGRDCm0QgeaZ4zFBHzfg2++rqa2JkqCcxDy0sH0g==" }, + "catharsis": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/catharsis/-/catharsis-0.9.0.tgz", + "integrity": "sha512-prMTQVpcns/tzFgFVkVp6ak6RykZyWb3gu8ckUpd6YkTlacOd3DXGJjIpD4Q6zJirizvaiAjSSHlOsA+6sNh2A==", + "dev": true, + "requires": { + "lodash": "^4.17.15" + } + }, "classnames": { "version": "2.5.1", "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.5.1.tgz", @@ -242,6 +300,18 @@ "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==" }, + "entities": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-2.1.0.tgz", + "integrity": "sha512-hCx1oky9PFrJ611mf0ifBLBRW8lUUVRlFolb5gWRfIELabBlbp9xZvrqZLZAs+NxFnbfQoeGd8wDkygjg7U85w==", + "dev": true + }, + "escape-string-regexp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", + "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", + "dev": true + }, "fecha": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", @@ -268,6 +338,12 @@ "is-property": "^1.0.2" } }, + "graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true + }, "history": { "version": "4.10.1", "resolved": "https://registry.npmjs.org/history/-/history-4.10.1.tgz", @@ -335,16 +411,72 @@ "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" }, + "js2xmlparser": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/js2xmlparser/-/js2xmlparser-4.0.2.tgz", + "integrity": "sha512-6n4D8gLlLf1n5mNLQPRfViYzu9RATblzPEtm1SthMX1Pjao0r9YI9nw7ZIfRxQMERS87mcswrg+r/OYrPRX6jA==", + "dev": true, + "requires": { + "xmlcreate": "^2.0.4" + } + }, + "jsdoc": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/jsdoc/-/jsdoc-4.0.2.tgz", + "integrity": "sha512-e8cIg2z62InH7azBBi3EsSEqrKx+nUtAS5bBcYTSpZFA+vhNPyhv8PTFZ0WsjOPDj04/dOLlm08EDcQJDqaGQg==", + "dev": true, + "requires": { + "@babel/parser": "^7.20.15", + "@jsdoc/salty": "^0.2.1", + "@types/markdown-it": "^12.2.3", + "bluebird": "^3.7.2", + "catharsis": "^0.9.0", + "escape-string-regexp": "^2.0.0", + "js2xmlparser": "^4.0.2", + "klaw": "^3.0.0", + "markdown-it": "^12.3.2", + "markdown-it-anchor": "^8.4.1", + "marked": "^4.0.10", + "mkdirp": "^1.0.4", + "requizzle": "^0.2.3", + "strip-json-comments": "^3.1.0", + "underscore": "~1.13.2" + } + }, "json5": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==" }, + "klaw": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/klaw/-/klaw-3.0.0.tgz", + "integrity": "sha512-0Fo5oir+O9jnXu5EefYbVK+mHMBeEVEy2cmctR1O1NECcCkPRreJKrS6Qt/j3KC2C148Dfo9i3pCmCMsdqGr0g==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.9" + } + }, "kuler": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==" }, + "linkify-it": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-3.0.3.tgz", + "integrity": "sha512-ynTsyrFSdE5oZ/O9GEf00kPngmOfVwazR5GKDq6EYfhlpFug3J2zybX56a2PRRpc9P+FuSoGNAwjlbDs9jJBPQ==", + "dev": true, + "requires": { + "uc.micro": "^1.0.1" + } + }, + "lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "dev": true + }, "logform": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/logform/-/logform-2.6.0.tgz", @@ -381,6 +513,37 @@ "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz", "integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==" }, + "markdown-it": { + "version": "12.3.2", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-12.3.2.tgz", + "integrity": "sha512-TchMembfxfNVpHkbtriWltGWc+m3xszaRD0CZup7GFFhzIgQqxIfn3eGj1yZpfuflzPvfkt611B2Q/Bsk1YnGg==", + "dev": true, + "requires": { + "argparse": "^2.0.1", + "entities": "~2.1.0", + "linkify-it": "^3.0.1", + "mdurl": "^1.0.1", + "uc.micro": "^1.0.5" + } + }, + "markdown-it-anchor": { + "version": "8.6.7", + "resolved": "https://registry.npmjs.org/markdown-it-anchor/-/markdown-it-anchor-8.6.7.tgz", + "integrity": "sha512-FlCHFwNnutLgVTflOYHPW2pPcl2AACqVzExlkGQNsi4CJgqOHN7YTgDd4LuhgN1BFO3TS0vLAruV1Td6dwWPJA==", + "dev": true + }, + "marked": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/marked/-/marked-4.3.0.tgz", + "integrity": "sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A==", + "dev": true + }, + "mdurl": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-1.0.1.tgz", + "integrity": "sha512-/sKlQJCBYVY9Ers9hqzKou4H6V5UWc/M59TH2dvkt+84itfnq7uFOMLpOiOS4ujvHP4etln18fmIxA5R5fll0g==", + "dev": true + }, "meteor-node-stubs": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/meteor-node-stubs/-/meteor-node-stubs-1.2.7.tgz", @@ -1158,6 +1321,12 @@ } } }, + "mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true + }, "moment": { "version": "2.30.1", "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz", @@ -1387,6 +1556,15 @@ "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==" }, + "requizzle": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/requizzle/-/requizzle-0.2.4.tgz", + "integrity": "sha512-JRrFk1D4OQ4SqovXOgdav+K8EAhSB/LJZqCz8tbX0KObcdeM15Ss59ozWMBWmmINMagCwmqn4ZNryUGpBsl6Jw==", + "dev": true, + "requires": { + "lodash": "^4.17.21" + } + }, "resolve-pathname": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/resolve-pathname/-/resolve-pathname-3.0.0.tgz", @@ -1447,6 +1625,12 @@ "safe-buffer": "~5.2.0" } }, + "strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true + }, "text-hex": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", @@ -1472,6 +1656,12 @@ "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" }, + "uc.micro": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-1.0.6.tgz", + "integrity": "sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==", + "dev": true + }, "uncontrollable": { "version": "7.2.1", "resolved": "https://registry.npmjs.org/uncontrollable/-/uncontrollable-7.2.1.tgz", @@ -1483,6 +1673,12 @@ "react-lifecycles-compat": "^3.0.4" } }, + "underscore": { + "version": "1.13.6", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz", + "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==", + "dev": true + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -1544,6 +1740,12 @@ "readable-stream": "^3.6.0", "triple-beam": "^1.3.0" } + }, + "xmlcreate": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/xmlcreate/-/xmlcreate-2.0.4.tgz", + "integrity": "sha512-nquOebG4sngPmGPICTS5EnxqhKbCmz5Ox5hsszI2T6U5qdrJizBc+0ilYSEjTSzU0yZcmvppztXe/5Al5fUwdg==", + "dev": true } } } diff --git a/components/webui/package.json b/components/webui/package.json index 49c589ce4..ec07db999 100644 --- a/components/webui/package.json +++ b/components/webui/package.json @@ -3,6 +3,7 @@ "private": true, "scripts": { "start": "meteor run", + "build-docs": "jsdoc -r client imports server tests launcher.js -d docs", "test": "meteor test --once --driver-package meteortesting:mocha", "test-app": "TEST_WATCH=1 meteor test --full-app --driver-package meteortesting:mocha", "visualize": "meteor --production --extra-packages bundle-visualizer" @@ -29,6 +30,9 @@ "winston": "^3.11.0", "winston-daily-rotate-file": "^4.7.1" }, + "devDependencies": { + "jsdoc": "^4.0.2" + }, "meteor": { "mainModule": { "client": "client/main.jsx", From 9f5bea3be17aa3b04b5e4eacb3f08349a30eb4c8 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Thu, 8 Feb 2024 13:43:17 -0500 Subject: [PATCH 48/60] Rename SqlDbArchivesTableName to SqlDbClpArchivesTableName for consistency. --- components/webui/server/main.js | 2 +- components/webui/settings.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/webui/server/main.js b/components/webui/server/main.js index d9a0a3dc7..0c2cb60d4 100644 --- a/components/webui/server/main.js +++ b/components/webui/server/main.js @@ -54,7 +54,7 @@ Meteor.startup(async () => { dbPassword: envVars.CLP_DB_PASS, }, { searchJobsTableName: Meteor.settings.private.SqlDbSearchJobsTableName, - clpArchivesTableName: Meteor.settings.private.SqlDbArchivesTableName, + clpArchivesTableName: Meteor.settings.private.SqlDbClpArchivesTableName, clpFilesTableName: Meteor.settings.private.SqlDbClpFilesTableName, }, ); diff --git a/components/webui/settings.json b/components/webui/settings.json index f6e96bacd..0084bc902 100644 --- a/components/webui/settings.json +++ b/components/webui/settings.json @@ -4,7 +4,7 @@ "SqlDbPort": 3306, "SqlDbName": "clp-db", "SqlDbSearchJobsTableName": "search_jobs", - "SqlDbArchivesTableName": "clp_archives", + "SqlDbClpArchivesTableName": "clp_archives", "SqlDbClpFilesTableName": "clp_files" }, "public": { From 926ad68c09d8d660a150bf76796c9ed65a07d40d Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Thu, 8 Feb 2024 13:47:05 -0500 Subject: [PATCH 49/60] Apply naming conventions to some variables.. --- .../imports/ui/IngestView/IngestView.jsx | 26 +++++++++---------- components/webui/imports/utils/misc.js | 10 +++---- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/components/webui/imports/ui/IngestView/IngestView.jsx b/components/webui/imports/ui/IngestView/IngestView.jsx index 41f1f732f..bae0c4393 100644 --- a/components/webui/imports/ui/IngestView/IngestView.jsx +++ b/components/webui/imports/ui/IngestView/IngestView.jsx @@ -100,15 +100,15 @@ const SpaceSavings = ({stats}) => { */ const Details = ({stats}) => { const { - begin_timestamp: beginTimeStamp, - end_timestamp: endTimeStamp, + begin_timestamp: beginTimestamp, + end_timestamp: endTimestamp, num_files: numFiles, num_messages: numMessages, } = stats; let timeRangeRow = null; - if (null !== endTimeStamp) { - let timestamp_format = "kkkk-MMM-dd HH:mm"; + if (null !== endTimestamp) { + let timestampFormat = "kkkk-MMM-dd HH:mm"; timeRangeRow = (
@@ -116,9 +116,9 @@ const Details = ({stats}) => {
- {DateTime.fromMillis(beginTimeStamp).toFormat(timestamp_format)} + {DateTime.fromMillis(beginTimestamp).toFormat(timestampFormat)} to - {DateTime.fromMillis(endTimeStamp).toFormat(timestamp_format)} + {DateTime.fromMillis(endTimestamp).toFormat(timestampFormat)} time range
@@ -126,9 +126,9 @@ const Details = ({stats}) => { ); } - let num_files_row = null; + let numFilesRow = null; if (null !== numFiles) { - num_files_row = ( + numFilesRow = (
@@ -141,9 +141,9 @@ const Details = ({stats}) => { ); } - let num_messages_row = null; + let numMessagesRow = null; if (null !== numMessages) { - num_messages_row = ( + numMessagesRow = (
@@ -156,7 +156,7 @@ const Details = ({stats}) => { ); } - if (!(timeRangeRow || num_files_row || num_messages_row)) { + if (!(timeRangeRow || numFilesRow || numMessagesRow)) { // No details to display return (<>); } @@ -170,8 +170,8 @@ const Details = ({stats}) => { {timeRangeRow} - {num_files_row} - {num_messages_row} + {numFilesRow} + {numMessagesRow}
diff --git a/components/webui/imports/utils/misc.js b/components/webui/imports/utils/misc.js index 4876f3076..6598fe196 100644 --- a/components/webui/imports/utils/misc.js +++ b/components/webui/imports/utils/misc.js @@ -13,14 +13,14 @@ const sleep = (seconds) => new Promise(r => setTimeout(r, seconds * 1000)); * @returns {string} */ const computeHumanSize = (num) => { - const si_prefixes = ["", "K", "M", "G", "T", "P", "E", "Z"]; - for (let i = 0; i < si_prefixes.length; ++i) { + const siPrefixes = ["", "K", "M", "G", "T", "P", "E", "Z"]; + for (let i = 0; i < siPrefixes.length; ++i) { if (Math.abs(num) < 1024.0) { - return "" + Math.round(num) + " " + si_prefixes[i] + "B"; + return `${Math.round(num)} ${siPrefixes[i]}B`; } num /= 1024.0; } - return Math.round(num) + " B"; + return `${Math.round(num)} B`; }; -export {sleep, computeHumanSize}; \ No newline at end of file +export {computeHumanSize, sleep}; From 1e58b1ffc3d3f9614f657e4530d81a2db570ccc2 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Thu, 8 Feb 2024 14:37:28 -0500 Subject: [PATCH 50/60] Support big numbers from MySQL; Cast stats to numbers before rendering. --- components/webui/imports/ui/IngestView/IngestView.jsx | 10 ++++------ components/webui/imports/utils/DbManager.js | 2 ++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/components/webui/imports/ui/IngestView/IngestView.jsx b/components/webui/imports/ui/IngestView/IngestView.jsx index bae0c4393..0fe43eda0 100644 --- a/components/webui/imports/ui/IngestView/IngestView.jsx +++ b/components/webui/imports/ui/IngestView/IngestView.jsx @@ -99,12 +99,10 @@ const SpaceSavings = ({stats}) => { * @returns {JSX.Element} */ const Details = ({stats}) => { - const { - begin_timestamp: beginTimestamp, - end_timestamp: endTimestamp, - num_files: numFiles, - num_messages: numMessages, - } = stats; + const beginTimestamp = Number(stats.begin_timestamp); + const endTimestamp = Number(stats.end_timestamp); + const numFiles = Number(stats.num_files); + const numMessages = Number(stats.num_messages); let timeRangeRow = null; if (null !== endTimestamp) { diff --git a/components/webui/imports/utils/DbManager.js b/components/webui/imports/utils/DbManager.js index 1e2de8b14..09f350521 100644 --- a/components/webui/imports/utils/DbManager.js +++ b/components/webui/imports/utils/DbManager.js @@ -50,6 +50,8 @@ const initDbManagers = async ({ database: dbName, user: dbUser, password: dbPassword, + bigNumberStrings: true, + supportBigNumbers: true, }); await dbConnection.connect(); From c0fcb78b033e6aa87802fd6ee5ae2f1afc09cd66 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Thu, 8 Feb 2024 15:17:32 -0500 Subject: [PATCH 51/60] Allow stats to be null. --- .../imports/ui/IngestView/IngestView.jsx | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/components/webui/imports/ui/IngestView/IngestView.jsx b/components/webui/imports/ui/IngestView/IngestView.jsx index 0fe43eda0..4f2e75607 100644 --- a/components/webui/imports/ui/IngestView/IngestView.jsx +++ b/components/webui/imports/ui/IngestView/IngestView.jsx @@ -99,10 +99,12 @@ const SpaceSavings = ({stats}) => { * @returns {JSX.Element} */ const Details = ({stats}) => { - const beginTimestamp = Number(stats.begin_timestamp); - const endTimestamp = Number(stats.end_timestamp); - const numFiles = Number(stats.num_files); - const numMessages = Number(stats.num_messages); + const { + begin_timestamp: beginTimestamp, + end_timestamp: endTimestamp, + num_files: numFiles, + num_messages: numMessages, + } = stats; let timeRangeRow = null; if (null !== endTimestamp) { @@ -114,9 +116,9 @@ const Details = ({stats}) => {
- {DateTime.fromMillis(beginTimestamp).toFormat(timestampFormat)} + {DateTime.fromMillis(Number(beginTimestamp)).toFormat(timestampFormat)} to - {DateTime.fromMillis(endTimestamp).toFormat(timestampFormat)} + {DateTime.fromMillis(Number(endTimestamp)).toFormat(timestampFormat)} time range
@@ -132,7 +134,7 @@ const Details = ({stats}) => {
- {numFiles.toLocaleString()} + {Number(numFiles).toLocaleString()} files
@@ -147,7 +149,9 @@ const Details = ({stats}) => {
- {numMessages.toLocaleString()} + {Number(numMessages).toLocaleString()} messages
From 59263cdbfa6de81aa861fe53744995ba607b086e Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Thu, 8 Feb 2024 15:26:47 -0500 Subject: [PATCH 52/60] Minor clean-up. --- components/webui/.gitignore | 2 +- .../api/ingestion/server/StatsDbManager.js | 18 +++++++++++++----- .../api/ingestion/server/publications.js | 10 +--------- .../webui/imports/api/search/server/methods.js | 1 - .../webui/imports/api/user/client/methods.js | 2 +- components/webui/imports/utils/DbManager.js | 5 ++--- 6 files changed, 18 insertions(+), 20 deletions(-) diff --git a/components/webui/.gitignore b/components/webui/.gitignore index 3465ead02..c40aa38d0 100644 --- a/components/webui/.gitignore +++ b/components/webui/.gitignore @@ -1,2 +1,2 @@ -node_modules/ docs/ +node_modules/ diff --git a/components/webui/imports/api/ingestion/server/StatsDbManager.js b/components/webui/imports/api/ingestion/server/StatsDbManager.js index 31ddf842f..194cd6218 100644 --- a/components/webui/imports/api/ingestion/server/StatsDbManager.js +++ b/components/webui/imports/api/ingestion/server/StatsDbManager.js @@ -11,12 +11,12 @@ const CLP_FILES_TABLE_COLUMN_NAMES = { }; /** - * Class for retrieving compression stats in the database. + * Class for retrieving compression stats from the database. */ class StatsDbManager { #sqlDbConnection; - #clpFilesTableName; #clpArchivesTableName; + #clpFilesTableName; /** * @param {mysql.Connection} sqlDbConnection @@ -34,9 +34,19 @@ class StatsDbManager { this.#clpFilesTableName = clpFilesTableName; } + /** + * @typedef {object} Stats + * @property {number|null} begin_timestamp + * @property {number|null} end_timestamp + * @property {number|null} total_uncompressed_size + * @property {number|null} total_compressed_size + * @property {number|null} num_files + * @property {number|null} num_messages + */ + /** * Queries compression stats. - * @returns {Promise} stats + * @returns {Promise} * @throws {Error} on error. */ async getCompressionStats() { @@ -56,8 +66,6 @@ class StatsDbManager { SUM(${CLP_FILES_TABLE_COLUMN_NAMES.NUM_MESSAGES}) AS num_messages FROM ${this.#clpFilesTableName}) b;`, ); - - return queryStats[0]; } } diff --git a/components/webui/imports/api/ingestion/server/publications.js b/components/webui/imports/api/ingestion/server/publications.js index 4ae04d285..6a2279285 100644 --- a/components/webui/imports/api/ingestion/server/publications.js +++ b/components/webui/imports/api/ingestion/server/publications.js @@ -49,9 +49,6 @@ const refreshCompressionStats = async () => { * @param {object} tableNames * @param {string} tableNames.clpArchivesTableName * @param {string} tableNames.clpFilesTableName - * @param {string} clpArchivesTableName - * @param {string} clpFilesTableName - * @returns {void} * @throws {Error} on error. */ const initStatsDbManager = (sqlDbConnection, { @@ -66,9 +63,6 @@ const initStatsDbManager = (sqlDbConnection, { refreshMeteorInterval = Meteor.setInterval(refreshCompressionStats, STATS_REFRESH_INTERVAL_MS); }; -/** - * @returns {void} - */ const deinitStatsDbManager = () => { if (null !== refreshMeteorInterval) { Meteor.clearInterval(refreshMeteorInterval); @@ -77,15 +71,13 @@ const deinitStatsDbManager = () => { }; /** - * Updates and Publishes compression statistics. + * Updates and publishes compression statistics. * * @param {string} publicationName * * @returns {Mongo.Cursor} */ Meteor.publish(Meteor.settings.public.StatsCollectionName, async () => { - // logger.debug(`Subscription '${Meteor.settings.public.StatsCollectionName}'`); - await refreshCompressionStats(); const filter = { diff --git a/components/webui/imports/api/search/server/methods.js b/components/webui/imports/api/search/server/methods.js index 7584dab12..2ca450322 100644 --- a/components/webui/imports/api/search/server/methods.js +++ b/components/webui/imports/api/search/server/methods.js @@ -14,7 +14,6 @@ let searchJobsDbManager = null; * @param {mysql.Connection} sqlDbConnection * @param {object} tableNames * @param {string} tableNames.searchJobsTableName - * @returns {void} * @throws {Error} on error. */ const initSearchJobsDbManager = (sqlDbConnection, {searchJobsTableName}) => { diff --git a/components/webui/imports/api/user/client/methods.js b/components/webui/imports/api/user/client/methods.js index d14040df7..63271dc0f 100644 --- a/components/webui/imports/api/user/client/methods.js +++ b/components/webui/imports/api/user/client/methods.js @@ -64,7 +64,7 @@ export const loginWithUsername = (username) => { * Attempts to log in a user using a stored username or register a new one if none is found. * * @returns {Promise} true if the login is successful - * false if there's an error during login or registration + * false if there's an error during login or registration */ export const login = async () => { let username = localStorage.getItem(LOCAL_STORAGE_KEY_USERNAME); diff --git a/components/webui/imports/utils/DbManager.js b/components/webui/imports/utils/DbManager.js index 09f350521..2f316e59e 100644 --- a/components/webui/imports/utils/DbManager.js +++ b/components/webui/imports/utils/DbManager.js @@ -39,7 +39,7 @@ const initDbManagers = async ({ clpFilesTableName }) => { if (null !== dbConnection) { - logger.error("This method should not be called twice. "); + logger.error("This method should not be called twice."); return; } @@ -63,8 +63,7 @@ const initDbManagers = async ({ clpFilesTableName, }); } catch (e) { - logger.error("Unable to create MySQL / mariadb connection.", - e.toString()); + logger.error("Unable to create MySQL / mariadb connection.", e.toString()); throw e; } }; From ee0b7364b40f5f6e808cc041ff6f592a1a8f9920 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Thu, 8 Feb 2024 15:27:09 -0500 Subject: [PATCH 53/60] Pass clp table names from startup script. --- .../clp-package-utils/clp_package_utils/scripts/start_clp.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 51b72d353..b97c0e152 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -13,6 +13,7 @@ import yaml from clp_py_utils.clp_config import ( + CLP_METADATA_TABLE_PREFIX, CLPConfig, COMPRESSION_SCHEDULER_COMPONENT_NAME, COMPRESSION_WORKER_COMPONENT_NAME, @@ -678,6 +679,8 @@ def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts "SqlDbPort": clp_config.database.port, "SqlDbName": clp_config.database.name, "SqlDbSearchJobsTableName": SEARCH_JOBS_TABLE_NAME, + "SqlDbClpArchivesTableName": f"{CLP_METADATA_TABLE_PREFIX}archives", + "SqlDbClpFilesTableName": f"{CLP_METADATA_TABLE_PREFIX}files", } } update_meteor_settings("", meteor_settings, meteor_settings_updates) From a1f8dde95cac22232ea297b91e7f1073f6020ef4 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Thu, 8 Feb 2024 16:00:10 -0500 Subject: [PATCH 54/60] Fix broken log. --- .../webui/imports/api/search/server/SearchJobsDbManager.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/webui/imports/api/search/server/SearchJobsDbManager.js b/components/webui/imports/api/search/server/SearchJobsDbManager.js index b6c779e92..a5c02f7ad 100644 --- a/components/webui/imports/api/search/server/SearchJobsDbManager.js +++ b/components/webui/imports/api/search/server/SearchJobsDbManager.js @@ -88,7 +88,7 @@ class SearchJobsDbManager { throw new Error(`Job ${jobId} was cancelled.`); } else if (JobStatus.SUCCESS !== status) { throw new Error(`Job ${jobId} exited with unexpected status=${status}: ` - `${Object.keys(JobStatus)[status]}.`); + + `${Object.keys(JobStatus)[status]}.`); } break; } From c3eae99207029799f7d4f7007241fcd193c27ad1 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Thu, 8 Feb 2024 17:59:46 -0500 Subject: [PATCH 55/60] Minor refactoring. --- components/webui/imports/ui/SearchView/SearchView.jsx | 8 ++++---- components/webui/imports/ui/SearchView/datetime.js | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/webui/imports/ui/SearchView/SearchView.jsx b/components/webui/imports/ui/SearchView/SearchView.jsx index 46f6f0bc0..8a366ae46 100644 --- a/components/webui/imports/ui/SearchView/SearchView.jsx +++ b/components/webui/imports/ui/SearchView/SearchView.jsx @@ -14,7 +14,7 @@ import SearchJobCollectionsManager from "../../api/search/SearchJobCollectionsMa import "react-datepicker/dist/react-datepicker.css"; import {LOCAL_STORAGE_KEYS} from "../constants"; -import {changeTimezoneToUtcWithoutChangingTime, DEFAULT_TIME_RANGE_GETTER} from "./datetime"; +import {changeTimezoneToUtcWithoutChangingTime, DEFAULT_TIME_RANGE} from "./datetime"; import {SearchControls} from "./SearchControls.jsx"; import {SearchResults} from "./SearchResults.jsx"; import {VISIBLE_RESULTS_LIMIT_INITIAL} from "./SearchResultsTable.jsx"; @@ -38,7 +38,7 @@ const SearchView = () => { // Query options const [queryString, setQueryString] = useState(""); - const [timeRange, setTimeRange] = useState(DEFAULT_TIME_RANGE_GETTER); + const [timeRange, setTimeRange] = useState(DEFAULT_TIME_RANGE); const [visibleSearchResultsLimit, setVisibleSearchResultsLimit] = useState( VISIBLE_RESULTS_LIMIT_INITIAL); const [fieldToSortBy, setFieldToSortBy] = useState({ @@ -116,8 +116,8 @@ const SearchView = () => { setLocalLastSearchSignal(SearchSignal.REQ_QUERYING); resetVisibleResultSettings(); - const timestampBeginMillis = changeTimezoneToUtcWithoutChangingTime(timeRange.begin). - getTime(); + const timestampBeginMillis = changeTimezoneToUtcWithoutChangingTime(timeRange.begin) + .getTime(); const timestampEndMillis = changeTimezoneToUtcWithoutChangingTime(timeRange.end).getTime(); const args = { diff --git a/components/webui/imports/ui/SearchView/datetime.js b/components/webui/imports/ui/SearchView/datetime.js index ca40949ef..080fd2f9f 100644 --- a/components/webui/imports/ui/SearchView/datetime.js +++ b/components/webui/imports/ui/SearchView/datetime.js @@ -111,6 +111,6 @@ export const changeTimezoneToUtcWithoutChangingTime = (date) => { )); }; -export const DEFAULT_TIME_RANGE_GETTER = computeTimeRange( +export const DEFAULT_TIME_RANGE = computeTimeRange( `${TIME_RANGE_UNIT.ALL}_${TIME_RANGE_MODIFIER.NONE}_0`, ); From ce7d7cf451dd3e5d12652152314a79cc8d0c47d8 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 9 Feb 2024 15:26:22 +0800 Subject: [PATCH 56/60] Minor refactoring and docstring fix; Replace path to NodeJS binary with variable NODEJS_BIN_DIR in Taskfile. --- Taskfile.yml | 2 +- .../imports/api/ingestion/collections.js | 1 + .../webui/imports/api/search/collections.js | 22 ++++++---- .../webui/imports/api/search/constants.js | 41 +++++++++++------- .../api/search/server/SearchJobsDbManager.js | 1 + .../imports/api/search/server/collections.js | 4 +- .../imports/api/search/server/methods.js | 1 + .../imports/api/search/server/publications.js | 1 + .../webui/imports/api/user/client/methods.js | 9 ++-- .../webui/imports/api/user/server/methods.js | 1 + components/webui/imports/ui/App.jsx | 4 +- .../imports/ui/SearchView/SearchControls.jsx | 5 ++- .../imports/ui/SearchView/SearchResults.jsx | 5 ++- .../ui/SearchView/SearchResultsHeader.jsx | 1 + .../ui/SearchView/SearchResultsTable.jsx | 42 ++++++++++++------- .../imports/ui/SearchView/SearchView.jsx | 5 ++- .../webui/imports/ui/SearchView/datetime.js | 16 +++++-- .../webui/imports/ui/Sidebar/Sidebar.jsx | 1 + components/webui/imports/ui/constants.js | 1 - components/webui/imports/utils/logger.js | 9 ++-- components/webui/server/main.js | 1 + 21 files changed, 116 insertions(+), 57 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index db7fc2f03..a2b2f7a2c 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -83,7 +83,7 @@ tasks: "{{.CORE_COMPONENT_BUILD_DIR}}/clo" "{{.CORE_COMPONENT_BUILD_DIR}}/clp" "{{.CORE_COMPONENT_BUILD_DIR}}/clp-s" - "{{.BUILD_DIR}}/nodejs/node/bin/node" + "{{.NODEJS_BIN_DIR}}/node" "{{.PACKAGE_BUILD_DIR}}/bin/" - "mkdir -p '{{.PACKAGE_BUILD_DIR}}/var/www/'" - >- diff --git a/components/webui/imports/api/ingestion/collections.js b/components/webui/imports/api/ingestion/collections.js index fd0c8f4ba..b886d2d66 100644 --- a/components/webui/imports/api/ingestion/collections.js +++ b/components/webui/imports/api/ingestion/collections.js @@ -1,5 +1,6 @@ import {Mongo} from "meteor/mongo"; + const StatsCollection = new Mongo.Collection(Meteor.settings.public.StatsCollectionName); const STATS_COLLECTION_ID_COMPRESSION = "compression_stats"; diff --git a/components/webui/imports/api/search/collections.js b/components/webui/imports/api/search/collections.js index 84a7a6d98..d8406188e 100644 --- a/components/webui/imports/api/search/collections.js +++ b/components/webui/imports/api/search/collections.js @@ -1,26 +1,28 @@ import {Mongo} from "meteor/mongo"; import {INVALID_JOB_ID, SearchSignal} from "./constants"; + /** * A MongoDB collection for storing metadata about search results. - * - * @constant */ -export const SearchResultsMetadataCollection = new Mongo.Collection(Meteor.settings.public.SearchResultsMetadataCollectionName); +const SearchResultsMetadataCollection = new Mongo.Collection( + Meteor.settings.public.SearchResultsMetadataCollectionName); + /** - * Initializes the search event collection by inserting a default document if the collection is empty. + * Initializes the search event collection by inserting a default document if the collection is + * empty. */ -export const initSearchEventCollection = () => { +const initSearchEventCollection = () => { // create the collection if not exists if (SearchResultsMetadataCollection.countDocuments() === 0) { SearchResultsMetadataCollection.insert({ _id: INVALID_JOB_ID.toString(), lastEvent: SearchSignal.NONE, errorMsg: null, - numTotalResults: -1 + numTotalResults: -1, }); } -} +}; /** * Adds the given sort to the find options for a MongoDB collection. @@ -28,11 +30,13 @@ export const initSearchEventCollection = () => { * (ASC = 1, DESC = -1). * @param {Object} findOptions */ -export const addSortToMongoFindOptions = (fieldToSortBy, findOptions) => { +const addSortToMongoFindOptions = (fieldToSortBy, findOptions) => { if (fieldToSortBy) { findOptions["sort"] = { [fieldToSortBy.name]: fieldToSortBy.direction, _id: fieldToSortBy.direction, }; } -} +}; + +export {addSortToMongoFindOptions, initSearchEventCollection, SearchResultsMetadataCollection}; diff --git a/components/webui/imports/api/search/constants.js b/components/webui/imports/api/search/constants.js index 9d7b92045..ee0e2b5f4 100644 --- a/components/webui/imports/api/search/constants.js +++ b/components/webui/imports/api/search/constants.js @@ -5,10 +5,9 @@ let enumSearchSignal; * This includes request and response signals for various search operations and their respective * states. * - * @constant * @type {Object} */ -export const SearchSignal = Object.freeze({ +const SearchSignal = Object.freeze({ NONE: (enumSearchSignal = 0), REQ_MASK: (enumSearchSignal = 0x10000000), @@ -21,10 +20,13 @@ export const SearchSignal = Object.freeze({ RESP_QUERYING: ++enumSearchSignal, }); -export const isSearchSignalReq = (s) => (0 !== (SearchSignal.REQ_MASK & s)); -export const isSearchSignalRsp = (s) => (0 !== (SearchSignal.RESP_MASK & s)); -export const isSearchSignalQuerying = (s) => ( - [SearchSignal.REQ_QUERYING, SearchSignal.RESP_QUERYING].includes(s) +const isSearchSignalReq = (s) => (0 !== (SearchSignal.REQ_MASK & s)); +const isSearchSignalResp = (s) => (0 !== (SearchSignal.RESP_MASK & s)); +const isSearchSignalQuerying = (s) => ( + [ + SearchSignal.REQ_QUERYING, + SearchSignal.RESP_QUERYING, + ].includes(s) ); let enumJobStatus; @@ -32,22 +34,31 @@ let enumJobStatus; * Enum of job statuses, matching the `SearchJobStatus` class in * `job_orchestration.search_scheduler.constants`. * - * @constant * @type {Object} */ -export const JobStatus = Object.freeze({ - PENDING: (enumJobStatus=0), +const JobStatus = Object.freeze({ + PENDING: (enumJobStatus = 0), RUNNING: ++enumJobStatus, SUCCESS: ++enumJobStatus, FAILED: ++enumJobStatus, CANCELLING: ++enumJobStatus, - CANCELLED: ++enumJobStatus -}) + CANCELLED: ++enumJobStatus, +}); -export const JOB_STATUS_WAITING_STATES = [ +const JOB_STATUS_WAITING_STATES = [ JobStatus.PENDING, JobStatus.RUNNING, - JobStatus.CANCELLING -] + JobStatus.CANCELLING, +]; + +const INVALID_JOB_ID = -1; -export const INVALID_JOB_ID = -1; +export { + SearchSignal, + isSearchSignalReq, + isSearchSignalResp, + isSearchSignalQuerying, + JobStatus, + JOB_STATUS_WAITING_STATES, + INVALID_JOB_ID, +}; diff --git a/components/webui/imports/api/search/server/SearchJobsDbManager.js b/components/webui/imports/api/search/server/SearchJobsDbManager.js index a5c02f7ad..b56377e61 100644 --- a/components/webui/imports/api/search/server/SearchJobsDbManager.js +++ b/components/webui/imports/api/search/server/SearchJobsDbManager.js @@ -3,6 +3,7 @@ import msgpack from "@msgpack/msgpack"; import {sleep} from "../../../utils/misc"; import {JOB_STATUS_WAITING_STATES, JobStatus} from "../constants"; + const SEARCH_JOBS_TABLE_COLUMN_NAMES = { ID: "id", STATUS: "status", diff --git a/components/webui/imports/api/search/server/collections.js b/components/webui/imports/api/search/server/collections.js index 7ca663cb4..5792105d9 100644 --- a/components/webui/imports/api/search/server/collections.js +++ b/components/webui/imports/api/search/server/collections.js @@ -1,3 +1,5 @@ import SearchJobCollectionsManager from "../SearchJobCollectionsManager"; -export const searchJobCollectionsManager = new SearchJobCollectionsManager(); +const searchJobCollectionsManager = new SearchJobCollectionsManager(); + +export {searchJobCollectionsManager}; diff --git a/components/webui/imports/api/search/server/methods.js b/components/webui/imports/api/search/server/methods.js index 2ca450322..098acd220 100644 --- a/components/webui/imports/api/search/server/methods.js +++ b/components/webui/imports/api/search/server/methods.js @@ -5,6 +5,7 @@ import {SearchSignal} from "../constants"; import {searchJobCollectionsManager} from "./collections"; import SearchJobsDbManager from "./SearchJobsDbManager"; + /** * @type {SearchJobsDbManager|null} */ diff --git a/components/webui/imports/api/search/server/publications.js b/components/webui/imports/api/search/server/publications.js index da1e1ca32..164a6acce 100644 --- a/components/webui/imports/api/search/server/publications.js +++ b/components/webui/imports/api/search/server/publications.js @@ -4,6 +4,7 @@ import {logger} from "/imports/utils/logger"; import {addSortToMongoFindOptions, SearchResultsMetadataCollection} from "../collections"; import {searchJobCollectionsManager} from "./collections"; + /** * Publishes search results metadata for a specific job. * diff --git a/components/webui/imports/api/user/client/methods.js b/components/webui/imports/api/user/client/methods.js index 63271dc0f..c27586585 100644 --- a/components/webui/imports/api/user/client/methods.js +++ b/components/webui/imports/api/user/client/methods.js @@ -1,6 +1,7 @@ import {Meteor} from "meteor/meteor"; import {v4 as uuidv4} from "uuid"; + // TODO: implement a full-fledged registration sys const LOCAL_STORAGE_KEY_USERNAME = "username"; const DUMMY_PASSWORD = "DummyPassword"; @@ -16,7 +17,7 @@ const CONST_MAX_LOGIN_RETRY = 3; * @returns {Promise} true if the registration and login are successful * false if there's an error during registration or login */ -export const registerAndLoginWithUsername = async (username) => { +const registerAndLoginWithUsername = async (username) => { return new Promise((resolve) => { Meteor.call("user.create", { username, @@ -42,7 +43,7 @@ export const registerAndLoginWithUsername = async (username) => { * false if there's an error during login or if the maximum login * retries are reached */ -export const loginWithUsername = (username) => { +const loginWithUsername = (username) => { return new Promise((resolve) => { Meteor.loginWithPassword(username, DUMMY_PASSWORD, (error) => { if (!error) { @@ -66,7 +67,7 @@ export const loginWithUsername = (username) => { * @returns {Promise} true if the login is successful * false if there's an error during login or registration */ -export const login = async () => { +const login = async () => { let username = localStorage.getItem(LOCAL_STORAGE_KEY_USERNAME); let result; @@ -79,3 +80,5 @@ export const login = async () => { return result; }; + +export {registerAndLoginWithUsername, loginWithUsername, login}; diff --git a/components/webui/imports/api/user/server/methods.js b/components/webui/imports/api/user/server/methods.js index 60518e430..c135dcece 100644 --- a/components/webui/imports/api/user/server/methods.js +++ b/components/webui/imports/api/user/server/methods.js @@ -2,6 +2,7 @@ import {Meteor} from "meteor/meteor"; import {Accounts} from "meteor/accounts-base"; import {logger} from "/imports/utils/logger"; + Meteor.methods({ /** * Creates a user account with a provided username and password. diff --git a/components/webui/imports/ui/App.jsx b/components/webui/imports/ui/App.jsx index 14d744431..1eed6d6d2 100644 --- a/components/webui/imports/ui/App.jsx +++ b/components/webui/imports/ui/App.jsx @@ -17,7 +17,7 @@ const ROUTES = [ {path: "/search", label: "Search", icon: faSearch, component: SearchView}, ]; -export const App = () => { +const App = () => { const [loggedIn, setLoggedIn] = React.useState(false); const [isSidebarCollapsed, setSidebarStateCollapsed] = React.useState( "true" === localStorage.getItem(LOCAL_STORAGE_KEYS.IS_SIDEBAR_COLLAPSED) @@ -70,3 +70,5 @@ export const App = () => { ); } + +export {App}; \ No newline at end of file diff --git a/components/webui/imports/ui/SearchView/SearchControls.jsx b/components/webui/imports/ui/SearchView/SearchControls.jsx index 9e5703690..83455c62f 100644 --- a/components/webui/imports/ui/SearchView/SearchControls.jsx +++ b/components/webui/imports/ui/SearchView/SearchControls.jsx @@ -20,6 +20,7 @@ import {isSearchSignalQuerying, isSearchSignalReq, SearchSignal} from "../../api import "./SearchControls.scss"; + /** * Renders a date picker control for selecting date and time. * @@ -168,7 +169,7 @@ const SearchFilterControlsDrawer = ({ * @param {function} onCancelOperation callback to cancel the ongoing search operation * @returns {JSX.Element} */ -export const SearchControls = ({ +const SearchControls = ({ queryString, setQueryString, timeRange, @@ -279,3 +280,5 @@ export const SearchControls = ({ />} ; }; + +export default SearchControls; diff --git a/components/webui/imports/ui/SearchView/SearchResults.jsx b/components/webui/imports/ui/SearchView/SearchResults.jsx index 386189656..7e4edef1f 100644 --- a/components/webui/imports/ui/SearchView/SearchResults.jsx +++ b/components/webui/imports/ui/SearchView/SearchResults.jsx @@ -2,6 +2,7 @@ import React from "react"; import {SearchResultsHeader} from "./SearchResultsHeader.jsx"; import {SearchResultsTable} from "./SearchResultsTable.jsx"; + /** * Renders the search results, which includes the search results header and the search results * table. @@ -17,7 +18,7 @@ import {SearchResultsTable} from "./SearchResultsTable.jsx"; * @param {function} setMaxLinesPerResult callback to set maxLinesPerResult * @returns {JSX.Element} */ -export const SearchResults = ({ +const SearchResults = ({ jobId, searchResults, resultsMetadata, @@ -54,3 +55,5 @@ export const SearchResults = ({ } ; }; + +export default SearchResults; diff --git a/components/webui/imports/ui/SearchView/SearchResultsHeader.jsx b/components/webui/imports/ui/SearchView/SearchResultsHeader.jsx index 16c3defd8..eac9cbe7d 100644 --- a/components/webui/imports/ui/SearchView/SearchResultsHeader.jsx +++ b/components/webui/imports/ui/SearchView/SearchResultsHeader.jsx @@ -15,6 +15,7 @@ import {SearchSignal} from "../../api/search/constants"; import "./SearchResultsHeader.scss"; + /** * Renders the header for the search results, which includes the job ID, the number of results * found, and a control for setting the maximum number of lines per search result. diff --git a/components/webui/imports/ui/SearchView/SearchResultsTable.jsx b/components/webui/imports/ui/SearchView/SearchResultsTable.jsx index bd38a3e4a..0ac92868b 100644 --- a/components/webui/imports/ui/SearchView/SearchResultsTable.jsx +++ b/components/webui/imports/ui/SearchView/SearchResultsTable.jsx @@ -3,27 +3,37 @@ import {FontAwesomeIcon} from "@fortawesome/react-fontawesome"; import React, {useEffect, useState} from "react"; import {Spinner, Table} from "react-bootstrap"; import ReactVisibilitySensor from "react-visibility-sensor"; +import "./SearchResultsTable.scss"; -export const VISIBLE_RESULTS_LIMIT_INITIAL = 10; -const VISIBLE_RESULTS_LIMIT_INCREMENT = 10; -import "./SearchResultsTable.scss"; +/** + * The initial visible results limit. + * + * @type {number} + * @constant + */ +const VISIBLE_RESULTS_LIMIT_INITIAL = 10; +/** + * The increment value for the visible results limit. + * + * @type {number} + * @constant + */ +const VISIBLE_RESULTS_LIMIT_INCREMENT = 10; /** - * Renders a table displaying search results, which includes features like sorting and dynamic - * loading of more results when scrolling to the bottom, up to the number of results available on - * the server. + * Represents a table component to display search results. * - * @param {Object[]} searchResults results to display - * @param {number} maxLinesPerResult maximum number of lines to display per search result - * @param {Object} fieldToSortBy used for sorting results - * @param {function} setFieldToSortBy callback to set fieldToSortBy - * @param {number} numResultsOnServer total number of results available on the server - * @param {number} visibleSearchResultsLimit limit of visible search results - * @param {function} setVisibleSearchResultsLimit callback to set visibleSearchResultsLimit - * @returns {JSX.Element} + * @param {Object} searchResults - The array of search results. + * @param {number} maxLinesPerResult - The maximum number of lines to show per search result. + * @param {Object} fieldToSortBy - The field to sort the search results by. + * @param {function} setFieldToSortBy - The function to set the field to sort by. + * @param {number} numResultsOnServer - The total number of search results on the server. + * @param {number} visibleSearchResultsLimit - The number of search results currently visible. + * @param {function} setVisibleSearchResultsLimit - The function to set the number of visible search results. + * @returns {JSX.Element} - The rendered SearchResultsTable component. */ -export const SearchResultsTable = ({ +const SearchResultsTable = ({ searchResults, maxLinesPerResult, fieldToSortBy, @@ -131,3 +141,5 @@ export const SearchResultsTable = ({ ); }; + +export {SearchResultsTable, VISIBLE_RESULTS_LIMIT_INITIAL}; diff --git a/components/webui/imports/ui/SearchView/SearchView.jsx b/components/webui/imports/ui/SearchView/SearchView.jsx index 8a366ae46..85e177955 100644 --- a/components/webui/imports/ui/SearchView/SearchView.jsx +++ b/components/webui/imports/ui/SearchView/SearchView.jsx @@ -15,10 +15,11 @@ import SearchJobCollectionsManager from "../../api/search/SearchJobCollectionsMa import "react-datepicker/dist/react-datepicker.css"; import {LOCAL_STORAGE_KEYS} from "../constants"; import {changeTimezoneToUtcWithoutChangingTime, DEFAULT_TIME_RANGE} from "./datetime"; -import {SearchControls} from "./SearchControls.jsx"; -import {SearchResults} from "./SearchResults.jsx"; +import SearchControls from "./SearchControls.jsx"; +import SearchResults from "./SearchResults.jsx"; import {VISIBLE_RESULTS_LIMIT_INITIAL} from "./SearchResultsTable.jsx"; + // for pseudo progress bar const PROGRESS_INCREMENT = 5; const PROGRESS_INTERVAL_MS = 100; diff --git a/components/webui/imports/ui/SearchView/datetime.js b/components/webui/imports/ui/SearchView/datetime.js index 080fd2f9f..adb41bd26 100644 --- a/components/webui/imports/ui/SearchView/datetime.js +++ b/components/webui/imports/ui/SearchView/datetime.js @@ -1,5 +1,6 @@ import {DateTime} from "luxon"; + const TIME_RANGE_UNIT = Object.freeze({ ALL: "all", MINUTE: "minute", @@ -37,7 +38,7 @@ const dateTimeToDateWithoutChangingTimestamp = (dateTime) => { }).toJSDate(); }; -export const TIME_RANGE_PRESET_LABEL = Object.freeze({ +const TIME_RANGE_PRESET_LABEL = Object.freeze({ [`${TIME_RANGE_UNIT.MINUTE}_${TIME_RANGE_MODIFIER.LAST}_15`]: "Last 15 Minutes", [`${TIME_RANGE_UNIT.MINUTE}_${TIME_RANGE_MODIFIER.LAST}_60`]: "Last 60 Minutes", [`${TIME_RANGE_UNIT.HOUR}_${TIME_RANGE_MODIFIER.LAST}_4`]: "Last 4 Hours", @@ -59,7 +60,7 @@ export const TIME_RANGE_PRESET_LABEL = Object.freeze({ * @param {string} token representing the time range to compute; format: `unit_modifier_amount` * @returns {Object} containing Date objects representing the computed begin and end time range */ -export const computeTimeRange = (token) => () => { +const computeTimeRange = (token) => () => { const [unit, modifier, amount] = token.split("_"); let endTime; let beginTime; @@ -99,7 +100,7 @@ export const computeTimeRange = (token) => () => { * @param {Date} date Date object to convert to UTC * @returns {Date} A new Date object with the same time values in UTC timezone */ -export const changeTimezoneToUtcWithoutChangingTime = (date) => { +const changeTimezoneToUtcWithoutChangingTime = (date) => { return new Date(Date.UTC( date.getFullYear(), date.getMonth(), @@ -111,6 +112,13 @@ export const changeTimezoneToUtcWithoutChangingTime = (date) => { )); }; -export const DEFAULT_TIME_RANGE = computeTimeRange( +const DEFAULT_TIME_RANGE = computeTimeRange( `${TIME_RANGE_UNIT.ALL}_${TIME_RANGE_MODIFIER.NONE}_0`, ); + +export { + TIME_RANGE_PRESET_LABEL, + computeTimeRange, + changeTimezoneToUtcWithoutChangingTime, + DEFAULT_TIME_RANGE, +}; diff --git a/components/webui/imports/ui/Sidebar/Sidebar.jsx b/components/webui/imports/ui/Sidebar/Sidebar.jsx index 945bd993b..67e55337f 100644 --- a/components/webui/imports/ui/Sidebar/Sidebar.jsx +++ b/components/webui/imports/ui/Sidebar/Sidebar.jsx @@ -7,6 +7,7 @@ import { faAngleDoubleRight, } from "@fortawesome/free-solid-svg-icons"; + /** * Renders a sidebar navigation component, which includes navigation links and a toggle for * collapsing or expanding the sidebar. diff --git a/components/webui/imports/ui/constants.js b/components/webui/imports/ui/constants.js index 7cc7b527f..0c2394a34 100644 --- a/components/webui/imports/ui/constants.js +++ b/components/webui/imports/ui/constants.js @@ -1,7 +1,6 @@ /** * Dictionary for local storage items used in the application. * - * @constant * @type {Object} */ const LOCAL_STORAGE_KEYS = Object.freeze({ diff --git a/components/webui/imports/utils/logger.js b/components/webui/imports/utils/logger.js index bad28aaf1..276aca80f 100644 --- a/components/webui/imports/utils/logger.js +++ b/components/webui/imports/utils/logger.js @@ -1,6 +1,7 @@ +import JSON5 from "json5"; import winston from "winston"; import "winston-daily-rotate-file"; -import JSON5 from "json5"; + const MAX_LOGS_FILE_SIZE = "100m"; const MAX_LOGS_RETENTION_DAYS = "30d"; @@ -78,7 +79,7 @@ const fileLineFuncLog = (level, ...args) => { }); }; -export let logger = Object.freeze({ +let logger = Object.freeze({ error: (...args) => (fileLineFuncLog("error", ...args)), warn: (...args) => (fileLineFuncLog("warn", ...args)), help: (...args) => (fileLineFuncLog("help", ...args)), @@ -98,7 +99,7 @@ export let logger = Object.freeze({ * @param {string} webuiLoggingLevel messages higher than this level will be logged * @param {boolean} [_isTraceEnabled=false] whether to log function & file names and line numbers */ -export const initLogger = (logsDir, webuiLoggingLevel, _isTraceEnabled = false) => { +const initLogger = (logsDir, webuiLoggingLevel, _isTraceEnabled = false) => { isTraceEnabled = _isTraceEnabled; winstonLogger = winston.createLogger({ @@ -128,3 +129,5 @@ export const initLogger = (logsDir, webuiLoggingLevel, _isTraceEnabled = false) logger.info("logger has been initialized"); }; + +export {logger, initLogger}; diff --git a/components/webui/server/main.js b/components/webui/server/main.js index 0c2cb60d4..94d7b547a 100644 --- a/components/webui/server/main.js +++ b/components/webui/server/main.js @@ -10,6 +10,7 @@ import "/imports/api/search/server/publications"; import "/imports/api/user/server/methods"; import {deinitDbManagers, initDbManagers} from "../imports/utils/DbManager"; + const DEFAULT_LOGS_DIR = "."; const DEFAULT_LOGGING_LEVEL = Meteor.isDevelopment ? "DEBUG" : "INFO"; From dada28f59eb7cad18af80a20450a3cd7e663c390 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 9 Feb 2024 16:08:17 +0800 Subject: [PATCH 57/60] Correct webui rebuild condition in Taskfile. --- Taskfile.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Taskfile.yml b/Taskfile.yml index a2b2f7a2c..6020f290d 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -193,6 +193,12 @@ tasks: cd "{{.WEBUI_BUILD_DIR}}/programs/server" PATH="{{.NODEJS_BIN_DIR}}":$PATH $(readlink -f "{{.NODEJS_BIN_DIR}}/npm") install sources: + - "./.meteor/*" + - "./client/**/*" + - "./imports/**/*" + - "./server/**/*" + - "./tests/**/*" + - "./*" - "{{.WEBUI_BUILD_DIR}}/**/*" lint-check: From dd4f5efe3cdf477f6260debfe9966e79c424c487 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 9 Feb 2024 16:13:36 +0800 Subject: [PATCH 58/60] Minor refactoring. --- components/webui/client/main.jsx | 6 +- .../api/search/SearchJobCollectionsManager.js | 1 + .../imports/api/search/server/publications.js | 2 +- .../webui/imports/api/user/server/methods.js | 4 +- components/webui/imports/ui/App.jsx | 55 +++++++++++++------ .../imports/ui/IngestView/IngestView.jsx | 4 +- .../imports/ui/SearchView/SearchControls.jsx | 9 +-- .../imports/ui/SearchView/SearchResults.jsx | 3 +- .../ui/SearchView/SearchResultsHeader.jsx | 11 ++-- .../ui/SearchView/SearchResultsTable.jsx | 4 +- .../imports/ui/SearchView/SearchView.jsx | 6 +- .../webui/imports/ui/Sidebar/Sidebar.jsx | 15 ++--- .../webui/imports/ui/Sidebar/Sidebar.scss | 12 ++-- components/webui/imports/utils/DbManager.js | 4 +- 14 files changed, 84 insertions(+), 52 deletions(-) diff --git a/components/webui/client/main.jsx b/components/webui/client/main.jsx index 1516793fc..3e681306f 100644 --- a/components/webui/client/main.jsx +++ b/components/webui/client/main.jsx @@ -1,9 +1,11 @@ import React from "react"; + +import {createBrowserHistory} from "history"; import {Meteor} from "meteor/meteor"; import {render} from "react-dom"; -import {App} from "/imports/ui/App.jsx"; import {Router, Switch} from "react-router"; -import {createBrowserHistory} from "history"; + +import {App} from "/imports/ui/App.jsx"; Meteor.startup(() => { const routes = ( diff --git a/components/webui/imports/api/search/SearchJobCollectionsManager.js b/components/webui/imports/api/search/SearchJobCollectionsManager.js index 72184950a..b057b4c69 100644 --- a/components/webui/imports/api/search/SearchJobCollectionsManager.js +++ b/components/webui/imports/api/search/SearchJobCollectionsManager.js @@ -4,6 +4,7 @@ */ class SearchJobCollectionsManager { #collections; + constructor() { this.#collections = new Map(); } diff --git a/components/webui/imports/api/search/server/publications.js b/components/webui/imports/api/search/server/publications.js index 164a6acce..af1b164a2 100644 --- a/components/webui/imports/api/search/server/publications.js +++ b/components/webui/imports/api/search/server/publications.js @@ -1,5 +1,5 @@ -import {Meteor} from "meteor/meteor"; import {logger} from "/imports/utils/logger"; +import {Meteor} from "meteor/meteor"; import {addSortToMongoFindOptions, SearchResultsMetadataCollection} from "../collections"; import {searchJobCollectionsManager} from "./collections"; diff --git a/components/webui/imports/api/user/server/methods.js b/components/webui/imports/api/user/server/methods.js index c135dcece..e60ecdec7 100644 --- a/components/webui/imports/api/user/server/methods.js +++ b/components/webui/imports/api/user/server/methods.js @@ -1,6 +1,6 @@ -import {Meteor} from "meteor/meteor"; -import {Accounts} from "meteor/accounts-base"; import {logger} from "/imports/utils/logger"; +import {Accounts} from "meteor/accounts-base"; +import {Meteor} from "meteor/meteor"; Meteor.methods({ diff --git a/components/webui/imports/ui/App.jsx b/components/webui/imports/ui/App.jsx index 1eed6d6d2..d235f0d43 100644 --- a/components/webui/imports/ui/App.jsx +++ b/components/webui/imports/ui/App.jsx @@ -1,45 +1,60 @@ import React from "react"; -import {Redirect, Route, Switch} from "react-router"; -import {faSearch, faFileUpload} from "@fortawesome/free-solid-svg-icons"; -import IngestView from './IngestView/IngestView.jsx'; -import SearchView from "./SearchView/SearchView.jsx"; -import Sidebar from "./Sidebar/Sidebar.jsx"; +import {faFileUpload, faSearch} from "@fortawesome/free-solid-svg-icons"; +import {Redirect, Route, Switch} from "react-router"; import {login} from "../api/user/client/methods"; import {LOCAL_STORAGE_KEYS} from "./constants"; +import IngestView from "./IngestView/IngestView.jsx"; +import SearchView from "./SearchView/SearchView.jsx"; +import Sidebar from "./Sidebar/Sidebar.jsx"; + import "./App.scss"; const ROUTES = [ - {path: "/ingest", label: "Ingest", icon: faFileUpload, component: IngestView}, - {path: "/search", label: "Search", icon: faSearch, component: SearchView}, + { + path: "/ingest", + label: "Ingest", + icon: faFileUpload, + component: IngestView, + }, + { + path: "/search", + label: "Search", + icon: faSearch, + component: SearchView, + }, ]; const App = () => { const [loggedIn, setLoggedIn] = React.useState(false); const [isSidebarCollapsed, setSidebarStateCollapsed] = React.useState( - "true" === localStorage.getItem(LOCAL_STORAGE_KEYS.IS_SIDEBAR_COLLAPSED) + "true" === localStorage.getItem(LOCAL_STORAGE_KEYS.IS_SIDEBAR_COLLAPSED), ); React.useEffect(async () => { const result = await login(); - setLoggedIn(result) + setLoggedIn(result); }, []); React.useEffect(() => { - localStorage.setItem(LOCAL_STORAGE_KEYS.IS_SIDEBAR_COLLAPSED, isSidebarCollapsed.toString()); + localStorage.setItem(LOCAL_STORAGE_KEYS.IS_SIDEBAR_COLLAPSED, + isSidebarCollapsed.toString()); }, [isSidebarCollapsed]); const handleSidebarToggle = () => { setSidebarStateCollapsed(!isSidebarCollapsed); - } + }; const Spinner = () =>
+ style={{ + width: "3rem", + height: "3rem", + }} role="status"> Loading...
@@ -49,26 +64,32 @@ const App = () => { - + - + ; - return (
+ return (
-
+
{!loggedIn ? : }
); -} +}; export {App}; \ No newline at end of file diff --git a/components/webui/imports/ui/IngestView/IngestView.jsx b/components/webui/imports/ui/IngestView/IngestView.jsx index 4f2e75607..0d630b107 100644 --- a/components/webui/imports/ui/IngestView/IngestView.jsx +++ b/components/webui/imports/ui/IngestView/IngestView.jsx @@ -1,9 +1,9 @@ -import {faChartBar, faClock, faEnvelope, faFileAlt, faHdd} from "@fortawesome/free-solid-svg-icons"; +import React from "react"; +import {faChartBar, faClock, faEnvelope, faFileAlt, faHdd} from "@fortawesome/free-solid-svg-icons"; import {FontAwesomeIcon} from "@fortawesome/react-fontawesome"; import {DateTime} from "luxon"; import {useTracker} from "meteor/react-meteor-data"; -import React from "react"; import {Col, Container, ProgressBar, Row} from "react-bootstrap"; import {StatsCollection} from "../../api/ingestion/collections"; diff --git a/components/webui/imports/ui/SearchView/SearchControls.jsx b/components/webui/imports/ui/SearchView/SearchControls.jsx index 83455c62f..2debf1f09 100644 --- a/components/webui/imports/ui/SearchView/SearchControls.jsx +++ b/components/webui/imports/ui/SearchView/SearchControls.jsx @@ -1,5 +1,7 @@ -import React, {useEffect, useState, useRef} from "react"; +import React, {useEffect, useRef, useState} from "react"; +import {faBars, faSearch, faTimes, faTrash} from "@fortawesome/free-solid-svg-icons"; +import {FontAwesomeIcon} from "@fortawesome/react-fontawesome"; import { Button, Col, @@ -11,13 +13,12 @@ import { Row, } from "react-bootstrap"; import DatePicker from "react-datepicker"; -import {FontAwesomeIcon} from "@fortawesome/react-fontawesome"; -import {faBars, faSearch, faTimes, faTrash} from "@fortawesome/free-solid-svg-icons"; +import {isSearchSignalQuerying, isSearchSignalReq, SearchSignal} from "../../api/search/constants"; import {computeTimeRange, TIME_RANGE_PRESET_LABEL} from "./datetime"; import {LOCAL_STORAGE_KEYS} from "../constants"; -import {isSearchSignalQuerying, isSearchSignalReq, SearchSignal} from "../../api/search/constants"; +import "react-datepicker/dist/react-datepicker.css"; import "./SearchControls.scss"; diff --git a/components/webui/imports/ui/SearchView/SearchResults.jsx b/components/webui/imports/ui/SearchView/SearchResults.jsx index 7e4edef1f..e850ab607 100644 --- a/components/webui/imports/ui/SearchView/SearchResults.jsx +++ b/components/webui/imports/ui/SearchView/SearchResults.jsx @@ -1,5 +1,6 @@ import React from "react"; -import {SearchResultsHeader} from "./SearchResultsHeader.jsx"; + +import SearchResultsHeader from "./SearchResultsHeader.jsx"; import {SearchResultsTable} from "./SearchResultsTable.jsx"; diff --git a/components/webui/imports/ui/SearchView/SearchResultsHeader.jsx b/components/webui/imports/ui/SearchView/SearchResultsHeader.jsx index eac9cbe7d..bce6765e6 100644 --- a/components/webui/imports/ui/SearchView/SearchResultsHeader.jsx +++ b/components/webui/imports/ui/SearchView/SearchResultsHeader.jsx @@ -1,6 +1,7 @@ +import React from "react"; + import {faCog} from "@fortawesome/free-solid-svg-icons"; import {FontAwesomeIcon} from "@fortawesome/react-fontawesome"; -import React from "react"; import { Button, Col, @@ -27,7 +28,7 @@ import "./SearchResultsHeader.scss"; * @param {function} setMaxLinesPerResult callback to set setMaxLinesPerResult * @returns {JSX.Element} */ -export const SearchResultsHeader = ({ +const SearchResultsHeader = ({ jobId, resultsMetadata, numResultsOnServer, @@ -80,8 +81,8 @@ export const SearchResultsHeader = ({ }> {(0 < numResultsOnServer) ? : <>} @@ -90,3 +91,5 @@ export const SearchResultsHeader = ({ ); }; + +export default SearchResultsHeader; diff --git a/components/webui/imports/ui/SearchView/SearchResultsTable.jsx b/components/webui/imports/ui/SearchView/SearchResultsTable.jsx index 0ac92868b..4803c4e31 100644 --- a/components/webui/imports/ui/SearchView/SearchResultsTable.jsx +++ b/components/webui/imports/ui/SearchView/SearchResultsTable.jsx @@ -1,8 +1,10 @@ +import React, {useEffect, useState} from "react"; + import {faSort, faSortDown, faSortUp} from "@fortawesome/free-solid-svg-icons"; import {FontAwesomeIcon} from "@fortawesome/react-fontawesome"; -import React, {useEffect, useState} from "react"; import {Spinner, Table} from "react-bootstrap"; import ReactVisibilitySensor from "react-visibility-sensor"; + import "./SearchResultsTable.scss"; diff --git a/components/webui/imports/ui/SearchView/SearchView.jsx b/components/webui/imports/ui/SearchView/SearchView.jsx index 85e177955..81c90c4b9 100644 --- a/components/webui/imports/ui/SearchView/SearchView.jsx +++ b/components/webui/imports/ui/SearchView/SearchView.jsx @@ -1,8 +1,9 @@ +import React, {useEffect, useRef, useState} from "react"; + import {faExclamationCircle} from "@fortawesome/free-solid-svg-icons"; import {FontAwesomeIcon} from "@fortawesome/react-fontawesome"; import {Meteor} from "meteor/meteor"; import {useTracker} from "meteor/react-meteor-data"; -import React, {useEffect, useRef, useState} from "react"; import {ProgressBar} from "react-bootstrap"; import { @@ -11,10 +12,9 @@ import { } from "../../api/search/collections"; import {INVALID_JOB_ID, isSearchSignalQuerying, SearchSignal} from "../../api/search/constants"; import SearchJobCollectionsManager from "../../api/search/SearchJobCollectionsManager"; - -import "react-datepicker/dist/react-datepicker.css"; import {LOCAL_STORAGE_KEYS} from "../constants"; import {changeTimezoneToUtcWithoutChangingTime, DEFAULT_TIME_RANGE} from "./datetime"; + import SearchControls from "./SearchControls.jsx"; import SearchResults from "./SearchResults.jsx"; import {VISIBLE_RESULTS_LIMIT_INITIAL} from "./SearchResultsTable.jsx"; diff --git a/components/webui/imports/ui/Sidebar/Sidebar.jsx b/components/webui/imports/ui/Sidebar/Sidebar.jsx index 67e55337f..f92b19615 100644 --- a/components/webui/imports/ui/Sidebar/Sidebar.jsx +++ b/components/webui/imports/ui/Sidebar/Sidebar.jsx @@ -1,11 +1,8 @@ import React from "react"; -import {NavLink} from "react-router-dom"; +import {faAngleDoubleLeft, faAngleDoubleRight} from "@fortawesome/free-solid-svg-icons"; import {FontAwesomeIcon} from "@fortawesome/react-fontawesome"; -import { - faAngleDoubleLeft, - faAngleDoubleRight, -} from "@fortawesome/free-solid-svg-icons"; +import {NavLink} from "react-router-dom"; /** @@ -17,7 +14,11 @@ import { * @param {function} onSidebarToggle callback to toggle the sidebar's collapsed state * @returns {JSX.Element} */ -const Sidebar = ({isSidebarCollapsed, routes, onSidebarToggle}) => { +const Sidebar = ({ + isSidebarCollapsed, + routes, + onSidebarToggle, +}) => { return ( {route["label"]} - ) + ), )}
diff --git a/components/webui/imports/ui/Sidebar/Sidebar.scss b/components/webui/imports/ui/Sidebar/Sidebar.scss index 1eb8b2068..c41fa4f0b 100644 --- a/components/webui/imports/ui/Sidebar/Sidebar.scss +++ b/components/webui/imports/ui/Sidebar/Sidebar.scss @@ -18,20 +18,20 @@ } #sidebar .brand { - align-items: center; /* center element vertically */ + align-items: center; /* center element vertically */ display: flex; - flex: 0 0 3rem; /* cannot grow, cannot shrink, 44px height */ - justify-content: center; /* center element horizontally */ + flex: 0 0 3rem; /* cannot grow, cannot shrink, 44px height */ + justify-content: center; /* center element horizontally */ border-left: 1px solid #111; font-size: 1.25rem; - white-space: nowrap; /* proper animation */ - overflow: hidden; /* proper animation */ + white-space: nowrap; /* proper animation */ + overflow: hidden; /* proper animation */ } .sidebar-menu { - flex: 1 1 auto; /* can shrink, can grow, auto height */ + flex: 1 1 auto; /* can shrink, can grow, auto height */ display: flex; flex-direction: column; overflow-y: auto; diff --git a/components/webui/imports/utils/DbManager.js b/components/webui/imports/utils/DbManager.js index 2f316e59e..b5a38f894 100644 --- a/components/webui/imports/utils/DbManager.js +++ b/components/webui/imports/utils/DbManager.js @@ -36,7 +36,7 @@ const initDbManagers = async ({ }, { searchJobsTableName, clpArchivesTableName, - clpFilesTableName + clpFilesTableName, }) => { if (null !== dbConnection) { logger.error("This method should not be called twice."); @@ -56,7 +56,7 @@ const initDbManagers = async ({ await dbConnection.connect(); initSearchJobsDbManager(dbConnection, { - searchJobsTableName + searchJobsTableName, }); initStatsDbManager(dbConnection, { clpArchivesTableName, From d9897cbc48f96fd631500337cd78c8d2ea4edf99 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 9 Feb 2024 16:17:17 +0800 Subject: [PATCH 59/60] Minor refactoring. --- components/webui/imports/ui/App.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/webui/imports/ui/App.jsx b/components/webui/imports/ui/App.jsx index d235f0d43..1e812c123 100644 --- a/components/webui/imports/ui/App.jsx +++ b/components/webui/imports/ui/App.jsx @@ -92,4 +92,4 @@ const App = () => {
); }; -export {App}; \ No newline at end of file +export {App}; From 90e1480f369985e4b582284356841661fd666246 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 9 Feb 2024 16:34:03 +0800 Subject: [PATCH 60/60] Minor refactoring. --- components/webui/imports/api/ingestion/server/publications.js | 4 +++- components/webui/imports/utils/DbManager.js | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/components/webui/imports/api/ingestion/server/publications.js b/components/webui/imports/api/ingestion/server/publications.js index 6a2279285..63d0405b5 100644 --- a/components/webui/imports/api/ingestion/server/publications.js +++ b/components/webui/imports/api/ingestion/server/publications.js @@ -1,7 +1,7 @@ +import {logger} from "/imports/utils/logger"; import {Meteor} from "meteor/meteor"; import {STATS_COLLECTION_ID_COMPRESSION, StatsCollection} from "../collections"; - import StatsDbManager from "./StatsDbManager"; @@ -78,6 +78,8 @@ const deinitStatsDbManager = () => { * @returns {Mongo.Cursor} */ Meteor.publish(Meteor.settings.public.StatsCollectionName, async () => { + logger.debug(`Subscription '${Meteor.settings.public.SearchResultsCollectionName}'`); + await refreshCompressionStats(); const filter = { diff --git a/components/webui/imports/utils/DbManager.js b/components/webui/imports/utils/DbManager.js index b5a38f894..a460c2f59 100644 --- a/components/webui/imports/utils/DbManager.js +++ b/components/webui/imports/utils/DbManager.js @@ -1,7 +1,8 @@ +import {logger} from "/imports/utils/logger"; import mysql from "mysql2/promise"; + import {deinitStatsDbManager, initStatsDbManager} from "../api/ingestion/server/publications"; import {initSearchJobsDbManager} from "../api/search/server/methods"; -import {logger} from "./logger"; /**