diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..06b040a --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +*.pyc +*.swp +*.eggs +.idea/ +.tox/ +build/ +dist/ +ambari_presto.egg-info/ +docs/_build +.pytest_cache/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..0d04ca8 --- /dev/null +++ b/README.md @@ -0,0 +1,23 @@ +# ambari-server-presto +Ambari custom server for [Presto](https://prestosql.io/) + +# Support Presto Tarball Installation +Tested on +* Starburst Distribution of Presto version 302-e.9 + + +# Installation +For `ambari-2.7.3`, `HDP-3.1`, `ambari-server-presto-0.1.tar.gz` +``` +wget ambari-server-presto-0.1.tar.gz + +sudo mkdir /var/lib/ambari-server/resources/stacks/HDP/3.1/services/PRESTO/ + +sudo tar -xvf ambari-server-presto-${version}.tar.gz +/var/lib/ambari-server/resources/stacks/HDP/3.1/services/PRESTO/ + +# please setup 'package/config.ini' + +# restart ambari +sudo ambari-server restart +``` diff --git a/configuration/config.properties.xml b/configuration/config.properties.xml new file mode 100644 index 0000000..5857a61 --- /dev/null +++ b/configuration/config.properties.xml @@ -0,0 +1,156 @@ + + + + + + + node-scheduler.include-coordinator + false + + True: coordinator is a worker. + False: coordinator is not a worker. + + + value-list + + + true + + + + false + + + + 1 + + + + + http-server.http.port + 18080 + + + + + + discovery.uri + http://coordinator:18080 + + Set 'port' to 'http-server.http.port' + + + + + memory.heap-headroom-per-node + 256 + + JVM for presto itself not for query. + + + int + 0 + 2048 + 128 + MB + + + + + query.max-memory + 512 + + + + int + 0 + 15360 + 512 + MB + + + + + query.max-total-memory + 512 + + Must equal or larger than 'query.max-memory'. + If no clue, equal to 'query.max-memory'. + + + int + 0 + 15360 + 512 + MB + + + + + query.max-memory-per-node + 512 + + + + int + 0 + 5120 + 512 + MB + + + + + query.max-total-memory-per-node + 512 + + Must equal or larger than 'query.max-memory-per-node'. + If no clue, equal to 'query.max-memory-per-node'. + + + int + 0 + 5120 + 512 + MB + + + + + experimental.reserved-pool-enabled + false + + False: disable reserved-pool. + If no clue, set to false. + + + value-list + + + true + + + + false + + + + 1 + + + + diff --git a/configuration/connectors.properties.xml b/configuration/connectors.properties.xml new file mode 100644 index 0000000..5c27607 --- /dev/null +++ b/configuration/connectors.properties.xml @@ -0,0 +1,44 @@ + + + + + + connectors.to.add + [] + + Modify this property to add connectors. The format should be json like + [{"catalog":"hive","config":["k1=v1","k2=v2"]}, {"catalog":"mysql","config":["k1=v1","k2=v2","k3=v3"]}]. + Note the single quotes around each value! This example will + create files connector1.properties, connector2.properties for Presto with entries + key1=value1 etc. If you don't want to add any connectors then leave the value as []; + Ambari will not accept an empty value for this property. For deleting connectors, please + use the connectors.to.remove property. + + + + connectors.to.delete + [] + + Modify this property to delete connectors. The format should be json like + ["hive", "mysql"]. Note the single quotes around each value! + This example will delete the files connector1.properties, connector2.properties + and connector3.properties in the Presto connector directory. If you don't want to delete + any connectors then leave the value as []; Ambari will not accept an empty value for + this property. + + + \ No newline at end of file diff --git a/configuration/jvm.config.xml b/configuration/jvm.config.xml new file mode 100644 index 0000000..ce7f643 --- /dev/null +++ b/configuration/jvm.config.xml @@ -0,0 +1,41 @@ + + + + + + jvm.config + -server +-Xmx1G +-XX:-UseBiasedLocking +-XX:+UseG1GC +-XX:G1HeapRegionSize=32M +-XX:+ExplicitGCInvokesConcurrent +-XX:+HeapDumpOnOutOfMemoryError +-XX:+UseGCOverheadLimit +-XX:+ExitOnOutOfMemoryError +-XX:ReservedCodeCacheSize=512M +-Djdk.nio.maxCachedBufferSize=2000000 +-DHADOOP_USER_NAME=hive + + A list of command line options used for launching the Java Virtual + Machine. The format of the file must be one option per line. These + options are not interpreted by the shell, so options containing spaces or + other special characters should not be quoted (as demonstrated by the + OnOutOfMemoryError option). + + + diff --git a/configuration/node.properties.xml b/configuration/node.properties.xml new file mode 100644 index 0000000..efdaebc --- /dev/null +++ b/configuration/node.properties.xml @@ -0,0 +1,54 @@ + + + + + + node.environment + test + + The name of the environment. All Presto nodes in a cluster must have the + same environment name. + + + + + node.data-dir + /var/lib/presto + + The location (filesystem path) of the data directory. Presto will store + logs and other data here. + + + + + catalog.config-dir + /etc/presto/catalog + + Configuration directory for plugins. This is where you should place + connector property files. + + + + + plugin.dir + /usr/lib/presto/plugin + + Library directory for plugins. + + + + diff --git a/metainfo.xml b/metainfo.xml new file mode 100644 index 0000000..fe4d3d1 --- /dev/null +++ b/metainfo.xml @@ -0,0 +1,84 @@ + + + + 2.0 + + + PRESTO + Presto + Presto is an open source distributed SQL query engine for running + interactive analytic queries against data sources of all sizes ranging + from gigabytes to petabytes. + + 308+ + + + PRESTO_COORDINATOR + Presto Coordinator + MASTER + 1 + true + + + PYTHON + 1200 + + + + + PRESTO_WORKER + Presto Worker + SLAVE + 0+ + true + + + PYTHON + + + + + PRESTO_CLI + Presto CLI + CLIENT + 1+ + true + + + PYTHON + + + + + + node.properties + config.properties + jvm.config + connectors.properties + + + + + theme.json + true + + + + + + diff --git a/package/config.ini b/package/config.ini new file mode 100644 index 0000000..1854f42 --- /dev/null +++ b/package/config.ini @@ -0,0 +1,7 @@ +[download] +presto_tar_file = /root/presto/presto-server-*.tar.gz +presto_cli_file = /root/presto/presto-cli-*.jar + + +[other] +java_home = /opt/java/current diff --git a/package/scripts/common.py b/package/scripts/common.py new file mode 100644 index 0000000..6f86a38 --- /dev/null +++ b/package/scripts/common.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +import os +try: + import ConfigParser as cp +except ImportError: + import configparser as cp +from common_func import exec_command, kv_print + +from resource_management.libraries.script.script import Script + +# Read config +package_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +config = cp.ConfigParser() +config.read(os.path.join(package_dir, 'config.ini')) + +# JAVA_HOME +java_home = config.get('other', 'java_home') +kv_print('java_home', java_home) + +# Tarball file +presto_tar_file_0 = config.get('download', 'presto_tar_file') +presto_tar_file = exec_command('ls {0}'.format(presto_tar_file_0)) +# presto_tar_name = presto_tar_file.split('/')[-1] + +presto_cli_file_0 = config.get('download', 'presto_cli_file') +presto_cli_file = exec_command('ls {0}'.format(presto_cli_file_0)) +kv_print('presto_tar_file', presto_tar_file) +kv_print('presto_cli_file', presto_cli_file) + + +# Config object that holds the configurations declared in the config xml file +script_config = Script.get_config() + +host_info = script_config['clusterHostInfo'] +host_level_params = script_config['hostLevelParams'] +kv_print('host_info', host_info) +kv_print('host_level_params', host_level_params) + + +# Config files +node_properties = script_config['configurations']['node.properties'] +jvm_config = script_config['configurations']['jvm.config'] +config_properties = script_config['configurations']['config.properties'] +connectors_properties = script_config['configurations']['connectors.properties'] + +connectors_to_add = connectors_properties['connectors.to.add'] +connectors_to_delete = connectors_properties['connectors.to.delete'] + +# Some constants +presto_install_dir = '/usr/lib/presto' +presto_bin_dir = presto_install_dir + '/bin' +presto_etc_dir = '/etc/presto' +presto_etc_link = presto_install_dir + '/etc' +presto_lib_dir = presto_install_dir + '/lib' +presto_plugin_dir = node_properties['plugin.dir'] +presto_catalog_dir = node_properties['catalog.config-dir'] + +need_init_dirs = [ + presto_install_dir, presto_etc_dir, presto_plugin_dir, presto_catalog_dir +] + +presto_launcher_script = presto_bin_dir + '/launcher' + +memory_configs = [ + 'memory.heap-headroom-per-node', + 'query.max-memory', + 'query.max-total-memory', + 'query.max-memory-per-node', + 'query.max-total-memory-per-node' +] diff --git a/package/scripts/common_func.py b/package/scripts/common_func.py new file mode 100644 index 0000000..da6486f --- /dev/null +++ b/package/scripts/common_func.py @@ -0,0 +1,54 @@ +import subprocess +import json +import os + + +def kv_print(k, v): + print('---- [kv_print] {0}={1}'.format(k, v)) + return + + +def exec_command(args): + """ + :param args: str or list + :return: stdout + """ + p = subprocess.Popen(args, stdout=subprocess.PIPE, shell=True) + print('---- exec command: {0}'.format(args)) + (o, _) = p.communicate() + stdout = o.decode('utf-8').strip() + return stdout + + +def init_dirs(): + from common import need_init_dirs, presto_etc_dir, presto_etc_link + mkdir_pattern = 'mkdir -p {0}' + for d in need_init_dirs: + exec_command(mkdir_pattern.format(d)) + exec_command('ln -s {0} {1}'.format(presto_etc_dir, presto_etc_link)) + return + + +def create_connectors(): + from common import presto_catalog_dir, connectors_to_add + + json_root = json.loads(connectors_to_add) + for connector in json_root: + connector_name = connector['catalog'] + connector_file = os.path.join(presto_catalog_dir, connector_name+'.properties') + print('---- add connector_file: {0}'.format(connector_file)) + with open(connector_file, 'w') as f: + for l in connector['config']: + f.write('{0}\n'.format(l)) + return + + +def delete_connectors(): + from common import presto_catalog_dir, connectors_to_delete + + json_root = json.loads(connectors_to_delete) + for connector_name in json_root: + connector_file = os.path.join(presto_catalog_dir, connector_name+'.properties') + print('---- remove connector_file: {0}'.format(connector_file)) + exec_command('rm -f {0}'.format(connector_file)) + return diff --git a/package/scripts/presto_cli.py b/package/scripts/presto_cli.py new file mode 100644 index 0000000..f42a644 --- /dev/null +++ b/package/scripts/presto_cli.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +from resource_management.libraries.script.script import Script +from resource_management.core.exceptions import ClientComponentHasNoStatus + + +class Cli(Script): + def install(self, env): + from common import presto_cli_file + from common_func import exec_command + + try: + exec_command('cp -r {1} {0}/presto-cli'.format('/usr/bin', presto_cli_file)) + exec_command('chmod +x {0}/presto-cli'.format('/usr/bin')) + except BaseException: + pass + + def status(self, env): + raise ClientComponentHasNoStatus() + + +if __name__ == '__main__': + Cli().execute() diff --git a/package/scripts/presto_client.py b/package/scripts/presto_client.py new file mode 100644 index 0000000..9bc54f8 --- /dev/null +++ b/package/scripts/presto_client.py @@ -0,0 +1,269 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Simple client to communicate with a Presto server. +""" +from httplib import HTTPConnection, HTTPException +import logging +import json +import socket +import sys +import time + +from urllib2 import HTTPError, urlopen, URLError + + +URL_TIMEOUT_MS = 5000 +NUM_ROWS = 1000 +DATA_RESP = 'data' +NEXT_URI_RESP = 'nextUri' +RETRY_TIMEOUT = 120 +SYSTEM_RUNTIME_NODES = 'select * from system.runtime.nodes' +SHOW_CATALOGS = 'show catalogs' +SLEEP_INTERVAL = 10 + +logging.basicConfig(stream=sys.stdout) +logger = logging.getLogger(__name__) + + +def smoketest_presto(client, all_hosts): + ensure_nodes_are_up(client, all_hosts) + ensure_catalogs_are_available(client) +# client.execute_query('select * from nation', schema='sf1', catalog='tpch') +# rows = client.get_rows() +# if len(rows) != 25: +# raise RuntimeError('Presto server failed to return the correct \ +# number of rows from nation table in TPCH connector. Expected 25 but got {0}'.format(len(rows))) + + +def ensure_catalogs_are_available(client): + rows = [] + elapsed_time = 0 + while elapsed_time < RETRY_TIMEOUT: + client.execute_query(SHOW_CATALOGS) + rows = client.get_rows() + if not rows: + time.sleep(SLEEP_INTERVAL) + logger.debug('Failed to load catalogs after ' + 'waiting for %d seconds. Retrying...' % elapsed_time) + elapsed_time += SLEEP_INTERVAL + else: + break + if not rows: + raise RuntimeError('Presto server failed to load all catalogs within \ +{0} seconds.'.format(RETRY_TIMEOUT)) + + +def ensure_nodes_are_up(client, all_hosts): + result = True + elapsed_time = 0 + while elapsed_time < RETRY_TIMEOUT: + result = client.execute_query(SYSTEM_RUNTIME_NODES) + if not result: + time.sleep(SLEEP_INTERVAL) + logger.debug('Status retrieval for the server failed after ' + 'waiting for %d seconds. Retrying...' % elapsed_time) + elapsed_time += SLEEP_INTERVAL + else: + break + if not result: + raise RuntimeError('Presto server failed to start within {0} seconds.'.format(RETRY_TIMEOUT)) + + # Verify that the nodes we expect to have registered with the Discovery + # service have actually registered correctly + elapsed_time = 0 + are_expected_nodes_up = False + while elapsed_time < RETRY_TIMEOUT: + client.execute_query(SYSTEM_RUNTIME_NODES) + nodes_returned_from_presto = [] + for row in client.get_rows(): + nodes_returned_from_presto.append(row[0]) + if len(nodes_returned_from_presto) == len(all_hosts): + are_expected_nodes_up = True + break + else: + time.sleep(SLEEP_INTERVAL) + logger.debug('Elapsed time {0}'.format(elapsed_time)) + logger.debug( + 'Number of hosts returned from Presto {0} do not match number of hosts specified by user {1}'.format( + nodes_returned_from_presto, all_hosts)) + elapsed_time += SLEEP_INTERVAL + if not are_expected_nodes_up: + raise RuntimeError( + 'Number of hosts returned from Presto {0} do not equal the number of hosts specified by user {1}'.format( + nodes_returned_from_presto, all_hosts)) + + +# This class was copied more or less verbatim from +# https://github.com/prestodb/presto-admin/blob/master/prestoadmin/prestoclient.py +class PrestoClient: + response_from_server = {} + # rows returned by the query + rows = [] + next_uri = '' + + def __init__(self, server, user, port=None): + self.server = server + self.user = user + self.port = port if port else None + + def clear_old_results(self): + if self.rows: + self.rows = [] + + if self.next_uri: + self.next_uri = '' + + if self.response_from_server: + self.response_from_server = {} + + def execute_query(self, sql, schema='sf1', catalog='tpch'): + """ + Execute a query connecting to Presto server using passed parameters. + + Client sends http POST request to the Presto server, page: + '/v1/statement'. Header information should + include: X-Presto-Catalog, X-Presto-Schema, X-Presto-User + + Args: + sql: SQL query to be executed + schema: Presto schema to be used while executing query + (default=default) + catalog: Catalog to be used by the server + + Returns: + True or False exit status + """ + if not sql: + raise InvalidArgumentError('SQL query missing') + + if not self.server: + raise InvalidArgumentError('Server IP missing') + + if not self.user: + raise InvalidArgumentError('Username missing') + + if not self.port: + raise InvalidArgumentError('Port missing') + + self.clear_old_results() + + headers = {'X-Presto-Catalog': catalog, + 'X-Presto-Schema': schema, + 'X-Presto-User': self.user} + answer = '' + try: + logger.info('Connecting to server at: ' + self.server + + ':' + str(self.port) + ' as user ' + self.user) + conn = HTTPConnection(self.server, int(self.port), False, + URL_TIMEOUT_MS) + conn.request('POST', '/v1/statement', sql, headers) + response = conn.getresponse() + + if response.status != 200: + conn.close() + logger.error('Connection error: ' + + str(response.status) + ' ' + response.reason) + return False + + answer = response.read() + conn.close() + + self.response_from_server = json.loads(answer) + logger.info('Query executed successfully') + return True + except (HTTPException, socket.error): + logger.error('Error connecting to presto server at: ' + + self.server + ':' + str(self.port)) + return False + except ValueError as e: + logger.error('Error connecting to Presto server: ' + str(e) + + ' error from server: ' + answer) + raise e + + def get_response_from(self, uri): + """ + Sends a GET request to the Presto server at the specified next_uri + and updates the response + """ + try: + conn = urlopen(uri, None, URL_TIMEOUT_MS) + answer = conn.read() + conn.close() + + self.response_from_server = json.loads(answer) + logger.info('GET request successful for uri: ' + uri) + return True + except (HTTPError, URLError) as e: + logger.error('Error opening the presto response uri: ' + + str(e.reason)) + return False + + def build_results_from_response(self): + """ + Build result from the response + + The reponse_from_server may contain up to 3 uri's. + 1. link to fetch the next packet of data ('nextUri') + 2. TODO: information about the query execution ('infoUri') + 3. TODO: cancel the query ('partialCancelUri'). + """ + if NEXT_URI_RESP in self.response_from_server: + self.next_uri = self.response_from_server[NEXT_URI_RESP] + else: + self.next_uri = '' + + if DATA_RESP in self.response_from_server: + if self.rows: + self.rows.extend(self.response_from_server[DATA_RESP]) + else: + self.rows = self.response_from_server[DATA_RESP] + + def get_rows(self, num_of_rows=NUM_ROWS): + """ + Get the rows returned from the query. + + The client sends GET requests to the server using the 'nextUri' + from the previous response until the servers response does not + contain anymore 'nextUri's. When there is no 'nextUri' the query is + finished + + Note that this can only be called once and does not page through + the results. + + Parameters: + num_of_rows: to be retrieved. 1000 by default + """ + if num_of_rows == 0: + return [] + + self.build_results_from_response() + + if not self.get_next_uri(): + return [] + + while self.get_next_uri(): + if not self.get_response_from(self.get_next_uri()): + return [] + if len(self.rows) <= num_of_rows: + self.build_results_from_response() + return self.rows + + def get_next_uri(self): + return self.next_uri + + +class InvalidArgumentError(ValueError): + pass diff --git a/package/scripts/presto_coordinator.py b/package/scripts/presto_coordinator.py new file mode 100644 index 0000000..6cfa734 --- /dev/null +++ b/package/scripts/presto_coordinator.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +from resource_management.libraries.script.script import Script +from resource_management.core.exceptions import ComponentIsNotRunning + +from common_func import exec_command + + +class Coordinator(Script): + def install(self, env): + from common import presto_install_dir, presto_plugin_dir, presto_launcher_script, \ + java_home, presto_tar_file + from common_func import init_dirs + + # 0: presto_tar_file; 1: presto_install_dir; 2: bin or lib or plugin; 3: strip-components + tar_pattern = 'tar -xzvf {0} -C {1} --strip-components={3} `tar -tf {0} | head -n 1`{2}' + # cleanup + exec_command('rm -rf {0}'.format(presto_install_dir)) + # init dirs + init_dirs() + # bin, java_home + exec_command(tar_pattern.format(presto_tar_file, presto_install_dir, 'bin', 1)) + exec_command('sed -i "2iexport JAVA_HOME={0}" {1}'.format(java_home, presto_launcher_script)) + exec_command('sed -i "3iexport PATH=\$JAVA_HOME/bin:\$PATH" {0}'.format(presto_launcher_script)) + # lib + exec_command(tar_pattern.format(presto_tar_file, presto_install_dir, 'lib', 1)) + # plugin + exec_command(tar_pattern.format(presto_tar_file, presto_plugin_dir, 'plugin', 2)) + # config + self.configure(env) + + def stop(self, env): + from common import presto_launcher_script + + exec_command('{0} stop'.format(presto_launcher_script)) + + def start(self, env): + from common import presto_launcher_script, config_properties, host_info + from presto_client import PrestoClient, smoketest_presto + + self.configure(env) + exec_command('{0} start'.format(presto_launcher_script)) + # test + if 'presto_worker_hosts' in host_info.keys(): + all_hosts = host_info['presto_worker_hosts'] + \ + host_info['presto_coordinator_hosts'] + else: + all_hosts = host_info['presto_coordinator_hosts'] + # use Set, coordinator could be worker + all_hosts = set(all_hosts) + smoketest_presto(PrestoClient('localhost', 'root', config_properties['http-server.http.port']), all_hosts) + + def status(self, env): + from common import presto_launcher_script + + stdout = exec_command('{0} status'.format(presto_launcher_script)) + if 'Not running' in stdout: + raise ComponentIsNotRunning(stdout) + + def configure(self, env): + import os + from common import node_properties, jvm_config, config_properties, \ + presto_etc_dir, memory_configs + from common_func import create_connectors, delete_connectors + + key_val_template = '{0}={1}\n' + hostname = exec_command('hostname') + + with open(os.path.join(presto_etc_dir, 'node.properties'), 'w') as f: + for key, value in node_properties.iteritems(): + f.write(key_val_template.format(key, value)) + f.write(key_val_template.format('node.id', hostname)) + + with open(os.path.join(presto_etc_dir, 'jvm.config'), 'w') as f: + f.write(jvm_config['jvm.config']) + + with open(os.path.join(presto_etc_dir, 'config.properties'), 'w') as f: + for key, value in config_properties.iteritems(): + if key in memory_configs: + value += 'MB' + f.write(key_val_template.format(key, value)) + f.write(key_val_template.format('coordinator', 'true')) + f.write(key_val_template.format('discovery-server.enabled', 'true')) + + create_connectors() + delete_connectors() + # This is a separate call because we always want the tpch connector to + # be available because it is used to smoketest the installation. + # create_connectors(node_properties, "{'tpch': ['connector.name=tpch']}") + + +if __name__ == '__main__': + Coordinator().execute() diff --git a/package/scripts/presto_worker.py b/package/scripts/presto_worker.py new file mode 100644 index 0000000..656d0e3 --- /dev/null +++ b/package/scripts/presto_worker.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +from resource_management.libraries.script.script import Script +from resource_management.core.exceptions import ComponentIsNotRunning + +from common_func import exec_command + + +class Worker(Script): + def install(self, env): + from common import presto_install_dir, presto_plugin_dir, presto_launcher_script, \ + java_home, presto_tar_file + from common_func import init_dirs + + # 0: presto_tar_file; 1: presto_install_dir; 2: bin or lib or plugin; 3: strip-components + tar_pattern = 'tar -xzvf {0} -C {1} --strip-components={3} `tar -tf {0} | head -n 1`{2}' + # cleanup + exec_command('rm -rf {0}'.format(presto_install_dir)) + # init dirs + init_dirs() + # bin, java_home + exec_command(tar_pattern.format(presto_tar_file, presto_install_dir, 'bin', 1)) + exec_command( + 'sed -i "2iexport JAVA_HOME={0}" {1}'.format(java_home, presto_launcher_script)) + exec_command( + 'sed -i "3iexport PATH=\$JAVA_HOME/bin:\$PATH" {0}'.format(presto_launcher_script)) + # lib + exec_command(tar_pattern.format(presto_tar_file, presto_install_dir, 'lib', 1)) + # plugin + exec_command(tar_pattern.format(presto_tar_file, presto_plugin_dir, 'plugin', 2)) + # config + self.configure(env) + + def stop(self, env): + from common import presto_launcher_script + + exec_command('{0} stop'.format(presto_launcher_script)) + + def start(self, env): + from common import presto_launcher_script + + self.configure(self) + exec_command('{0} start'.format(presto_launcher_script)) + + def status(self, env): + from common import presto_launcher_script + + stdout = exec_command('{0} status'.format(presto_launcher_script)) + if 'Not running' in stdout: + raise ComponentIsNotRunning(stdout) + + def configure(self, env): + import os + from common import node_properties, jvm_config, config_properties, \ + presto_etc_dir, memory_configs + from common_func import create_connectors, delete_connectors + + key_val_template = '{0}={1}\n' + hostname = exec_command('hostname') + + with open(os.path.join(presto_etc_dir, 'node.properties'), 'w') as f: + for key, value in node_properties.iteritems(): + f.write(key_val_template.format(key, value)) + f.write(key_val_template.format('node.id', hostname)) + + with open(os.path.join(presto_etc_dir, 'jvm.config'), 'w') as f: + f.write(jvm_config['jvm.config']) + + with open(os.path.join(presto_etc_dir, 'config.properties'), 'w') as f: + for key, value in config_properties.iteritems(): + if key == 'node-scheduler.include-coordinator': + continue + if key in memory_configs: + value += 'MB' + f.write(key_val_template.format(key, value)) + f.write(key_val_template.format('coordinator', 'false')) + + create_connectors() + delete_connectors() + # This is a separate call because we always want the tpch connector to + # be available because it is used to smoketest the installation. + # create_connectors(node_properties, "{'tpch': ['connector.name=tpch']}") + + +if __name__ == '__main__': + Worker().execute() diff --git a/themes/theme.json b/themes/theme.json new file mode 100644 index 0000000..d56793c --- /dev/null +++ b/themes/theme.json @@ -0,0 +1,283 @@ +{ + "name": "default", + "description": "Default theme for Presto service", + "configuration": { + "layouts": [ + { + "name": "default", + "tabs": [ + { + "name": "settings", + "display-name": "Settings", + "layout": { + "tab-columns": "2", + "tab-rows": "3", + "sections": [ + { + "name": "section-node-config", + "display-name": "Node Config", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "subsections": [ + { + "name": "subsection-node-config", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + }, + { + "name": "section-general-config", + "display-name": "General Config", + "row-index": "0", + "column-index": "1", + "row-span": "3", + "column-span": "1", + "subsections": [ + { + "name": "subsection-general-config", + "row-index": "0", + "column-index": "0", + "row-span": "3", + "column-span": "1" + } + ] + }, + { + "name": "section-jvm-config", + "display-name": "JVM Config", + "row-index": "1", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "subsections": [ + { + "name": "subsection-jvm-config", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + }, + { + "name": "section-connector-config", + "display-name": "Connectors", + "row-index": "2", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "subsections": [ + { + "name": "subsection-connector-config", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + } + ] + } + } + ] + } + ], + "placement": { + "configuration-layout": "default", + "configs": [ + { + "config": "node.properties/node.environment", + "subsection-name": "subsection-node-config" + }, + { + "config": "node.properties/node.data-dir", + "subsection-name": "subsection-node-config" + }, + { + "config": "node.properties/catalog.config-dir", + "subsection-name": "subsection-node-config" + }, + { + "config": "node.properties/plugin.dir", + "subsection-name": "subsection-node-config" + }, + { + "config": "jvm.config/jvm.config", + "subsection-name": "subsection-jvm-config" + }, + { + "config": "connectors.properties/connectors.to.add", + "subsection-name": "subsection-connector-config" + }, + { + "config": "connectors.properties/connectors.to.delete", + "subsection-name": "subsection-connector-config" + }, + { + "config": "config.properties/node-scheduler.include-coordinator", + "subsection-name": "subsection-general-config" + }, + { + "config": "config.properties/http-server.http.port", + "subsection-name": "subsection-general-config" + }, + { + "config": "config.properties/discovery.uri", + "subsection-name": "subsection-general-config" + }, + { + "config": "config.properties/memory.heap-headroom-per-node", + "subsection-name": "subsection-general-config" + }, + { + "config": "config.properties/query.max-memory", + "subsection-name": "subsection-general-config" + }, + { + "config": "config.properties/query.max-total-memory", + "subsection-name": "subsection-general-config" + }, + { + "config": "config.properties/query.max-memory-per-node", + "subsection-name": "subsection-general-config" + }, + { + "config": "config.properties/query.max-total-memory-per-node", + "subsection-name": "subsection-general-config" + }, + { + "config": "config.properties/experimental.reserved-pool-enabled", + "subsection-name": "subsection-general-config" + } + ] + }, + "widgets": [ + { + "config": "node.properties/node.environment", + "widget": { + "type": "text-area" + } + }, + { + "config": "node.properties/node.data-dir", + "widget": { + "type": "directory" + } + }, + { + "config": "node.properties/catalog.config-dir", + "widget": { + "type": "directory" + } + }, + { + "config": "node.properties/plugin.dir", + "widget": { + "type": "directory" + } + }, + { + "config": "jvm.config/jvm.config", + "widget": { + "type": "text-area" + } + }, + { + "config": "connectors.properties/connectors.to.add", + "widget": { + "type": "text-area" + } + }, + { + "config": "connectors.properties/connectors.to.delete", + "widget": { + "type": "text-area" + } + }, + { + "config": "config.properties/node-scheduler.include-coordinator", + "widget": { + "type": "toggle" + } + }, + { + "config": "config.properties/http-server.http.port", + "widget": { + "type": "text-area" + } + }, + { + "config": "config.properties/discovery.uri", + "widget": { + "type": "text-area" + } + }, + { + "config": "config.properties/memory.heap-headroom-per-node", + "widget": { + "type": "slider", + "units": [ + { + "unit-name": "MB" + } + ] + } + }, + { + "config": "config.properties/query.max-memory", + "widget": { + "type": "slider", + "units": [ + { + "unit-name": "MB" + } + ] + } + }, + { + "config": "config.properties/query.max-total-memory", + "widget": { + "type": "slider", + "units": [ + { + "unit-name": "MB" + } + ] + } + }, + { + "config": "config.properties/query.max-memory-per-node", + "widget": { + "type": "slider", + "units": [ + { + "unit-name": "MB" + } + ] + } + }, + { + "config": "config.properties/query.max-total-memory-per-node", + "widget": { + "type": "slider", + "units": [ + { + "unit-name": "MB" + } + ] + } + }, + { + "config": "config.properties/experimental.reserved-pool-enabled", + "widget": { + "type": "toggle" + } + } + ] + } +} \ No newline at end of file