From eee2e7c8339fe126a7d5ca00afb90eee58178e1d Mon Sep 17 00:00:00 2001 From: syerushalmy Date: Tue, 16 Aug 2016 14:10:50 +0300 Subject: [PATCH 1/9] Added new DS for Cassandra and ScyllaDB --- redash/query_runner/cass.py | 137 ++++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 redash/query_runner/cass.py diff --git a/redash/query_runner/cass.py b/redash/query_runner/cass.py new file mode 100644 index 0000000000..8759f66ee8 --- /dev/null +++ b/redash/query_runner/cass.py @@ -0,0 +1,137 @@ +import json +import sys +import logging + +from redash.query_runner import * +from redash.utils import JSONEncoder + +logger = logging.getLogger(__name__) + +class Cassandra(BaseQueryRunner): + @classmethod + def configuration_schema(cls): + return { + 'type': 'object', + 'properties': { + 'host': { + 'type': 'string', + }, + 'port': { + 'type': 'number', + 'default': 9042, + }, + 'Keyspace': { + 'type': 'string', + 'title': 'Keyspace name' + }, + 'username': { + 'type': 'string', + 'title': 'Username' + }, + 'password': { + 'type': 'string', + 'title': 'Password' + } + }, + 'required': ['Keyspace'] + } + + @classmethod + def type(cls): + return "Cassandra" + + @classmethod + def enabled(cls): + try: + from cassandra.cluster import Cluster + except ImportError: + return False + + return True + + def _get_tables(self, schema): + query = """ + select columnfamily_name from system.schema_columnfamilies where keyspace_name = '{}'; + """.format(self.configuration['Keyspace']) + + results = self.run_query(query) + return results, error + + def run_query(self, query): + from cassandra.cluster import Cluster + connection = None + try: + if self.configuration.get('username', '') and self.configuration.get('password', ''): + auth_provider = PlainTextAuthProvider(username='{}'.format(self.configuration.get('username', '')), + password='{}'.format(self.configuration.get('password', ''))) + connection = Cluster([self.configuration.get('host', '')], auth_provider=auth_provider) + else: + connection = Cluster([self.configuration.get('host', '')]) + + session = connection.connect() + logger.info("Cassandra running query: %s", query) + result = session.execute(query) + + column_names = [] + columns = [] + for i in result.column_names: + column_names.append(i) + + for column_name in column_names: + + columns.append({ + 'name': column_name, + 'friendly_name': column_name, + 'type': 'string' + }) + rows = [dict(zip(column_names, row)) for row in result] + + data = {'columns': columns, 'rows': rows} + json_data = json.dumps(data, cls=JSONEncoder) + + error = None + + except cassandra.cluster.Error, e: + error = e.args[1] + except KeyboardInterrupt: + error = "Query cancelled by user." + except Exception as e: + raise sys.exc_info()[1], None, sys.exc_info()[2] + + return json_data, error + +class ScyllaDB(Cassandra): + @classmethod + def type(cls): + return "scylla" + + @classmethod + def configuration_schema(cls): + return { + 'type': 'object', + 'properties': { + 'host': { + 'type': 'string', + }, + 'port': { + 'type': 'number', + 'default': 9042, + }, + 'Keyspace': { + 'type': 'string', + 'title': 'Keyspace name' + }, + 'username': { + 'type': 'string', + 'title': 'Username' + }, + 'password': { + 'type': 'string', + 'title': 'Password' + } + }, + 'required': ['Keyspace'] + } + +register(Cassandra) +register(ScyllaDB) From 87d77d4d27e7c986b922d3be68b04ec99ea8a1e6 Mon Sep 17 00:00:00 2001 From: syerushalmy Date: Tue, 16 Aug 2016 14:12:35 +0300 Subject: [PATCH 2/9] Added cassandra-driver to requirements_all_ds.txt file --- requirements_all_ds.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements_all_ds.txt b/requirements_all_ds.txt index 6a37763af2..674567e3b0 100644 --- a/requirements_all_ds.txt +++ b/requirements_all_ds.txt @@ -16,3 +16,4 @@ botocore==1.4.4 sasl>=0.1.3 thrift>=0.8.0 thrift_sasl>=0.1.0 +cassandra-driver==3.1.1 From 37873196ec03256bef4c9fad290ff4036118adb6 Mon Sep 17 00:00:00 2001 From: syerushalmy Date: Tue, 16 Aug 2016 15:59:51 +0300 Subject: [PATCH 3/9] Fixed some syntax --- redash/query_runner/cass.py | 57 +++++++++++-------------------------- 1 file changed, 17 insertions(+), 40 deletions(-) diff --git a/redash/query_runner/cass.py b/redash/query_runner/cass.py index 8759f66ee8..f0856f93c5 100644 --- a/redash/query_runner/cass.py +++ b/redash/query_runner/cass.py @@ -7,7 +7,17 @@ logger = logging.getLogger(__name__) +try: + from cassandra.cluster import Cluster + enabled = True +except ImportError: + enabled = False + class Cassandra(BaseQueryRunner): + @classmethod + def enabled(cls): + return enabled + @classmethod def configuration_schema(cls): return { @@ -20,7 +30,7 @@ def configuration_schema(cls): 'type': 'number', 'default': 9042, }, - 'Keyspace': { + 'keyspace': { 'type': 'string', 'title': 'Keyspace name' }, @@ -33,22 +43,13 @@ def configuration_schema(cls): 'title': 'Password' } }, - 'required': ['Keyspace'] + 'required': ['Keyspace', 'host'] } @classmethod def type(cls): return "Cassandra" - @classmethod - def enabled(cls): - try: - from cassandra.cluster import Cluster - except ImportError: - return False - - return True - def _get_tables(self, schema): query = """ select columnfamily_name from system.schema_columnfamilies where keyspace_name = '{}'; @@ -69,7 +70,7 @@ def run_query(self, query): connection = Cluster([self.configuration.get('host', '')]) session = connection.connect() - logger.info("Cassandra running query: %s", query) + logger.debug("Cassandra running query: %s", query) result = session.execute(query) column_names = [] @@ -101,37 +102,13 @@ def run_query(self, query): return json_data, error class ScyllaDB(Cassandra): + + def __init__(self, configuration): + super(ScyllaDB, self).__init__(configuration) + @classmethod def type(cls): return "scylla" - @classmethod - def configuration_schema(cls): - return { - 'type': 'object', - 'properties': { - 'host': { - 'type': 'string', - }, - 'port': { - 'type': 'number', - 'default': 9042, - }, - 'Keyspace': { - 'type': 'string', - 'title': 'Keyspace name' - }, - 'username': { - 'type': 'string', - 'title': 'Username' - }, - 'password': { - 'type': 'string', - 'title': 'Password' - } - }, - 'required': ['Keyspace'] - } - register(Cassandra) register(ScyllaDB) From bd5039ad95f1b667e9c405e4cfa1379600d54eab Mon Sep 17 00:00:00 2001 From: syerushalmy Date: Tue, 16 Aug 2016 16:04:21 +0300 Subject: [PATCH 4/9] Fixed little syntax error --- redash/query_runner/cass.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redash/query_runner/cass.py b/redash/query_runner/cass.py index f0856f93c5..299f6cb7fe 100644 --- a/redash/query_runner/cass.py +++ b/redash/query_runner/cass.py @@ -43,7 +43,7 @@ def configuration_schema(cls): 'title': 'Password' } }, - 'required': ['Keyspace', 'host'] + 'required': ['keyspace', 'host'] } @classmethod From 986dc686bb5f24e101f9e0c54b0e04051c77bda1 Mon Sep 17 00:00:00 2001 From: syerushalmy Date: Wed, 17 Aug 2016 10:42:20 +0300 Subject: [PATCH 5/9] Removed unnessecery exception throw --- redash/query_runner/cass.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/redash/query_runner/cass.py b/redash/query_runner/cass.py index 299f6cb7fe..01c7f9dca5 100644 --- a/redash/query_runner/cass.py +++ b/redash/query_runner/cass.py @@ -96,8 +96,6 @@ def run_query(self, query): error = e.args[1] except KeyboardInterrupt: error = "Query cancelled by user." - except Exception as e: - raise sys.exc_info()[1], None, sys.exc_info()[2] return json_data, error From 37271c746c4fa644cb7e80e8069f8462c24af218 Mon Sep 17 00:00:00 2001 From: syerushalmy Date: Wed, 17 Aug 2016 10:59:09 +0300 Subject: [PATCH 6/9] Switched to fetch_columns instead of messy code --- redash/query_runner/cass.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/redash/query_runner/cass.py b/redash/query_runner/cass.py index 01c7f9dca5..64c60d4382 100644 --- a/redash/query_runner/cass.py +++ b/redash/query_runner/cass.py @@ -73,18 +73,10 @@ def run_query(self, query): logger.debug("Cassandra running query: %s", query) result = session.execute(query) - column_names = [] - columns = [] - for i in result.column_names: - column_names.append(i) - - for column_name in column_names: - - columns.append({ - 'name': column_name, - 'friendly_name': column_name, - 'type': 'string' - }) + column_names = result.column_names + + columns = self.fetch_columns(map(lambda c: (c, 'string'), column_names)) + rows = [dict(zip(column_names, row)) for row in result] data = {'columns': columns, 'rows': rows} From b308e0275c35c99e779f5ed62fc3a04f1c8b670b Mon Sep 17 00:00:00 2001 From: syerushalmy Date: Tue, 23 Aug 2016 16:55:55 +0300 Subject: [PATCH 7/9] Removed cursor.close() from hive_ds because its not needed --- redash/query_runner/hive_ds.py | 1 - 1 file changed, 1 deletion(-) diff --git a/redash/query_runner/hive_ds.py b/redash/query_runner/hive_ds.py index 4dc131216c..683240b949 100644 --- a/redash/query_runner/hive_ds.py +++ b/redash/query_runner/hive_ds.py @@ -119,7 +119,6 @@ def run_query(self, query): data = {'columns': columns, 'rows': rows} json_data = json.dumps(data, cls=JSONEncoder) error = None - cursor.close() except KeyboardInterrupt: connection.cancel() error = "Query cancelled by user." From 7cce9d5d6ebdca8b46f697d84dd2ecf923e895f5 Mon Sep 17 00:00:00 2001 From: syerushalmy Date: Tue, 23 Aug 2016 20:04:17 +0300 Subject: [PATCH 8/9] Added Auth importer for cassandra --- redash/query_runner/cass.py | 1 + 1 file changed, 1 insertion(+) diff --git a/redash/query_runner/cass.py b/redash/query_runner/cass.py index 64c60d4382..65bf3114ed 100644 --- a/redash/query_runner/cass.py +++ b/redash/query_runner/cass.py @@ -63,6 +63,7 @@ def run_query(self, query): connection = None try: if self.configuration.get('username', '') and self.configuration.get('password', ''): + from cassandra.auth import PlainTextAuthProvider auth_provider = PlainTextAuthProvider(username='{}'.format(self.configuration.get('username', '')), password='{}'.format(self.configuration.get('password', ''))) connection = Cluster([self.configuration.get('host', '')], auth_provider=auth_provider) From d41b84eb2e811669ba3336933c744ac4f2d3a0c2 Mon Sep 17 00:00:00 2001 From: syerushalmy Date: Wed, 24 Aug 2016 10:19:20 +0300 Subject: [PATCH 9/9] Fixed syntax error at _get_tables --- redash/query_runner/cass.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redash/query_runner/cass.py b/redash/query_runner/cass.py index 65bf3114ed..73afc539f4 100644 --- a/redash/query_runner/cass.py +++ b/redash/query_runner/cass.py @@ -53,7 +53,7 @@ def type(cls): def _get_tables(self, schema): query = """ select columnfamily_name from system.schema_columnfamilies where keyspace_name = '{}'; - """.format(self.configuration['Keyspace']) + """.format(self.configuration['keyspace']) results = self.run_query(query) return results, error