feat: a native SQLAlchemy dialect for Superset

docs/docs/databases/meta-database.mdx

@@ -0,0 +1,48 @@
+---
+title: Querying across databases
+hide_title: true
+sidebar_position: 42
+version: 1
+---
+
+## Querying across databases
+
+Superset offers an experimental feature for querying across different databases. This is done via a special database called "Superset meta database" that uses the "superset://" SQLAlchemy URI. When using the database it's possible to query any table in any of the configured databases using the following syntax:
+
+```sql
+SELECT * FROM "database name.[[catalog.].schema].table name";
+```
+
+For example:
+
+```sql
+SELECT * FROM "examples.birth_names";
+```
+
+Spaces are allowed, but periods in the names must be replaced by `%2E`. Eg:
+
+```sql
+SELECT * FROM "Superset meta database.examples%2Ebirth_names";
+```
+
+The query above returns the same rows as `SELECT * FROM "examples.birth_names"`, and also shows that the meta database can query tables from any table — even itself!
+
+## Considerations
+
+Before enabling this feature, there are a few considerations that you should have in mind. First, the meta database enforces permissions on the queried tables, so users should only have access via the database to tables that they originally have access to. Nevertheless, the meta database is a new surface for potential attacks, and bugs could allow users to see data they should not.
+
+Second, there are performance considerations. The meta database will push any filtering, sorting, and limiting to the underlying databases, but any aggregations and joins will happen in memory in the process running the query. Because of this, it's recommended to run the database in async mode, so queries are executed in Celery workers, instead of the web workers. Additionally, it's possible to specify a hard limit on how many rows are returned from the underlying databases.
+
+## Enabling the meta database
+
+To enable the Superset meta database, first you need to set the `ENABLE_SUPERSET_META_DB` feature flag to true. Then, add a new database of type "Superset meta database" with the SQLAlchemy URI "superset://".
+
+If you enable DML in the meta database users will be able to run DML queries on underlying databases **as long as DML is also enabled in them**. This allows users to run queries that move data across databases.
+
+Second, you might want to change the value of `SUPERSET_META_DB_LIMIT`. The default value is 1000, and defines how many are read from each database before any aggregations and joins are executed. You can also set this value `None` if you only have small tables.
+
+Additionally, you might want to restrict the databases to with the meta database has access to. This can be done in the database configuration, under "Advanced" -> "Other" -> "ENGINE PARAMETERS" and adding:
+
+```json
+{"allowed_dbs":["Google Sheets","examples"]}
+```

requirements/base.txt

@@ -13,6 +13,8 @@ amqp==5.1.0
     # via kombu
 apispec[yaml]==6.3.0
     # via flask-appbuilder
+apsw==3.42.0.1
+    # via shillelagh
 async-timeout==4.0.2
     # via redis
 attrs==23.1.0
@@ -33,10 +35,14 @@ cachelib==0.4.1
     # via apache-superset
 celery==5.2.2
     # via apache-superset
+certifi==2023.7.22
+    # via requests
 cffi==1.15.1
     # via
     #   cryptography
     #   pynacl
+charset-normalizer==3.2.0
+    # via requests
 click==8.1.3
     # via
     #   apache-superset
@@ -123,7 +129,9 @@ geographiclib==1.52
 geopy==2.2.0
     # via apache-superset
 greenlet==2.0.2
-    # via sqlalchemy
+    # via
+    #   shillelagh
+    #   sqlalchemy
 gunicorn==20.1.0
     # via apache-superset
 hashids==1.3.1
@@ -133,11 +141,14 @@ holidays==0.28
 humanize==3.11.0
     # via apache-superset
 idna==3.2
-    # via email-validator
+    # via
+    #   email-validator
+    #   requests
 importlib-metadata==6.6.0
     # via
     #   apache-superset
     #   flask
+    #   shillelagh
 importlib-resources==5.12.0
     # via limits
 isodate==0.6.0
@@ -205,6 +216,7 @@ packaging==23.1
     #   deprecation
     #   limits
     #   marshmallow
+    #   shillelagh
 pandas[performance]==2.0.3
     # via apache-superset
 paramiko==2.11.0
@@ -244,6 +256,7 @@ python-dateutil==2.8.2
     #   flask-appbuilder
     #   holidays
     #   pandas
+    #   shillelagh
 python-dotenv==0.19.0
     # via apache-superset
 python-editor==1.0.4
@@ -262,10 +275,14 @@ pyyaml==6.0.1
     #   apispec
 redis==4.5.4
     # via apache-superset
+requests==2.31.0
+    # via shillelagh
 rich==13.3.4
     # via flask-limiter
 selenium==3.141.0
     # via apache-superset
+shillelagh==1.2.6
+    # via apache-superset
 shortid==0.1.2
     # via apache-superset
 simplejson==3.17.3
@@ -287,6 +304,7 @@ sqlalchemy==1.4.36
     #   flask-appbuilder
     #   flask-sqlalchemy
     #   marshmallow-sqlalchemy
+    #   shillelagh
     #   sqlalchemy-utils
 sqlalchemy-utils==0.38.3
     # via
@@ -303,10 +321,13 @@ typing-extensions==4.4.0
     #   apache-superset
     #   flask-limiter
     #   limits
+    #   shillelagh
 tzdata==2023.3
     # via pandas
 urllib3==1.26.6
-    # via selenium
+    # via
+    #   requests
+    #   selenium
 vine==5.0.0
     # via
     #   amqp

requirements/development.txt

@@ -26,12 +26,8 @@ botocore==1.29.130
     #   s3transfer
 cached-property==1.5.2
     # via tableschema
-certifi==2023.5.7
-    # via requests
 chardet==5.1.0
     # via tabulator
-charset-normalizer==3.1.0
-    # via requests
 decorator==5.1.1
     # via ipython
 dill==0.3.6

setup.cfg

@@ -30,7 +30,7 @@ combine_as_imports = true
 include_trailing_comma = true
 line_length = 88
 known_first_party = superset
-known_third_party =alembic,apispec,backoff,cachelib,celery,click,colorama,cron_descriptor,croniter,cryptography,dateutil,deprecation,flask,flask_appbuilder,flask_babel,flask_caching,flask_compress,flask_jwt_extended,flask_login,flask_migrate,flask_sqlalchemy,flask_talisman,flask_testing,flask_wtf,freezegun,geohash,geopy,holidays,humanize,isodate,jinja2,jwt,markdown,markupsafe,marshmallow,msgpack,nh3,numpy,pandas,parameterized,parsedatetime,pgsanity,pkg_resources,polyline,prison,progress,pyarrow,sqlalchemy_bigquery,pyhive,pyparsing,pytest,pytest_mock,pytz,redis,requests,selenium,setuptools,simplejson,slack,sqlalchemy,sqlalchemy_utils,sqlparse,typing_extensions,urllib3,werkzeug,wtforms,wtforms_json,yaml
+known_third_party =alembic,apispec,backoff,cachelib,celery,click,colorama,cron_descriptor,croniter,cryptography,dateutil,deprecation,flask,flask_appbuilder,flask_babel,flask_caching,flask_compress,flask_jwt_extended,flask_login,flask_migrate,flask_sqlalchemy,flask_talisman,flask_testing,flask_wtf,freezegun,geohash,geopy,holidays,humanize,isodate,jinja2,jwt,markdown,markupsafe,marshmallow,msgpack,nh3,numpy,pandas,parameterized,parsedatetime,pgsanity,pkg_resources,polyline,prison,progress,pyarrow,sqlalchemy_bigquery,pyhive,pyparsing,pytest,pytest_mock,pytz,redis,requests,selenium,setuptools,shillelagh,simplejson,slack,sqlalchemy,sqlalchemy_utils,sqlparse,typing_extensions,urllib3,werkzeug,wtforms,wtforms_json,yaml
 multi_line_output = 3
 order_by_type = false
 

setup.py

@@ -67,6 +67,10 @@ def get_git_sha() -> str:
         "sqlalchemy.dialects": [
             "postgres.psycopg2 = sqlalchemy.dialects.postgresql:dialect",
             "postgres = sqlalchemy.dialects.postgresql:dialect",
+            "superset = superset.extensions.metadb:SupersetAPSWDialect",
+        ],
+        "shillelagh.adapter": [
+            "superset=superset.extensions.metadb:SupersetShillelaghAdapter"
         ],
     },
     install_requires=[
@@ -115,6 +119,7 @@ def get_git_sha() -> str:
         "PyJWT>=2.4.0, <3.0",
         "redis>=4.5.4, <5.0",
         "selenium>=3.141.0, <4.10.0",
+        "shillelagh>=1.2.6,<2.0",
         "shortid",
         "sshtunnel>=0.4.0, <0.5",
         "simplejson>=3.15.0",
@@ -158,7 +163,7 @@ def get_git_sha() -> str:
         "excel": ["xlrd>=1.2.0, <1.3"],
         "firebird": ["sqlalchemy-firebird>=0.7.0, <0.8"],
         "firebolt": ["firebolt-sqlalchemy>=0.0.1"],
-        "gsheets": ["shillelagh[gsheetsapi]>=1.0.14, <2"],
+        "gsheets": ["shillelagh[gsheetsapi]>=1.2.6, <2"],
         "hana": ["hdbcli==2.4.162", "sqlalchemy_hana==0.4.0"],
         "hive": ["pyhive[hive]>=0.6.5", "tableschema", "thrift>=0.14.1, <1.0.0"],
         "impala": ["impyla>0.16.2, <0.17"],
@@ -181,7 +186,7 @@ def get_git_sha() -> str:
         "redshift": ["sqlalchemy-redshift>=0.8.1, < 0.9"],
         "rockset": ["rockset-sqlalchemy>=0.0.1, <1.0.0"],
         "shillelagh": [
-            "shillelagh[datasetteapi,gsheetsapi,socrata,weatherapi]>=1.1.1, <2"
+            "shillelagh[datasetteapi,gsheetsapi,socrata,weatherapi]>=1.2.6,<2"
         ],
         "snowflake": ["snowflake-sqlalchemy>=1.2.4, <2"],
         "spark": ["pyhive[hive]>=0.6.5", "tableschema", "thrift>=0.14.1, <1.0.0"],

superset/config.py

@@ -480,6 +480,12 @@ class D3Format(TypedDict, total=False):
     # or to disallow users from viewing other users profile page
     # Do not show user info or profile in the menu
     "MENU_HIDE_USER_INFO": False,
+    # Allows users to add a ``superset://`` DB that can query across databases. This is
+    # an experimental feature with potential security and performance risks, so use with
+    # caution. If the feature is enabled you can also set a limit for how much data is
+    # returned from each database in the ``SUPERSET_META_DB_LIMIT`` configuration value
+    # in this file.
+    "ENABLE_SUPERSET_META_DB": False,
 }
 
 # ------------------------------
@@ -871,6 +877,9 @@ class D3Format(TypedDict, total=False):
 # the SQL Lab UI
 DEFAULT_SQLLAB_LIMIT = 1000
 
+# The limit for the Superset Meta DB when the feature flag ENABLE_SUPERSET_META_DB is on
+SUPERSET_META_DB_LIMIT: int | None = 1000
+
 # Adds a warning message on sqllab save query and schedule query modals.
 SQLLAB_SAVE_WARNING_MESSAGE = None
 SQLLAB_SCHEDULE_WARNING_MESSAGE = None

superset/db_engine_specs/__init__.py

@@ -41,7 +41,7 @@
 from sqlalchemy.engine.default import DefaultDialect
 from sqlalchemy.engine.url import URL
 
-from superset import app
+from superset import app, feature_flag_manager
 from superset.db_engine_specs.base import BaseEngineSpec
 
 logger = logging.getLogger(__name__)
@@ -120,6 +120,7 @@ def get_engine_spec(backend: str, driver: Optional[str] = None) -> type[BaseEngi
 }
 
 
+# pylint: disable=too-many-branches
 def get_available_engine_specs() -> dict[type[BaseEngineSpec], set[str]]:
     """
     Return available engine specs and installed drivers for them.
@@ -172,6 +173,8 @@ def get_available_engine_specs() -> dict[type[BaseEngineSpec], set[str]]:
 
         # do not add denied db engine specs to available list
         dbs_denylist = app.config["DBS_AVAILABLE_DENYLIST"]
+        if not feature_flag_manager.is_feature_enabled("ENABLE_SUPERSET_META_DB"):
+            dbs_denylist["superset"] = {""}
         dbs_denylist_engines = dbs_denylist.keys()
 
         if (

superset/db_engine_specs/superset.py

@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+A native Superset database.
+"""
+
+from superset.db_engine_specs.shillelagh import ShillelaghEngineSpec
+
+
+class SupersetEngineSpec(ShillelaghEngineSpec):
+    """
+    Internal engine for Superset
+
+    This DB engine spec is a meta-database. It uses the shillelagh library
+    to build a DB that can operate across different Superset databases.
+    """
+
+    engine = "superset"
+    engine_name = "Superset meta database"
+    drivers = {"": "Native driver"}
+    default_driver = ""
+    sqlalchemy_uri_placeholder = "superset://"
+
+    supports_file_upload = False