Merge pull request #1 from laroo/restapi_poc

Django RestAPI Engine POC: A proof of concept that implements a custom Django database engine that connects through (any) RestAPI

django_restapi_engine/base.py

@@ -0,0 +1,204 @@
+import datetime, calendar
+
+from django.db.backends.base.base import BaseDatabaseWrapper
+from django.db.backends.base.client import BaseDatabaseClient
+from django.db.backends.base.creation import BaseDatabaseCreation
+from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.backends.base.features import BaseDatabaseFeatures
+from django.db.backends.base.operations import BaseDatabaseOperations
+from django.db.backends.base.introspection import BaseDatabaseIntrospection
+
+
+class Database(object):
+    class Error(Exception):
+        pass
+
+    class InterfaceError(Error):
+        pass
+
+    class DatabaseError(Error):
+        pass
+
+    class DataError(DatabaseError):
+        pass
+
+    class OperationalError(DatabaseError):
+        pass
+
+    class IntegrityError(DatabaseError):
+        pass
+
+    class InternalError(DatabaseError):
+        pass
+
+    class ProgrammingError(DatabaseError):
+        pass
+
+    class NotSupportedError(DatabaseError):
+        pass
+
+
+class DatabaseIntrospection(BaseDatabaseIntrospection):
+    pass
+
+
+class DatabaseFeatures(BaseDatabaseFeatures):
+    atomic_transactions = False
+    allows_group_by_pk = False
+    empty_fetchmany_value = []
+    has_bulk_insert = False
+    has_select_for_update = False
+    has_zoneinfo_database = False
+    related_fields_match_type = False
+    supports_regex_backreferencing = False
+    supports_sequence_reset = False
+    update_can_self_select = False
+    uses_custom_query_class = False
+
+
+class DatabaseOperations(BaseDatabaseOperations):
+    # compiler_module = "sqlany_django.compiler"
+    compiler_module = "django_db_backend_restapi.compiler"
+
+    def quote_name(self, name):
+        if name.startswith('"') and name.endswith('"'):
+            return name
+        return '"{}"'.format(name)
+
+    def adapt_datefield_value(self, value):
+        if value is None:
+            return None
+        return datetime.datetime.utcfromtimestamp(calendar.timegm(value.timetuple()))
+
+    def adapt_datetimefield_value(self, value):
+        return value
+
+    def adapt_timefield_value(self, value):
+        if value is None:
+            return None
+
+        if isinstance(value, str):
+            return datetime.datetime.strptime(value, '%H:%M:%S')
+
+        return datetime.datetime(1900, 1, 1, value.hour, value.minute, value.second, value.microsecond)
+
+    def convert_datefield_value(self, value, expression, connection, context):
+        if isinstance(value, datetime.datetime):
+            value = value.date()
+        return value
+
+    def convert_timefield_value(self, value, expression, connection, context):
+        if isinstance(value, datetime.datetime):
+            value = value.time()
+        return value
+
+    def get_db_converters(self, expression):
+        converters = super(DatabaseOperations, self).get_db_converters(expression)
+        internal_type = expression.output_field.get_internal_type()
+        if internal_type == 'DateField':
+            converters.append(self.convert_datefield_value)
+        elif internal_type == 'TimeField':
+            converters.append(self.convert_timefield_value)
+        return converters
+
+    def sql_flush(self, style, tables, sequences, allow_cascade=False):
+        # TODO: Need to implement this fully
+        return ['ALTER TABLE']
+
+
+class DatabaseWrapper(BaseDatabaseWrapper):
+    vendor = 'restapi'
+    display_name = 'RestAPI'
+
+    SchemaEditorClass = BaseDatabaseSchemaEditor
+    Database = Database
+
+    client_class = BaseDatabaseClient
+    creation_class = BaseDatabaseCreation
+    features_class = DatabaseFeatures
+    introspection_class = DatabaseIntrospection 
+    ops_class = DatabaseOperations
+
+    operators = {
+        'exact': '= %s',
+        'iexact': 'iLIKE %.*s',
+        'contains': 'LIKE %s',
+        'icontains': 'iLIKE %s',
+        'regex': 'REGEXP BINARY %s',
+        'iregex': 'REGEXP %s',
+        'gt': '> %s',
+        'gte': '>= %s',
+        'lt': '< %s',
+        'lte': '<= %s',
+        'startswith': 'LIKE %s',
+        'endswith': 'LIKE %s',
+        'istartswith': 'iLIKE %s',
+        'iendswith': 'iLIKE %s',
+    }
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def is_usable(self):
+        if self.connection is not None:
+            return True
+        return False
+
+    def get_connection_params(self):
+        """
+        Default method to acquire database connection parameters.
+        Sets connection parameters to match settings.py, and sets
+        default values to blank fields.
+        """
+        valid_settings = {
+            'NAME': 'name',
+            'URL': 'url',
+            'AUTH_TOKEN': 'auth_token',
+        }
+        connection_params = {
+            'name': 'restapi_test',
+            'enforce_schema': True
+        }
+        for setting_name, kwarg in valid_settings.items():
+            try:
+                setting = self.settings_dict[setting_name]
+            except KeyError:
+                continue
+
+            if setting or setting is False:
+                connection_params[kwarg] = setting
+
+        return connection_params
+
+    def get_new_connection(self, connection_params):
+        name = connection_params.pop('name')
+        self.connection = name
+        return self.connection
+
+    def _set_autocommit(self, autocommit):
+        """
+        Default method must be overridden, eventhough not used.
+        TODO: For future reference, setting two phase commits and rollbacks
+        might require populating this method.
+        """
+        pass
+
+    def init_connection_state(self):
+        pass
+
+    def _close(self):
+        """
+        Closes the client connection to the database.
+        """
+        if self.connection:
+            self.connection = None
+
+    def _rollback(self):
+        raise NotImplementedError
+
+    def _commit(self):
+        """
+        Commit routine
+        TODO: two phase commits are not supported yet.
+        """
+        pass

django_restapi_engine/compiler.py

@@ -0,0 +1,135 @@
+import collections
+import pdb
+import re
+from functools import partial
+from itertools import chain
+
+from django.core.exceptions import EmptyResultSet, FieldError
+from django.db import DatabaseError, NotSupportedError
+from django.db.models.constants import LOOKUP_SEP
+from django.db.models.expressions import F, OrderBy, RawSQL, Ref, Value
+from django.db.models.functions import Cast, Random
+from django.db.models.query_utils import Q, select_related_descend
+from django.db.models.sql.constants import (
+    CURSOR, GET_ITERATOR_CHUNK_SIZE, MULTI, NO_RESULTS, ORDER_DIR, SINGLE,
+)
+from django.db.models.sql.query import Query, get_order_dir
+from django.db.transaction import TransactionManagementError
+from django.utils.functional import cached_property
+from django.utils.hashable import make_hashable
+from django.utils.regex_helper import _lazy_re_compile
+
+
+from django.db.models.sql.query import Query
+from django.db.models.sql.subqueries import UpdateQuery
+
+from django.utils.module_loading import import_string
+from .rest_api_handler import BaseRestApiHandler
+
+from django.db.models.sql.compiler import SQLCompiler as DefaultSQLCompiler
+from django.db.models.sql.compiler import SQLUpdateCompiler as DefaultSQLUpdateCompiler
+from django.db.models.sql.compiler import SQLInsertCompiler as DefaultSQLInsertCompiler
+from django.db.models.sql.compiler import SQLDeleteCompiler as DefaultSQLDeleteCompiler
+
+from django.db.models.expressions import Col
+from django.db.models.lookups import Exact, In
+
+from django.db.models.sql.subqueries import UpdateQuery, InsertQuery, DeleteQuery
+
+
+class RestApiCompilerMixin:
+
+    def __init__(self,
+                 query: "django.db.models.sql.query.Query",
+                 connection: "django_db_backend_restapi.base.DatabaseWrapper",
+                 using: str
+                 ):
+        super().__init__(query, connection, using)
+
+        default_handler_class = self.connection.settings_dict['DEFAULT_HANDLER_CLASS']
+        handler_class = import_string(default_handler_class)
+        self.handler: BaseRestApiHandler = handler_class()
+
+
+class SQLCompiler(RestApiCompilerMixin, DefaultSQLCompiler):
+
+
+    def execute_sql(self, result_type=MULTI, chunked_fetch=False, chunk_size=GET_ITERATOR_CHUNK_SIZE):
+        """
+        Run the query against the database and return the result(s). The
+        return value is a single data item if result_type is SINGLE, or an
+        iterator over the results if the result_type is MULTI.
+
+        result_type is either MULTI (use fetchmany() to retrieve all rows),
+        SINGLE (only retrieve a single row), or None. In this last case, the
+        cursor is returned if any query is executed, since it's used by
+        subclasses such as InsertQuery). It's possible, however, that no query
+        is needed, as the filters describe an empty set. In that case, None is
+        returned, to avoid any unnecessary database interaction.
+        """
+        self.pre_sql_setup()
+
+        model = self.query.model
+        model_pk_field = model._meta.pk
+
+        single_where_node = self.query.where.children[0] if self.query.where and len(self.query.where.children) == 1 else None
+
+        if isinstance(single_where_node, Exact) and single_where_node.lhs.target == model_pk_field:
+            row = self.handler.get(model=model, pk=single_where_node.rhs, columns=self.select)
+            return iter([[row]])
+
+        rows = self.handler.list(model=model, columns=self.select, query=self.query)
+        return iter([rows])
+
+
+class SQLInsertCompiler(RestApiCompilerMixin, DefaultSQLInsertCompiler):
+
+    def execute_sql(self, returning_fields=None):
+        self.pre_sql_setup()
+        self.query: InsertQuery
+
+        model = self.query.model
+
+        assert len(self.query.objs) == 1
+
+        row = self.handler.insert(model=model, obj=self.query.objs[0], fields=self.query.fields, returning_fields=returning_fields)
+        return [row]
+
+
+class SQLDeleteCompiler(RestApiCompilerMixin, DefaultSQLDeleteCompiler):
+
+    def execute_sql(self, result_type=MULTI, chunked_fetch=False, chunk_size=GET_ITERATOR_CHUNK_SIZE):
+        self.pre_sql_setup()
+        self.query: DeleteQuery
+
+        model = self.query.model
+        model_pk_field = model._meta.pk
+
+        single_where_node = self.query.where.children[0] if self.query.where and len(self.query.where.children) == 1 else None
+
+        if isinstance(single_where_node, Exact) and single_where_node.lhs.target == model_pk_field:
+            self.handler.delete(model=model, pk=single_where_node.rhs)
+        elif isinstance(single_where_node, In) and single_where_node.lhs.target == model_pk_field:
+            for pk in single_where_node.rhs:
+                self.handler.delete(model=model, pk=pk)
+
+
+class SQLUpdateCompiler(RestApiCompilerMixin, DefaultSQLUpdateCompiler):
+
+    def execute_sql(self, result_type):
+        self.pre_sql_setup()
+        self.query: UpdateQuery
+
+        model = self.query.model
+        model_pk_field = model._meta.pk
+
+        single_where_node = self.query.where.children[0] if self.query.where and len(self.query.where.children) == 1 else None
+
+        if isinstance(single_where_node, Exact) and single_where_node.lhs.target == model_pk_field:
+            num_rows_updated = self.handler.update(model=model, pk=single_where_node.rhs, values=self.query.values)
+            return num_rows_updated
+        raise NotImplementedError("Unsupported UPDATE")
+
+
+class SQLAggregateCompiler(RestApiCompilerMixin, DefaultSQLCompiler):
+    pass

django_restapi_engine/rest_api_handler.py

@@ -0,0 +1,17 @@
+
+class BaseRestApiHandler:
+
+    def list(self, *, model, columns, query):
+        raise NotImplementedError
+
+    def get(self, *, model, pk, columns):
+        raise NotImplementedError
+
+    def insert(self, *, model, obj, fields, returning_fields):
+        raise NotImplementedError
+
+    def update(self, *, model, pk, values):
+        raise NotImplementedError
+
+    def delete(self, *, model, pk):
+        raise NotImplementedError

example_projects/jsonplaceholder_project/README.md

@@ -0,0 +1,35 @@
+
+# Example project: JSONPlaceholder
+
+This example project uses the `Todo` rest API from [JSONPlaceholder](https://jsonplaceholder.typicode.com/)
+
+Please note that this API is readonly and changes (insert, update & delete) won't be applied! 
+
+## Running example project
+
+
+1. Create and activate Virtual Environment
+
+        python3 -m venv .venv
+        source .venv/bin/activate
+
+2.  Install requirements:
+
+        pip install -r requirements.txt
+
+3.  Create local SQLite database:
+
+        python manage.py migrate
+
+4.  Load sample data:
+
+        python manage.py loaddata mysite/fixtures/initial_data
+
+5.  Run testing server:
+
+        python manage.py runserver
+
+6. Take a look at:
+    - Todo view: <http://127.0.0.1:8000/todos/>
+    - Django Admin: <http://localhost:8000/admin/> (log in with username `admin` and password `admin`)
+    - Run manage.py command: `python manage.py fetch_todo`