From eccccb19b542606325cef78e750ba51ad47dfbad Mon Sep 17 00:00:00 2001 From: pdelboca Date: Wed, 22 Feb 2023 11:11:16 -0300 Subject: [PATCH 01/14] Drop support old ckan versions and PY2 --- .github/workflows/test.yml | 22 +- MANIFEST.in | 2 - ckanext/xloader/action.py | 6 - ckanext/xloader/command.py | 5 +- ckanext/xloader/controllers.py | 7 - ckanext/xloader/jobs.py | 1 - ckanext/xloader/loader.py | 50 +--- ckanext/xloader/parser.py | 52 +---- ckanext/xloader/paster.py | 219 ------------------ ckanext/xloader/plugin.py | 55 +---- .../package/resource_edit_base.html | 6 - .../templates-bs2/xloader/resource_data.html | 88 ------- ckanext/xloader/utils.py | 8 - full_text_function.sql | 16 -- 14 files changed, 32 insertions(+), 505 deletions(-) delete mode 100644 ckanext/xloader/controllers.py delete mode 100644 ckanext/xloader/paster.py delete mode 100644 ckanext/xloader/templates-bs2/package/resource_edit_base.html delete mode 100644 ckanext/xloader/templates-bs2/xloader/resource_data.html delete mode 100644 full_text_function.sql diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e963e1f6..a2ebb89f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,22 +10,20 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 with: - python-version: '3.6' + python-version: '3.10' - name: Install requirements run: pip install flake8 pycodestyle - name: Check syntax run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --extend-exclude ckan - #- name: Run flake8 - # run: flake8 . --count --max-line-length=127 --statistics --exclude ckan test: needs: lint strategy: matrix: - ckan-version: ["2.10", 2.9, 2.9-py2, 2.8, 2.7] + ckan-version: ["2.10", 2.9] fail-fast: false name: CKAN ${{ matrix.ckan-version }} @@ -54,7 +52,7 @@ jobs: CKAN_REDIS_URL: redis://redis:6379/1 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Install requirements run: | pip install -r requirements.txt @@ -64,17 +62,7 @@ jobs: # Replace default path to CKAN core config file with the one on the container sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini - name: Setup extension (CKAN >= 2.9) - if: ${{ matrix.ckan-version != '2.7' && matrix.ckan-version != '2.8' }} run: | ckan -c test.ini db init - - name: Setup extension (CKAN 2.8) - if: ${{ matrix.ckan-version == '2.8' }} - run: | - paster --plugin=ckan db init -c test.ini - - name: Setup extension (CKAN 2.7) - if: ${{ matrix.ckan-version == '2.7' }} - run: | - psql -d "postgresql://datastore_write:pass@postgres/datastore_test" -f full_text_function.sql - paster --plugin=ckan db init -c test.ini - name: Run tests run: pytest --ckan-ini=test.ini --cov=ckanext.xloader --disable-warnings ckanext/xloader/tests diff --git a/MANIFEST.in b/MANIFEST.in index 820ba404..6c6f273b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,5 @@ -include full_text_function.sql include *requirements*.txt include CHANGELOG include LICENSE include README.rst recursive-include ckanext/xloader/templates *.html -recursive-include ckanext/xloader/templates-bs2 *.html diff --git a/ckanext/xloader/action.py b/ckanext/xloader/action.py index 983e1919..3fa26803 100644 --- a/ckanext/xloader/action.py +++ b/ckanext/xloader/action.py @@ -11,7 +11,6 @@ import ckan.plugins as p from dateutil.parser import parse as parse_date from dateutil.parser import isoparse as parse_iso_date -from six import text_type as str import ckanext.xloader.schema @@ -158,11 +157,6 @@ def xloader_submit(context, data_dict): job = enqueue_job( jobs.xloader_data_into_datastore, [data], rq_kwargs=dict(timeout=timeout) ) - except TypeError: - # This except provides support for 2.7. - job = _enqueue( - jobs.xloader_data_into_datastore, [data], timeout=timeout - ) except Exception: log.exception('Unable to enqueued xloader res_id=%s', res_id) return False diff --git a/ckanext/xloader/command.py b/ckanext/xloader/command.py index ffde1c68..7f2c000a 100644 --- a/ckanext/xloader/command.py +++ b/ckanext/xloader/command.py @@ -119,10 +119,7 @@ def _submit_resource(self, resource, user, indent=0): self.error_occured = True def print_status(self): - try: - import ckan.lib.jobs as rq_jobs - except ImportError: - import ckanext.rq.jobs as rq_jobs + import ckan.lib.jobs as rq_jobs jobs = rq_jobs.get_queue().jobs if not jobs: print('No jobs currently queued') diff --git a/ckanext/xloader/controllers.py b/ckanext/xloader/controllers.py deleted file mode 100644 index a1ab3c3f..00000000 --- a/ckanext/xloader/controllers.py +++ /dev/null @@ -1,7 +0,0 @@ -import ckan.plugins as p -import ckanext.xloader.utils as utils - - -class ResourceDataController(p.toolkit.BaseController): - def resource_data(self, id, resource_id): - return utils.resource_data(id, resource_id) diff --git a/ckanext/xloader/jobs.py b/ckanext/xloader/jobs.py index b2e01d69..4c4068f9 100644 --- a/ckanext/xloader/jobs.py +++ b/ckanext/xloader/jobs.py @@ -9,7 +9,6 @@ import datetime import traceback import sys -from six import text_type as str from six.moves.urllib.parse import urlsplit import requests diff --git a/ckanext/xloader/loader.py b/ckanext/xloader/loader.py index e1de12ae..2851e72b 100644 --- a/ckanext/xloader/loader.py +++ b/ckanext/xloader/loader.py @@ -9,7 +9,6 @@ from decimal import Decimal import psycopg2 -from six import text_type as str from six.moves import zip from tabulator import Stream, TabulatorException from unidecode import unidecode @@ -21,25 +20,11 @@ from .parser import XloaderCSVParser from .utils import headers_guess, type_guess -try: - from ckan.plugins.toolkit import config -except ImportError: - # older versions of ckan - from pylons import config - -if tk.check_ckan_version(min_version='2.7'): - import ckanext.datastore.backend.postgres as datastore_db - get_write_engine = datastore_db.get_write_engine -else: - # older versions of ckan - def get_write_engine(): - from ckanext.datastore.db import _get_engine - from pylons import config - data_dict = {'connection_url': config['ckan.datastore.write_url']} - return _get_engine(data_dict) - import ckanext.datastore.db as datastore_db +from ckan.plugins.toolkit import config +import ckanext.datastore.backend.postgres as datastore_db +get_write_engine = datastore_db.get_write_engine create_indexes = datastore_db.create_indexes _drop_indexes = datastore_db._drop_indexes @@ -356,12 +341,11 @@ def row_iterator(): "": 'numeric', "": 'numeric', "": 'numeric', - "": 'timestamp', # Python 2 "": 'text', "": 'text', "": 'numeric', "": 'numeric', - "": 'timestamp', # Python 3 + "": 'timestamp', } @@ -514,29 +498,3 @@ def calculate_record_count(resource_id, logger): conn = engine.connect() conn.execute("ANALYZE \"{resource_id}\";" .format(resource_id=resource_id)) - - -################################ -# datastore copied code # -# (for use with older ckans that lack this) - -def _create_fulltext_trigger(connection, resource_id): - connection.execute( - u'''CREATE TRIGGER zfulltext - BEFORE INSERT OR UPDATE ON {table} - FOR EACH ROW EXECUTE PROCEDURE populate_full_text_trigger()'''.format( - table=identifier(resource_id))) - - -def identifier(s): - # "%" needs to be escaped, otherwise connection.execute thinks it is for - # substituting a bind parameter - return u'"' + s.replace(u'"', u'""').replace(u'\0', '').replace('%', '%%')\ - + u'"' - - -def literal_string(s): - return u"'" + s.replace(u"'", u"''").replace(u'\0', '') + u"'" - -# end of datastore copied code # -################################ diff --git a/ckanext/xloader/parser.py b/ckanext/xloader/parser.py index 81ec4583..b2a6f889 100644 --- a/ckanext/xloader/parser.py +++ b/ckanext/xloader/parser.py @@ -7,19 +7,12 @@ import six from ckan.plugins.toolkit import asbool from dateutil.parser import isoparser, parser -try: - from dateutil.parser import ParserError -except ImportError: - ParserError = ValueError +from dateutil.parser import ParserError from tabulator import helpers from tabulator.parser import Parser -try: - from ckan.plugins.toolkit import config -except ImportError: - # older versions of ckan - from pylons import config +from ckan.plugins.toolkit import config CSV_SAMPLE_LINES = 100 @@ -42,13 +35,6 @@ class XloaderCSVParser(Parser): def __init__(self, loader, force_parse=False, **options): super(XloaderCSVParser, self).__init__(loader, force_parse, **options) - - # Make bytes - if six.PY2: - for key, value in options.items(): - if isinstance(value, six.string_types): - options[key] = six.text_type(value) - # Set attributes self.__loader = loader self.__options = options @@ -136,30 +122,14 @@ def type_value(value): return value - # For PY2 encode/decode - if six.PY2: - # Reader requires utf-8 encoded stream - bytes = iterencode(self.__chars, 'utf-8') - sample, dialect = self.__prepare_dialect(bytes) - items = csv.reader(chain(sample, bytes), dialect=dialect) - for row_number, item in enumerate(items, start=1): - values = [] - for value in item: - value = value.decode('utf-8') - value = type_value(value) - values.append(value) - yield row_number, None, list(values) - - # For PY3 use chars - else: - sample, dialect = self.__prepare_dialect(self.__chars) - items = csv.reader(chain(sample, self.__chars), dialect=dialect) - for row_number, item in enumerate(items, start=1): - values = [] - for value in item: - value = type_value(value) - values.append(value) - yield row_number, None, list(values) + sample, dialect = self.__prepare_dialect(self.__chars) + items = csv.reader(chain(sample, self.__chars), dialect=dialect) + for row_number, item in enumerate(items, start=1): + values = [] + for value in item: + value = type_value(value) + values.append(value) + yield row_number, None, list(values) def __prepare_dialect(self, stream): @@ -175,7 +145,7 @@ def __prepare_dialect(self, stream): # Get dialect try: - separator = b'' if six.PY2 else '' + separator = '' delimiter = self.__options.get('delimiter', ',\t;|') dialect = csv.Sniffer().sniff(separator.join(sample), delimiter) if not dialect.escapechar: diff --git a/ckanext/xloader/paster.py b/ckanext/xloader/paster.py deleted file mode 100644 index 7e58a119..00000000 --- a/ckanext/xloader/paster.py +++ /dev/null @@ -1,219 +0,0 @@ -from __future__ import print_function -import sys - -import ckan.lib.cli as cli -import ckan.plugins as p -import ckan.model as model - -import ckanext.datastore.helpers as h -from ckanext.xloader.command import XloaderCmd - -# Paster command for CKAN 2.8 and below - - -class xloaderCommand(cli.CkanCommand): - '''xloader commands - - Usage: - - xloader submit [options] - Submit the given datasets' resources to be xloaded into the - DataStore. (They are added to the queue for CKAN's background task - worker.) - - where is one of: - - - Submit a particular dataset's resources - - - Submit a particular dataset's resources - - all - Submit all datasets' resources to the DataStore - - all-existing - Re-submits all the resources already in the - DataStore. (Ignores any resources that have not been stored - in DataStore, e.g. because they are not tabular) - - options: - - --dry-run - doesn't actually submit any resources - - --ignore-format - submit resources even if they have a format - not in the configured ckanext.xloader.formats - - xloader status - Shows status of jobs - ''' - - summary = __doc__.split('\n')[0] - usage = __doc__ - min_args = 1 - - def __init__(self, name): - super(xloaderCommand, self).__init__(name) - self.error_occured = False - - self.parser.add_option('-y', dest='yes', - action='store_true', default=False, - help='Always answer yes to questions') - self.parser.add_option('--ignore-format', - action='store_true', default=False, - help='Submit even if the resource.format is not' - ' in ckanext.xloader.formats') - self.parser.add_option('--dry-run', - action='store_true', default=False, - help='Don\'t actually submit anything') - - def command(self): - cmd = XloaderCmd(self.options.dry_run) - if not self.args: - print(self.usage) - sys.exit(1) - if self.args[0] == 'submit': - if len(self.args) < 2: - self.parser.error('This command requires an argument') - if self.args[1] == 'all': - self._load_config() - cmd._setup_xloader_logger() - cmd._submit_all() - elif self.args[1] == 'all-existing': - self._confirm_or_abort() - self._load_config() - cmd._setup_xloader_logger() - cmd._submit_all_existing() - else: - pkg_name_or_id = self.args[1] - self._load_config() - cmd._setup_xloader_logger() - cmd._submit_package(pkg_name_or_id) - self._handle_command_status(cmd.error_occured) - elif self.args[0] == 'status': - self._load_config() - cmd.print_status() - else: - self.parser.error('Unrecognized command') - - def _handle_command_status(self, error_occured): - if error_occured: - print('Finished but saw errors - see above for details') - sys.exit(1) - - def _confirm_or_abort(self): - if self.options.yes or self.options.dry_run: - return - question = ( - "Data in any datastore resource that isn't in their source files " - "(e.g. data added using the datastore API) will be permanently " - "lost. Are you sure you want to proceed?" - ) - answer = cli.query_yes_no(question, default=None) - if not answer == 'yes': - print("Aborting...") - sys.exit(0) - - -class MigrateTypesCommand(cli.CkanCommand): - '''Migrate command - - Turn existing resource field types into Data Dictionary overrides. - This is intended to simplify migration from DataPusher to XLoader, - by allowing you to reuse the types that DataPusher has guessed. - - Usage: - - migrate_types [options] [resource-spec] - Add the given resources' field types to the Data Dictionary. - - where resource-spec is one of: - - - Migrate a particular resource - - all - Migrate all resources (this is the default) - - ''' - summary = __doc__.split('\n')[0] - usage = __doc__ - min_args = 0 - - def __init__(self, name): - super(MigrateTypesCommand, self).__init__(name) - self.error_occured = False - - self.parser.add_option('-t', '--include-text', - action='store_true', default=False, - help='Add Data Dictionary overrides even for text fields') - - self.parser.add_option('--force', - action='store_true', default=False, - help='Overwrite existing data dictionary if it exists') - - def command(self): - self._load_config() - if not self.args or len(self.args) == 0 or self.args[0] == 'all': - self._migrate_all() - else: - self._migrate_resource(self.args[0]) - self._handle_command_status() - - def _migrate_all(self): - session = model.Session - resource_count = session.query(model.Resource).filter_by(state='active').count() - print("Updating {} resource(s)".format(resource_count)) - resources_done = 0 - for resource in session.query(model.Resource).filter_by(state='active'): - resources_done += 1 - self._migrate_resource(resource.id, - prefix='[{}/{}]: '.format(resources_done, - resource_count)) - if resources_done % 100 == 0: - print("[{}/{}] done".format(resources_done, resource_count)) - print("[{}/{}] done".format(resources_done, resource_count)) - - def _migrate_resource(self, resource_id, prefix=''): - data_dict = h.datastore_dictionary(resource_id) - - def print_status(status): - if self.options.verbose: - print("{}{}: {}".format(prefix, resource_id, status)) - - if not data_dict: - print_status("not found") - return - - fields = [] - for field in data_dict: - if field['type'] == 'text' and not self.options.include_text: - type_override = '' - else: - type_override = field['type'] - - if 'info' not in field: - field.update({'info': {'notes': '', - 'type_override': type_override, - 'label': ''}}) - elif self.options.force: - field['info'].update({'type_override': type_override}) - else: - print_status("skipped") - return - - fields.append({ - 'id': field['id'], - 'type': field['type'], - 'info': field['info'] - }) - - try: - p.toolkit.get_action('datastore_create')(None, { - 'resource_id': resource_id, - 'force': True, - 'fields': fields - }) - print_status("updated") - except Exception as e: - self.error_occured = True - print("{}: failed, {}".format(resource_id, e)) - - def _handle_command_status(self): - if self.error_occured: - print('Finished but saw errors - see above for details') - sys.exit(1) diff --git a/ckanext/xloader/plugin.py b/ckanext/xloader/plugin.py index 09e7b17a..ef948376 100644 --- a/ckanext/xloader/plugin.py +++ b/ckanext/xloader/plugin.py @@ -49,44 +49,25 @@ class xloaderPlugin(plugins.SingletonPlugin): plugins.implements(plugins.IAuthFunctions) plugins.implements(plugins.ITemplateHelpers) plugins.implements(plugins.IResourceController, inherit=True) + plugins.implements(plugins.IClick) + plugins.implements(plugins.IBlueprint) - if toolkit.check_ckan_version("2.9"): - plugins.implements(plugins.IClick) - plugins.implements(plugins.IBlueprint) + # IClick + def get_commands(self): + from ckanext.xloader.cli import get_commands - # IClick - def get_commands(self): - from ckanext.xloader.cli import get_commands + return get_commands() - return get_commands() + # IBlueprint + def get_blueprint(self): + from ckanext.xloader.views import get_blueprints - # IBlueprint - def get_blueprint(self): - from ckanext.xloader.views import get_blueprints - - return get_blueprints() - - else: - plugins.implements(plugins.IRoutes, inherit=True) - - # IRoutes - def before_map(self, m): - m.connect( - "xloader.resource_data", - "/dataset/{id}/resource_data/{resource_id}", - controller="ckanext.xloader.controllers:ResourceDataController", - action="resource_data", - ckan_icon="cloud-upload", - ) - return m + return get_blueprints() # IConfigurer def update_config(self, config): - templates_base = config.get( - "ckan.base_templates_folder", "templates-bs2" - ) # for ckan < 2.8 - toolkit.add_template_directory(config, templates_base) + toolkit.add_template_directory(config, 'templates/') # IConfigurable @@ -104,20 +85,6 @@ def configure(self, config_): ) ) - if toolkit.check_ckan_version(max_version="2.7.99"): - # populate_full_text_trigger() needs to be defined, and this was - # introduced in CKAN 2.8 when you installed datastore e.g.: - # paster datastore set-permissions - # However before CKAN 2.8 we need to check the user has defined - # this function manually. - connection = get_write_engine().connect() - if not fulltext_function_exists(connection): - raise Exception( - "populate_full_text_trigger is not defined. " - "See ckanext-xloader's README.rst for more " - "details." - ) - # IResourceUrlChange def notify(self, resource): diff --git a/ckanext/xloader/templates-bs2/package/resource_edit_base.html b/ckanext/xloader/templates-bs2/package/resource_edit_base.html deleted file mode 100644 index 34403521..00000000 --- a/ckanext/xloader/templates-bs2/package/resource_edit_base.html +++ /dev/null @@ -1,6 +0,0 @@ -{% ckan_extends %} - -{% block inner_primary_nav %} - {{ super() }} - {{ h.build_nav_icon('xloader.resource_data', _('DataStore'), id=pkg.name, resource_id=res.id) }} -{% endblock %} diff --git a/ckanext/xloader/templates-bs2/xloader/resource_data.html b/ckanext/xloader/templates-bs2/xloader/resource_data.html deleted file mode 100644 index ace37859..00000000 --- a/ckanext/xloader/templates-bs2/xloader/resource_data.html +++ /dev/null @@ -1,88 +0,0 @@ -{% extends "package/resource_edit_base.html" %} - -{% block subtitle %}{{ h.dataset_display_name(pkg) }} - {{ h.resource_display_name(res) }}{% endblock %} - -{% block primary_content_inner %} - - {% set action = h.url_for(controller='ckanext.xloader.controllers:ResourceDataController', action='resource_data', id=pkg.name, resource_id=res.id) %} - {% set show_table = true %} - -
- -
- - {% if status.error and status.error.message %} - {% set show_table = false %} -
- {{ _('Upload error:') }} {{ status.error.message }} -
- {% elif status.task_info and status.task_info.error %} -
- {% if status.task_info.error is string %} - {# DataPusher < 0.0.3 #} - {{ _('Error:') }} {{ status.task_info.error }} - {% elif status.task_info.error is mapping %} - {{ _('Error:') }} {{ status.task_info.error.message }} - {% for error_key, error_value in status.task_info.error.items() %} - {% if error_key != "message" and error_value %} -
- {{ error_key }}: - {{ error_value }} - {% endif %} - {% endfor %} - {% elif status.task_info.error is iterable %} - {{ _('Error traceback:') }} - 
{{ ''.join(status.task_info.error) }}
- {% endif %} -
- {% endif %} - - - - - - - - - - - - - {% if status.status %} - - {% else %} - - {% endif %} - -
{{ _('Status') }}{{ h.xloader_status_description(status) }}
{{ _('Last updated') }}{{ h.time_ago_from_timestamp(status.last_updated) }}{{ _('Never') }}
- - {% if status.status and status.task_info and show_table %} -

{{ _('Upload Log') }}

-
    - {% for item in status.task_info.logs %} - {% set icon = 'ok' if item.level == 'INFO' else 'exclamation' %} - {% set class = ' failure' if icon == 'exclamation' else ' success' %} - {% set popover_content = 'test' %} -
  • - -

    - {% for line in item.message.strip().split('\n') %} - {{ line | urlize }}
    - {% endfor %} - - {{ h.time_ago_from_timestamp(item.timestamp) }} - {{ _('Details') }} - -

    -
    • - {% endfor %} -
  • - -

    {{ _('End of log') }}

    -
    • -
- {% endif %} - -{% endblock %} diff --git a/ckanext/xloader/utils.py b/ckanext/xloader/utils.py index 2fbfc72d..cbffaa2f 100644 --- a/ckanext/xloader/utils.py +++ b/ckanext/xloader/utils.py @@ -8,8 +8,6 @@ from collections import defaultdict from decimal import Decimal -from six import text_type as str - import ckan.plugins as p @@ -90,12 +88,6 @@ def set_resource_metadata(update_dict): extras.update(update_dict) q.update({'extras': extras}, synchronize_session=False) - # TODO: Remove resource_revision_table when dropping support for 2.8 - if hasattr(model, 'resource_revision_table'): - model.Session.query(model.resource_revision_table).filter( - model.ResourceRevision.id == update_dict['resource_id'], - model.ResourceRevision.current is True - ).update({'extras': extras}, synchronize_session=False) model.Session.commit() # get package with updated resource from solr diff --git a/full_text_function.sql b/full_text_function.sql deleted file mode 100644 index 8a604258..00000000 --- a/full_text_function.sql +++ /dev/null @@ -1,16 +0,0 @@ --- _full_text fields are now updated by a trigger when set to NULL --- copied from https://github.com/ckan/ckan/pull/3786/files#diff-33d20faeb53559a9b8940bcb418cb5b4R75 -CREATE OR REPLACE FUNCTION populate_full_text_trigger() RETURNS trigger -AS $body$ - BEGIN - IF NEW._full_text IS NOT NULL THEN - RETURN NEW; - END IF; - NEW._full_text := ( - SELECT to_tsvector(string_agg(value, ' ')) - FROM json_each_text(row_to_json(NEW.*)) - WHERE key NOT LIKE '\_%'); - RETURN NEW; - END; -$body$ LANGUAGE plpgsql; -ALTER FUNCTION populate_full_text_trigger() OWNER TO ckan_default; From b4f0c8da4dbd8fc69ed21347ba4ee9b042f1b631 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Wed, 22 Feb 2023 11:20:37 -0300 Subject: [PATCH 02/14] Re introduce helper functions --- ckanext/xloader/loader.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ckanext/xloader/loader.py b/ckanext/xloader/loader.py index 2851e72b..e189c52b 100644 --- a/ckanext/xloader/loader.py +++ b/ckanext/xloader/loader.py @@ -141,9 +141,6 @@ def load_csv(csv_filepath, resource_id, mimetype='text/csv', logger=None): raise LoaderError('Could not create the database table: {}' .format(e)) connection = context['connection'] = engine.connect() - if not fulltext_trigger_exists(connection, resource_id): - logger.info('Trigger created') - _create_fulltext_trigger(connection, resource_id) # datstore_active is switched on by datastore_create - TODO temporarily # disable it until the load is complete @@ -498,3 +495,14 @@ def calculate_record_count(resource_id, logger): conn = engine.connect() conn.execute("ANALYZE \"{resource_id}\";" .format(resource_id=resource_id)) + + +def identifier(s): + # "%" needs to be escaped, otherwise connection.execute thinks it is for + # substituting a bind parameter + return u'"' + s.replace(u'"', u'""').replace(u'\0', '').replace('%', '%%')\ + + u'"' + + +def literal_string(s): + return u"'" + s.replace(u"'", u"''").replace(u'\0', '') + u"'" From 56d011bf35cf759b42141d9a499db31d19919099 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Wed, 22 Feb 2023 11:22:26 -0300 Subject: [PATCH 03/14] Fix lint --- ckanext/xloader/loader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckanext/xloader/loader.py b/ckanext/xloader/loader.py index e189c52b..afc3c980 100644 --- a/ckanext/xloader/loader.py +++ b/ckanext/xloader/loader.py @@ -39,7 +39,7 @@ def load_csv(csv_filepath, resource_id, mimetype='text/csv', logger=None): file_format = os.path.splitext(csv_filepath)[1].strip('.') with Stream(csv_filepath, format=file_format) as stream: header_offset, headers = headers_guess(stream.sample) - except TabulatorException as e: + except TabulatorException: try: file_format = mimetype.lower().split('/')[-1] with Stream(csv_filepath, format=file_format) as stream: @@ -240,7 +240,7 @@ def load_table(table_filepath, resource_id, mimetype='text/csv', logger=None): with Stream(table_filepath, format=file_format, custom_parsers={'csv': XloaderCSVParser}) as stream: header_offset, headers = headers_guess(stream.sample) - except TabulatorException as e: + except TabulatorException: try: file_format = mimetype.lower().split('/')[-1] with Stream(table_filepath, format=file_format, From 8942f233cae773b70c0dcb0e5e83f739514d2dc4 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Wed, 22 Feb 2023 11:33:51 -0300 Subject: [PATCH 04/14] Remove references to no longer supported version --- README.rst | 106 ++++++++++++----------------------------------------- setup.py | 11 ++---- 2 files changed, 27 insertions(+), 90 deletions(-) diff --git a/README.rst b/README.rst index a09da596..81cfe72e 100644 --- a/README.rst +++ b/README.rst @@ -69,8 +69,8 @@ DataPusher - job queue is done by ckan-service-provider which is bespoke, complicated and stores jobs in its own database (sqlite by default). XLoader - job queue is done by RQ, which is simpler, is backed by Redis, allows -access to the CKAN model and is CKAN's default queue technology (since CKAN -2.7). You can also debug jobs easily using pdb. Job results are stored in +access to the CKAN model and is CKAN's default queue technology. +You can also debug jobs easily using pdb. Job results are stored in Sqlite by default, and for production simply specify CKAN's database in the config and it's held there - easy. @@ -98,7 +98,7 @@ Caveat - column types Note: With XLoader, all columns are stored in DataStore's database as 'text' type (whereas DataPusher did some rudimentary type guessing - see 'Robustness' above). However once a resource is xloaded, an admin can use the resource's -Data Dictionary tab (CKAN 2.7 onwards) to change these types to numeric or +Data Dictionary tab to change these types to numeric or datestamp and re-load the file. When migrating from DataPusher to XLoader you can preserve the types of existing resources by using the ``migrate_types`` command. @@ -116,13 +116,10 @@ Compatibility with core CKAN versions: =============== ============= CKAN version Compatibility =============== ============= -2.3 no longer tested and you must install ckanext-rq -2.4 no longer tested and you must install ckanext-rq -2.5 no longer tested and you must install ckanext-rq -2.6 no longer tested and you must install ckanext-rq -2.7 yes -2.8 yes -2.9 yes (both Python2 and Python3) +2.7 no longer supported (last supported version: 0.12.2) +2.8 no longer supported (last supported version: 0.12.2) +2.9 yes (Python3) (last supported version for Python 2.7: 0.12.2)) +2.10 yes =============== ============= ------------ @@ -144,24 +141,7 @@ To install XLoader: pip install -r https://raw.githubusercontent.com/ckan/ckanext-xloader/master/requirements.txt pip install -U requests[security] -4. If you are using CKAN version before 2.8.x you need to define the - ``populate_full_text_trigger`` in your database - :: - - sudo -u postgres psql datastore_default -f full_text_function.sql - - If successful it will print - :: - - CREATE FUNCTION - ALTER FUNCTION - - NB this assumes you used the defaults for the database name and username. - If in doubt, check your config's ``ckan.datastore.write_url``. If you don't have - database name ``datastore_default`` and username ``ckan_default`` then adjust - the psql option and ``full_text_function.sql`` before running this. - -5. Add ``xloader`` to the ``ckan.plugins`` setting in your CKAN +4. Add ``xloader`` to the ``ckan.plugins`` setting in your CKAN config file (by default the config file is located at ``/etc/ckan/default/production.ini``). @@ -170,12 +150,12 @@ To install XLoader: Ensure ``datastore`` is also listed, to enable CKAN DataStore. -6. Starting CKAN 2.10 you will need to set an API Token to be able to +5. Starting CKAN 2.10 you will need to set an API Token to be able to execute jobs against the server:: ckanext.xloader.api_token = -7. If it is a production server, you'll want to store jobs info in a more +6. If it is a production server, you'll want to store jobs info in a more robust database than the default sqlite file. It can happily use the main CKAN postgres db by adding this line to the config, but with the same value as you have for ``sqlalchemy.url``:: @@ -184,35 +164,13 @@ To install XLoader: (This step can be skipped when just developing or testing.) -8. Restart CKAN. For example if you've deployed CKAN with Apache on Ubuntu:: +7. Restart CKAN. For example if you've deployed CKAN with Apache on Ubuntu:: sudo service apache2 reload -9. Run the worker. First test it on the command-line. If you have CKAN version 2.9 or above:: - - ckan -c /etc/ckan/default/ckan.ini jobs worker - - otherwise:: - - paster --plugin=ckan jobs -c /etc/ckan/default/ckan.ini worker - - or if you have CKAN version 2.6.x or less (and are therefore using ckanext-rq):: - - paster --plugin=ckanext-rq jobs -c /etc/ckan/default/ckan.ini worker - - Test it will load a CSV ok by submitting a `CSV in the web interface `_ - or in another shell:: +8. Run the worker:: - paster --plugin=ckanext-xloader xloader submit -c /etc/ckan/default/ckan.ini - - Clearly, running the worker on the command-line is only for testing - for - production services see: - - http://docs.ckan.org/en/ckan-2.7.0/maintaining/background-tasks.html#using-supervisor - - If you have CKAN version 2.6.x or less then you'll need to download - `supervisor-ckan-worker.conf `_ and adjust the ``command`` to reference - ckanext-rq. + ckan -c /etc/ckan/default/ckan.ini jobs worker --------------- @@ -304,7 +262,7 @@ in the directory up from your local ckan repo:: git clone https://github.com/ckan/ckanext-xloader.git cd ckanext-xloader - python setup.py develop + pip install -e . pip install -r requirements.txt pip install -r dev-requirements.txt @@ -346,35 +304,31 @@ command-line interface. e.g. :: - [2.9] ckan -c /etc/ckan/default/ckan.ini xloader submit - [pre-2.9] paster --plugin=ckanext-xloader xloader submit -c /etc/ckan/default/ckan.ini + ckan -c /etc/ckan/default/ckan.ini xloader submit For debugging you can try xloading it synchronously (which does the load directly, rather than asking the worker to do it) with the ``-s`` option:: - [2.9] ckan -c /etc/ckan/default/ckan.ini xloader submit -s - [pre-2.9] paster --plugin=ckanext-xloader xloader submit -s -c /etc/ckan/default/ckan.ini + ckan -c /etc/ckan/default/ckan.ini xloader submit -s See the status of jobs:: - [2.9] ckan -c /etc/ckan/default/ckan.ini xloader status - [pre-2.9] paster --plugin=ckanext-xloader xloader status -c /etc/ckan/default/development.ini + ckan -c /etc/ckan/default/ckan.ini xloader status Submit all datasets' resources to the DataStore:: - [2.9] ckan -c /etc/ckan/default/ckan.ini xloader submit all - [pre-2.9] paster --plugin=ckanext-xloader xloader submit all -c /etc/ckan/default/ckan.ini + ckan -c /etc/ckan/default/ckan.ini xloader submit all Re-submit all the resources already in the DataStore (Ignores any resources that have not been stored in DataStore e.g. because they are not tabular):: - [2.9] ckan -c /etc/ckan/default/ckan.ini xloader submit all-existing - [pre-2.9] paster --plugin=ckanext-xloader xloader submit all-existing -c /etc/ckan/default/ckan.ini + ckan -c /etc/ckan/default/ckan.ini xloader submit all-existing + **Full list of XLoader CLI commands**:: - [2.9] ckan -c /etc/ckan/default/ckan.ini xloader --help - [pre-2.9] paster --plugin=ckanext-xloader xloader --help + ckan -c /etc/ckan/default/ckan.ini xloader --help + Jobs and workers ---------------- @@ -387,8 +341,7 @@ Useful commands: Clear (delete) all outstanding jobs:: - CKAN 2.9, Python 3 ckan -c /etc/ckan/default/ckan.ini jobs clear [QUEUES] - CKAN <2.9, Python 2 paster --plugin=ckanext-xloader xloader jobs clear [QUEUES] -c /etc/ckan/default/development.ini + ckan -c /etc/ckan/default/ckan.ini jobs clear [QUEUES] If having trouble with the worker process, restarting it can help:: @@ -409,13 +362,6 @@ exist** Your DataStore permissions have not been set-up - see: -**When editing a package, all its existing resources get re-loaded by xloader** - -This behavior was documented in -`Issue 75 `_ and is related -to a bug in CKAN that is fixed in versions 2.6.9, 2.7.7, 2.8.4 -and 2.9.0+. - ----------------- Running the Tests ----------------- @@ -426,12 +372,8 @@ The first time, your test datastore database needs the trigger applied:: To run the tests, do:: - nosetests --nologcapture --with-pylons=test.ini - -To run the tests and produce a coverage report, first make sure you have -coverage installed in your virtualenv (``pip install coverage``) then run:: + pytest ckan-ini=test.ini ckanext/xloader/tests - nosetests --nologcapture --with-pylons=test.ini --with-coverage --cover-package=ckanext.xloader --cover-inclusive --cover-erase --cover-tests ---------------------------------- Releasing a New Version of XLoader diff --git a/setup.py b/setup.py index 7d7eea80..b8035fe8 100644 --- a/setup.py +++ b/setup.py @@ -38,12 +38,11 @@ # Pick your license as you wish (should match "license" above) 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)', - # Specify the Python versions you support here. In particular, ensure - # that you indicate whether you support Python 2, Python 3 or both. - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.6', + # Specify the Python versions you support here. 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], @@ -85,10 +84,6 @@ [babel.extractors] ckan = ckan.lib.extract:extract_ckan - [paste.paster_command] - xloader = ckanext.xloader.paster:xloaderCommand - migrate_types = ckanext.xloader.paster:MigrateTypesCommand - ''', # If you are changing from the default layout of your extension, you may From f8113b61dfee7d3bb1e43c44bbc51d5146b25b54 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Wed, 22 Feb 2023 11:36:30 -0300 Subject: [PATCH 05/14] Add csrf_token support --- ckanext/xloader/templates/xloader/resource_data.html | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ckanext/xloader/templates/xloader/resource_data.html b/ckanext/xloader/templates/xloader/resource_data.html index f533d1e2..a94ad631 100644 --- a/ckanext/xloader/templates/xloader/resource_data.html +++ b/ckanext/xloader/templates/xloader/resource_data.html @@ -9,6 +9,7 @@ {% block upload_ds_button %}
+ {{ h.csrf_input() if 'csrf_input' in h }} @@ -22,10 +23,7 @@ {% elif status.task_info and status.task_info.error %}
- {% if status.task_info.error is string %} - {# DataPusher < 0.0.3 #} - {{ _('Error:') }} {{ status.task_info.error }} - {% elif status.task_info.error is mapping %} + {% if status.task_info.error is mapping %} {{ _('Error:') }} {{ status.task_info.error.message }} {% for error_key, error_value in status.task_info.error.items() %} {% if error_key != "message" and error_value %} From b6fdbd28122b27c37aa3a90c5c67b1fd1f94d70b Mon Sep 17 00:00:00 2001 From: pdelboca Date: Wed, 22 Feb 2023 11:41:59 -0300 Subject: [PATCH 06/14] Fix typo on template directory --- ckanext/xloader/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/xloader/plugin.py b/ckanext/xloader/plugin.py index ef948376..42a19fa5 100644 --- a/ckanext/xloader/plugin.py +++ b/ckanext/xloader/plugin.py @@ -67,7 +67,7 @@ def get_blueprint(self): # IConfigurer def update_config(self, config): - toolkit.add_template_directory(config, 'templates/') + toolkit.add_template_directory(config, 'templates') # IConfigurable From 063c3ba1b2fd4dce520018a91ed6ab0a07472e71 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Wed, 22 Feb 2023 13:51:47 -0300 Subject: [PATCH 07/14] Add config_declaration.yaml file. --- README.rst | 73 +-------------- ckanext/xloader/config_declaration.yaml | 116 ++++++++++++++++++++++++ ckanext/xloader/plugin.py | 26 ++++++ 3 files changed, 143 insertions(+), 72 deletions(-) create mode 100644 ckanext/xloader/config_declaration.yaml diff --git a/README.rst b/README.rst index 81cfe72e..a17ac872 100644 --- a/README.rst +++ b/README.rst @@ -179,78 +179,7 @@ Config settings Configuration: -:: - - # The connection string for the jobs database used by XLoader. The - # default of an sqlite file is fine for development. For production use a - # Postgresql database. - ckanext.xloader.jobs_db.uri = sqlite:////tmp/xloader_jobs.db - - # The formats that are accepted. If the value of the resource.format is - # anything else then it won't be 'xloadered' to DataStore (and will therefore - # only be available to users in the form of the original download/link). - # Case insensitive. - # (optional, defaults are listed in plugin.py - DEFAULT_FORMATS). - ckanext.xloader.formats = csv application/csv xls application/vnd.ms-excel - - # The maximum size of files to load into DataStore. In bytes. Default is 1 GB. - ckanext.xloader.max_content_length = 1000000000 - - # By default, xloader will first try to add tabular data to the DataStore - # with a direct PostgreSQL COPY. This is relatively fast, but does not - # guess column types. If this fails, xloader falls back to a method more - # like DataPusher's behaviour. This has the advantage that the column types - # are guessed. However it is more error prone and far slower. - # To always skip the direct PostgreSQL COPY and use type guessing, set - # this option to True. - ckanext.xloader.use_type_guessing = False - - # Deprecated: use ckanext.xloader.use_type_guessing instead. - ckanext.xloader.just_load_with_messytables = False - - # Whether ambiguous dates should be parsed day first. Defaults to False. - # If set to True, dates like '01.02.2022' will be parsed as day = 01, - # month = 02. - # NB: isoformat dates like '2022-01-02' will be parsed as YYYY-MM-DD, and - # this option will not override that. - # See https://dateutil.readthedocs.io/en/stable/parser.html#dateutil.parser.parse - # for more details. - ckanext.xloader.parse_dates_dayfirst = False - - # Whether ambiguous dates should be parsed year first. Defaults to False. - # If set to True, dates like '01.02.03' will be parsed as year = 2001, - # month = 02, day = 03. See https://dateutil.readthedocs.io/en/stable/parser.html#dateutil.parser.parse - # for more details. - ckanext.xloader.parse_dates_yearfirst = False - - # The maximum time for the loading of a resource before it is aborted. - # Give an amount in seconds. Default is 60 minutes - ckanext.xloader.job_timeout = 3600 - - # Ignore the file hash when submitting to the DataStore, if set to True - # resources are always submitted (if their format matches), if set to - # False (default), resources are only submitted if their hash has changed. - ckanext.xloader.ignore_hash = False - - # When loading a file that is bigger than `max_content_length`, xloader can - # still try and load some of the file, which is useful to display a - # preview. Set this option to the desired number of lines/rows that it - # loads in this case. - # If the file-type is supported (CSV, TSV) an excerpt with the number of - # `max_excerpt_lines` lines will be submitted while the `max_content_length` - # is not exceeded. - # If set to 0 (default) files that exceed the `max_content_length` will - # not be loaded into the datastore. - ckanext.xloader.max_excerpt_lines = 100 - - # Requests verifies SSL certificates for HTTPS requests. Setting verify to - # False should only be enabled during local development or testing. Default - # to True. - ckanext.xloader.ssl_verify = True - - # Uses a specific API token for the xloader_submit action instead of the - # apikey of the site_user - ckanext.xloader.api_token = ckan-provided-api-token +See the extension's `config_declaration.yaml `_ file. ------------------------ diff --git a/ckanext/xloader/config_declaration.yaml b/ckanext/xloader/config_declaration.yaml new file mode 100644 index 00000000..97e9f5de --- /dev/null +++ b/ckanext/xloader/config_declaration.yaml @@ -0,0 +1,116 @@ +version: 1 +groups: + - annotation: ckanext-xloader settings + options: + - key: ckanext.xloader.jobs_db.uri + example: sqlite:////tmp/xloader_jobs.db + description: | + The connection string for the jobs database used by XLoader. The + default of an sqlite file is fine for development. For production use a + Postgresql database. + validators: not_missing + required: true + - key: ckanext.xloader.api_token + example: eyJ0eXAiOiJKV1QiLCJh.eyJqdGkiOiJ0M2VNUFlQWFg0VU.8QgV8em4RA + description: | + Uses a specific API token for the xloader_submit action instead of the + apikey of the site_user. Will be mandatory when dropping support for + CKAN 2.9. + required: false + - key: ckanext.xloader.formats + example: csv application/csv xls application/vnd.ms-excel + description: | + The formats that are accepted. If the value of the resource.format is + anything else then it won't be 'xloadered' to DataStore (and will therefore + only be available to users in the form of the original download/link). + Case insensitive. Defaults are listed in plugin.py. + required: false + - key: ckanext.xloader.max_content_length + default: 1000000000 + example: 100000 + description: | + The connection string for the jobs database used by XLoader. The + default of an sqlite file is fine for development. For production use a + Postgresql database. + type: int + required: false + - key: ckanext.xloader.use_type_guessing + default: False + example: False + description: | + By default, xloader will first try to add tabular data to the DataStore + with a direct PostgreSQL COPY. This is relatively fast, but does not + guess column types. If this fails, xloader falls back to a method more + like DataPusher's behaviour. This has the advantage that the column types + are guessed. However it is more error prone and far slower. + To always skip the direct PostgreSQL COPY and use type guessing, set + this option to True. + type: bool + required: false + legacy_key: ckanext.xloader.just_load_with_messytables + - key: ckanext.xloader.parse_dates_dayfirst + default: False + example: False + description: | + Whether ambiguous dates should be parsed day first. Defaults to False. + If set to True, dates like '01.02.2022' will be parsed as day = 01, + month = 02. + NB: isoformat dates like '2022-01-02' will be parsed as YYYY-MM-DD, and + this option will not override that. + See https://dateutil.readthedocs.io/en/stable/parser.html#dateutil.parser.parse + for more details. + type: bool + required: false + - key: ckanext.xloader.parse_dates_yearfirst + default: False + example: False + description: | + Whether ambiguous dates should be parsed year first. Defaults to False. + If set to True, dates like '01.02.03' will be parsed as year = 2001, + month = 02, day = 03. See https://dateutil.readthedocs.io/en/stable/parser.html#dateutil.parser.parse + for more details. + type: bool + required: false + - key: ckanext.xloader.job_timeout + default: 3600 + example: 3600 + description: | + The maximum time for the loading of a resource before it is aborted. + Give an amount in seconds. Default is 60 minutes + type: int + required: false + - key: ckanext.xloader.ignore_hash + default: False + example: False + description: | + Ignore the file hash when submitting to the DataStore, if set to True + resources are always submitted (if their format matches), if set to + False (default), resources are only submitted if their hash has changed. + type: bool + required: false + - key: ckanext.xloader.max_excerpt_lines + default: 0 + example: 100 + description: | + When loading a file that is bigger than `max_content_length`, xloader can + still try and load some of the file, which is useful to display a + preview. Set this option to the desired number of lines/rows that it + loads in this case. + If the file-type is supported (CSV, TSV) an excerpt with the number of + `max_excerpt_lines` lines will be submitted while the `max_content_length` + is not exceeded. + If set to 0 (default) files that exceed the `max_content_length` will + not be loaded into the datastore. + type: int + required: false + - key: ckanext.xloader.ssl_verify + default: True + example: True + description: | + Requests verifies SSL certificates for HTTPS requests. Setting verify to + False should only be enabled during local development or testing. Default + to True. + type: bool + required: false + + diff --git a/ckanext/xloader/plugin.py b/ckanext/xloader/plugin.py index 42a19fa5..a12205fc 100644 --- a/ckanext/xloader/plugin.py +++ b/ckanext/xloader/plugin.py @@ -8,6 +8,12 @@ from . import action, auth, helpers as xloader_helpers, utils from .loader import fulltext_function_exists, get_write_engine +IConfigDeclaration = None +if toolkit.check_ckan_version("2.10"): + import yaml + import os + IConfigDeclaration = plugins.IConfigDeclaration + log = logging.getLogger(__name__) @@ -51,6 +57,24 @@ class xloaderPlugin(plugins.SingletonPlugin): plugins.implements(plugins.IResourceController, inherit=True) plugins.implements(plugins.IClick) plugins.implements(plugins.IBlueprint) + if IConfigDeclaration: + plugins.implements(IConfigDeclaration) + + # IConfigDeclaration + def declare_config_options(self, declaration, key): + """ckanext-xloader config declaration. + + Migrate to blanket's decorator when dropping support + for CKAN 2.9. + """ + dir_path = os.path.dirname(os.path.realpath(__file__)) + file_path = dir_path + "/config_declaration.yaml" + with open(file_path, 'r') as stream: + try: + result = yaml.safe_load(stream) + except yaml.YAMLError as e: + print(e) + declaration.load_dict(result) # IClick def get_commands(self): @@ -69,6 +93,8 @@ def get_blueprint(self): def update_config(self, config): toolkit.add_template_directory(config, 'templates') + + # IConfigurable def configure(self, config_): From 01b9f44c9d8bded72cc10561bef12bd329f24370 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Wed, 22 Feb 2023 13:52:30 -0300 Subject: [PATCH 08/14] Do not try and catch --- ckanext/xloader/plugin.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ckanext/xloader/plugin.py b/ckanext/xloader/plugin.py index a12205fc..546a745e 100644 --- a/ckanext/xloader/plugin.py +++ b/ckanext/xloader/plugin.py @@ -70,10 +70,7 @@ def declare_config_options(self, declaration, key): dir_path = os.path.dirname(os.path.realpath(__file__)) file_path = dir_path + "/config_declaration.yaml" with open(file_path, 'r') as stream: - try: - result = yaml.safe_load(stream) - except yaml.YAMLError as e: - print(e) + result = yaml.safe_load(stream) declaration.load_dict(result) # IClick From 7a658adefd1dbb2beff0ff8dfa38ee32e99daa87 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Wed, 22 Feb 2023 13:58:27 -0300 Subject: [PATCH 09/14] Increase version number. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b8035fe8..dc261c74 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ # Versions should comply with PEP440. For a discussion on single-sourcing # the version across setup.py and the project code, see # http://packaging.python.org/en/latest/tutorial.html#version - version='0.12.2', + version='0.13.0', description='Express Loader - quickly load data into CKAN DataStore''', long_description=long_description, From c48b2821c956f813a0714ff4c27ab9f85801f69a Mon Sep 17 00:00:00 2001 From: pdelboca Date: Wed, 22 Feb 2023 13:59:04 -0300 Subject: [PATCH 10/14] Fix lint --- ckanext/xloader/plugin.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ckanext/xloader/plugin.py b/ckanext/xloader/plugin.py index 546a745e..941c0184 100644 --- a/ckanext/xloader/plugin.py +++ b/ckanext/xloader/plugin.py @@ -90,8 +90,6 @@ def get_blueprint(self): def update_config(self, config): toolkit.add_template_directory(config, 'templates') - - # IConfigurable def configure(self, config_): From 07e2a4f09df0dd91460491c6ebc6ac657c94c8b2 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Thu, 2 Mar 2023 08:39:56 -0300 Subject: [PATCH 11/14] Use blanket config_declarations --- ckanext/xloader/plugin.py | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/ckanext/xloader/plugin.py b/ckanext/xloader/plugin.py index 941c0184..dbde8ed5 100644 --- a/ckanext/xloader/plugin.py +++ b/ckanext/xloader/plugin.py @@ -8,11 +8,13 @@ from . import action, auth, helpers as xloader_helpers, utils from .loader import fulltext_function_exists, get_write_engine -IConfigDeclaration = None -if toolkit.check_ckan_version("2.10"): - import yaml - import os - IConfigDeclaration = plugins.IConfigDeclaration +try: + config_declarations = toolkit.blanket.config_declarations +except AttributeError: + # CKAN 2.9 does not have config_declarations. + # Remove when dropping support. + def config_declarations(cls): + return cls log = logging.getLogger(__name__) @@ -47,6 +49,7 @@ def is_it_an_xloader_format(cls, format_): return format_.lower() in cls._formats +@config_declarations class xloaderPlugin(plugins.SingletonPlugin): plugins.implements(plugins.IConfigurer) plugins.implements(plugins.IConfigurable) @@ -57,21 +60,6 @@ class xloaderPlugin(plugins.SingletonPlugin): plugins.implements(plugins.IResourceController, inherit=True) plugins.implements(plugins.IClick) plugins.implements(plugins.IBlueprint) - if IConfigDeclaration: - plugins.implements(IConfigDeclaration) - - # IConfigDeclaration - def declare_config_options(self, declaration, key): - """ckanext-xloader config declaration. - - Migrate to blanket's decorator when dropping support - for CKAN 2.9. - """ - dir_path = os.path.dirname(os.path.realpath(__file__)) - file_path = dir_path + "/config_declaration.yaml" - with open(file_path, 'r') as stream: - result = yaml.safe_load(stream) - declaration.load_dict(result) # IClick def get_commands(self): From 0bf0cc9c107eb1bb4580c96d01ade2ee10ed3361 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Thu, 2 Mar 2023 08:41:09 -0300 Subject: [PATCH 12/14] Use SQLite as default in config_declaration --- ckanext/xloader/config_declaration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/xloader/config_declaration.yaml b/ckanext/xloader/config_declaration.yaml index 97e9f5de..fac48ff9 100644 --- a/ckanext/xloader/config_declaration.yaml +++ b/ckanext/xloader/config_declaration.yaml @@ -3,7 +3,7 @@ groups: - annotation: ckanext-xloader settings options: - key: ckanext.xloader.jobs_db.uri - example: sqlite:////tmp/xloader_jobs.db + default: sqlite:////tmp/xloader_jobs.db description: | The connection string for the jobs database used by XLoader. The default of an sqlite file is fine for development. For production use a From 8bdfbe473e222503e3b1c691664033d63379a261 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Thu, 2 Mar 2023 08:42:13 -0300 Subject: [PATCH 13/14] Use better syntax for readability --- ckanext/xloader/config_declaration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/xloader/config_declaration.yaml b/ckanext/xloader/config_declaration.yaml index fac48ff9..b31f12e2 100644 --- a/ckanext/xloader/config_declaration.yaml +++ b/ckanext/xloader/config_declaration.yaml @@ -26,7 +26,7 @@ groups: Case insensitive. Defaults are listed in plugin.py. required: false - key: ckanext.xloader.max_content_length - default: 1000000000 + default: 1_000_000_000 example: 100000 description: | The connection string for the jobs database used by XLoader. The From 2f92909bc0fd5b60eb577b6021e1eaeded267b21 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Thu, 2 Mar 2023 08:43:42 -0300 Subject: [PATCH 14/14] Updating to major version since it is dropping support for old CKAN 2.8 API --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index dc261c74..3bead14d 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ # Versions should comply with PEP440. For a discussion on single-sourcing # the version across setup.py and the project code, see # http://packaging.python.org/en/latest/tutorial.html#version - version='0.13.0', + version='1.0.0', description='Express Loader - quickly load data into CKAN DataStore''', long_description=long_description,