From cf04a5c5c38443f3d98e0e7b8a4ed0ceede90aa0 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Mon, 17 Apr 2023 12:50:09 +1000 Subject: [PATCH] [QOLDEV-347] fix validation errors on empty strings, #182 - replace empty strings with None if they have types that will choke on empty string --- ckanext/xloader/loader.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ckanext/xloader/loader.py b/ckanext/xloader/loader.py index afc3c980..75bddf51 100644 --- a/ckanext/xloader/loader.py +++ b/ckanext/xloader/loader.py @@ -318,9 +318,16 @@ def row_iterator(): logger.info('Copying to database...') count = 0 + # Some types cannot be stored as empty strings and must be converted to None, + # https://github.com/ckan/ckanext-xloader/issues/182 + non_empty_types = ['timestamp', 'numeric'] for i, records in enumerate(chunky(result, 250)): count += len(records) logger.info('Saving chunk {number}'.format(number=i)) + for row in records: + for column_index, column_name in enumerate(row): + if headers_dicts[column_index]['type'] in non_empty_types and row[column_name] == '': + row[column_name] = None send_resource_to_datastore(resource_id, headers_dicts, records) logger.info('...copying done')