Adjustments for successful OpenSearch indexing

Why these changes are being introduced:

After an MITAardvark transformation was mostly completed, a couple of small bugs were discovered when attempting to index the
transformed TIMDEX records into OpenSearch.  This commit adddresses those bugs, both full fixes and workarounds, to allow
continued pipeline testing and improvements to the MITAardvark transformation.

How this addresses that need:
* adds kind property to Dates and Identifiers
* uses locations.value instead of locations.geodata

Side effects of this change:
* Anticipating successful indexing attempts from transformed records

Relevant ticket(s):
* https://mitlibraries.atlassian.net/browse/GDT-54

tests/fixtures/aardvark_records.jsonl

@@ -1,2 +1,2 @@
-{"dct_accessRights_s": "Access rights", "dct_references_s": "", "dct_title_s": "Test title 1", "gbl_mdModified_dt": "", "gbl_mdVersion_s": "", "gbl_resourceClass_sm": "", "id": "mit:123", "locn_geometry": ""}
-{"dct_accessRights_s": "Access rights", "dct_references_s": "", "dct_title_s": "Test title 2", "gbl_mdModified_dt": "", "gbl_mdVersion_s": "", "gbl_resourceClass_sm": "", "id": "ogm:456", "locn_geometry": ""}
+{"dct_accessRights_s": "Access rights", "dct_references_s": "", "dct_title_s": "Test title 1", "gbl_mdModified_dt": "", "gbl_mdVersion_s": "", "gbl_resourceClass_sm": "", "id": "mit:123", "locn_geometry": "ENVELOPE(-111.1, -104.0, 45.0, 40.9)"}
+{"dct_accessRights_s": "Access rights", "dct_references_s": "", "dct_title_s": "Test title 2", "gbl_mdModified_dt": "", "gbl_mdVersion_s": "", "gbl_resourceClass_sm": "", "id": "ogm:456", "locn_geometry": "ENVELOPE(-111.1, -104.0, 45.0, 40.9)"}

tests/sources/json/test_aardvark.py

@@ -24,6 +24,11 @@ def test_aardvark_transform_returns_timdex_record(aardvark_records):
         citation="Test title 1. Geospatial data. https://example.com/gismit:123",
         content_type=["Geospatial data"],
         rights=[timdex.Rights(description="Access rights", kind="Access")],
+        locations=[
+            timdex.Location(
+                value="ENVELOPE(-111.1, -104.0, 45.0, 40.9)", kind="Geometry"
+            )
+        ],
     )
 
 
@@ -75,6 +80,7 @@ def test_aardvark_get_dates_success(aardvark_record_all_fields):
         timdex.Date(kind="Coverage", value="1945"),
         timdex.Date(kind="Coverage", value="1946"),
         timdex.Date(
+            kind="Coverage",
             range=timdex.Date_Range(gte="1943", lte="1946"),
         ),
     ]
@@ -97,7 +103,7 @@ def test_parse_solr_date_range_invalid_date_range_string_raises_error():
 
 def test_aardvark_get_identifiers_success(aardvark_record_all_fields):
     assert MITAardvark.get_identifiers(next(aardvark_record_all_fields)) == [
-        timdex.Identifier(value="abc123")
+        timdex.Identifier(value="abc123", kind="Not specified")
     ]
 
 
@@ -131,8 +137,10 @@ def test_aardvark_get_links_logs_warning_for_invalid_json(caplog):
 
 def test_aardvark_get_locations_success(aardvark_record_all_fields):
     assert MITAardvark.get_locations(next(aardvark_record_all_fields), "123") == [
-        timdex.Location(kind="Bounding Box", geodata=[-111.1, -104.0, 45.0, 40.9]),
-        timdex.Location(kind="Geometry", geodata=[-111.1, -104.0, 45.0, 40.9]),
+        timdex.Location(
+            kind="Bounding Box", value="ENVELOPE(-111.1, -104.0, 45.0, 40.9)"
+        ),
+        timdex.Location(kind="Geometry", value="ENVELOPE(-111.1, -104.0, 45.0, 40.9)"),
     ]
 
 

transmogrifier/sources/json/aardvark.py

@@ -3,7 +3,6 @@
 import re
 
 import transmogrifier.models as timdex
-from transmogrifier.helpers import parse_geodata_string
 from transmogrifier.sources.transformer import JSON, JSONTransformer
 
 logger = logging.getLogger(__name__)
@@ -228,9 +227,10 @@ def _range_dates(
             )
             range_dates.append(
                 timdex.Date(
+                    kind="Coverage",
                     range=timdex.Date_Range(
                         gte=date_range_values[0], lte=date_range_values[1]
-                    )
+                    ),
                 )
             )
         return range_dates
@@ -262,7 +262,7 @@ def parse_solr_date_range_string(
     def get_identifiers(source_record: dict) -> list[timdex.Identifier]:
         """Get values from source record for TIMDEX identifiers field."""
         return [
-            timdex.Identifier(value=identifier_value)
+            timdex.Identifier(value=identifier_value, kind="Not specified")
             for identifier_value in source_record.get("dct_identifier_sm", [])
         ]
 
@@ -303,15 +303,27 @@ def get_locations(
             if aardvark_location_field not in source_record:
                 continue
             try:
-                if geodata_points := parse_geodata_string(
-                    source_record[aardvark_location_field], source_record_id
-                ):
-                    locations.append(
-                        timdex.Location(
-                            geodata=geodata_points,
-                            kind=kind_value,
-                        )
+                # NOTE: the following is a temporary workaround until the locations
+                #   mapping is updated to support "geopoint" and "geoshape".
+
+                # previous logic for coordinate extraction to locations.geodata
+                # if geodata_points := parse_geodata_string(
+                #     source_record[aardvark_location_field], source_record_id
+                # ):
+                #     locations.append(
+                #         timdex.Location(
+                #             geodata=geodata_points,
+                #             kind=kind_value,
+                #         )
+                #     )
+
+                # temporary workaround to geoemtry as locations.value string
+                locations.append(
+                    timdex.Location(
+                        value=source_record[aardvark_location_field],
+                        kind=kind_value,
                     )
+                )
             except ValueError as exception:
                 logger.warning(exception)
         return locations