Adding "Missing Value" issue to the log when Geometry is empty.

digital_land/phase/harmonise.py

@@ -59,6 +59,17 @@ def process(self, stream):
                 if value and ":" not in value:
                     o[typology] = "%s:%s" % (block["dataset"], value)
 
+            # ensure geometry field is not empty
+            for typology in ["geography"]:
+                # logging error when both geometry & point are empty
+                # TO-DO: will replace this code once we get mandatory list from Specification
+                for field in row:
+                    if field in ["geometry", "point"]:
+                        if (
+                            row.get("geometry") == "" or row.get("geometry") is None
+                        ) and (row.get("point") == "" or row.get("point") is None):
+                            self.issues.log_issue(field, "missing value", "")
+
             # migrate wikipedia URLs to a reference compatible with dbpedia CURIEs with a wikipedia-en prefix
             if row.get("wikipedia", "").startswith("http"):
                 self.issues.log_issue(
@@ -68,4 +79,5 @@ def process(self, stream):
                     "https://en.wikipedia.org/wiki/", ""
                 )
             block["row"] = o
+
             yield block

digital_land/phase/map.py

@@ -33,17 +33,17 @@ def headers(self, fieldnames):
         matched = []
         for header in sorted(fieldnames):
             fieldname = normalise(header)
-
             for pattern, value in self.columns.items():
                 if fieldname == pattern:
                     matched.append(value)
                     headers[header] = value
-
             # stop if we found a match
+
+        for header in sorted(fieldnames):
             if header in headers:
                 continue
-
-            if fieldname in self.normalised_fieldnames:
+            fieldname = normalise(header)
+            if fieldname not in matched and fieldname in self.normalised_fieldnames:
                 headers[header] = self.normalised_fieldnames[fieldname]
                 continue
 
@@ -74,14 +74,7 @@ def process(self, stream):
                 if headers[header] == "IGNORE":
                     continue
 
-                value = row.get(header)
-
-                if value is not None and value != "":
-                    o[headers[header]] = value
-
-            for header in self.normalised_fieldnames.values():
-                if header not in o:
-                    o[header] = ""
+                o[headers[header]] = row.get(header)
 
             block["row"] = o
 

tests/data/specification/field.csv

@@ -24,3 +24,4 @@ entity,,string,1,,,,
 reference-entity,,string,1,,,,
 field,,string,1,,,,
 value,,string,1,,,,
+geometry,Geometry,wkt,1,,,,

tests/unit/phase/test_map.py

@@ -105,12 +105,9 @@ def test_map_column_names_with_underscores_when_column_not_in_specification():
 
     m = MapPhase(fieldnames, columns)
     output = TestPipeline(
-        m, "Organisation_Label,PermissionDate\r\ncol-1-val,col-2-val\r\n"
-    )
-    assert (
-        output
-        == "Organisation_Label,PermissionDate,SiteNameAddress\r\ncol-1-val,col-2-val,\r\n"
+        m, "Organisation_Label,PermissionDate,test\r\ncol-1-val,col-2-val,\r\n"
     )
+    assert output == "Organisation_Label,PermissionDate\r\ncol-1-val,col-2-val\r\n"
 
 
 def test_map_column_names_with_underscores_when_column_in_specification():
@@ -128,10 +125,9 @@ def test_map_column_names_with_underscores_when_column_in_specification():
 
     m = MapPhase(fieldnames, columns)
     output = TestPipeline(
-        m, "Organisation_Label,end_date,SiteNameAddress\r\ncol-1-val,col-2-val\r\n"
+        m, "Organisation_Label,end_date,SiteNameAddress\r\ncol-1-val,col-2-val,\r\n"
     )
-
     assert (
         output
-        == "Organisation-Label,Organisation_Label,SiteNameAddress,end-date,end_date\r\ncol-1-val,,,col-2-val,\r\n"
+        == "Organisation-Label,SiteNameAddress,end-date\r\ncol-1-val,,col-2-val\r\n"
     )

tests/unit/test_harmonise.py

@@ -34,3 +34,24 @@ def test_harmonise():
     assert output[0]["row"] == {"field-integer": "123"}, "pass through valid data"
     assert output[1]["row"] == {"field-integer": "321"}, "whitespace trimmed"
     assert output[2]["row"] == {"field-integer": ""}, "remove bad data"
+
+
+def test_harmonise_geometry():
+    specification = Specification("tests/data/specification")
+    issues = IssueLog()
+
+    h = HarmonisePhase(specification=specification, issues=issues)
+    reader = FakeDictReader(
+        [
+            {"organisation": "test_org"},
+        ]
+    )
+    output = list(h.process(reader))
+
+    assert len(output) == 1
+
+    # It should have an issue logged for the empty "geometry" field
+    for issue in issues.rows:
+        assert issue["field"] in ["geometry", "point"]
+        assert issue["issue-type"] == "missing value"
+        assert issue["value"] == ""

tests/unit/test_specification.py

@@ -94,6 +94,7 @@ def test_field_names():
             "field",
             "reference-entity",
             "fact",
+            "geometry",
         ]
     )
 
@@ -133,6 +134,7 @@ def test_current_fieldnames():
             "line-number",
             "resource",
             "value",
+            "geometry",
         ]
     )