populate item with processing datetimes (#63)

* populate item with processing datetimes * use community landsat schema
stactools-packages · Nov 9, 2023 · f568472 · f568472
1 parent 73e518b
commit f568472
Show file tree

Hide file tree

Showing 18 changed files with 277 additions and 237 deletions.
diff --git a/CHANGELOG.MD b/CHANGELOG.MD
@@ -6,6 +6,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [Unreleased]
 
+### Added
+
+- Populate item with processing datetime field `landsat:product_generated` ([#63](https://github.com/stactools-packages/landsat/pull/63))
+
+### Changed
+
+- Replace use of externally-hosted Landsat Extension <https://landsat.usgs.gov/stac/landsat-extension/v1.1.1/schema.json> with community Landsat Extension <https://stac-extensions.github.io/landsat/v2.0.0/schema.json> ([#63](https://github.com/stactools-packages/landsat/pull/63))
+
 ## [0.4.1] - 2023-08-20
 
 ### Added

diff --git a/README.md b/README.md
@@ -15,7 +15,7 @@
 - STAC extensions used:
   - [classification](https://github.com/stac-extensions/classification)
   - [eo](https://github.com/stac-extensions/eo)
-  - [landsat](https://landsat.usgs.gov/stac/landsat-extension/v1.1.1/schema.json)
+  - [landsat](https://stac-extensions.github.io/landsat/v2.0.0/schema.json)
   - [proj](https://github.com/stac-extensions/projection/)
   - [raster](https://github.com/stac-extensions/raster)
   - [scientific](https://github.com/stac-extensions/scientific)

diff --git a/examples/landsat-c2-l1/LM01_L1GS_001010_19720908_02_T2/LM01_L1GS_001010_19720908_02_T2.json b/examples/landsat-c2-l1/LM01_L1GS_001010_19720908_02_T2/LM01_L1GS_001010_19720908_02_T2.json
@@ -509,7 +509,7 @@
     "https://stac-extensions.github.io/eo/v1.0.0/schema.json",
     "https://stac-extensions.github.io/view/v1.0.0/schema.json",
     "https://stac-extensions.github.io/projection/v1.0.0/schema.json",
-    "https://landsat.usgs.gov/stac/landsat-extension/v1.1.1/schema.json",
+    "https://stac-extensions.github.io/landsat/v2.0.0/schema.json",
     "https://stac-extensions.github.io/classification/v1.0.0/schema.json",
     "https://stac-extensions.github.io/scientific/v1.0.0/schema.json"
   ],

diff --git a/examples/landsat-c2-l1/LM02_L1GS_001004_19750411_02_T2/LM02_L1GS_001004_19750411_02_T2.json b/examples/landsat-c2-l1/LM02_L1GS_001004_19750411_02_T2/LM02_L1GS_001004_19750411_02_T2.json
@@ -509,7 +509,7 @@
     "https://stac-extensions.github.io/eo/v1.0.0/schema.json",
     "https://stac-extensions.github.io/view/v1.0.0/schema.json",
     "https://stac-extensions.github.io/projection/v1.0.0/schema.json",
-    "https://landsat.usgs.gov/stac/landsat-extension/v1.1.1/schema.json",
+    "https://stac-extensions.github.io/landsat/v2.0.0/schema.json",
     "https://stac-extensions.github.io/classification/v1.0.0/schema.json",
     "https://stac-extensions.github.io/scientific/v1.0.0/schema.json"
   ],

diff --git a/examples/landsat-c2-l1/LM03_L1GS_001001_19780510_02_T2/LM03_L1GS_001001_19780510_02_T2.json b/examples/landsat-c2-l1/LM03_L1GS_001001_19780510_02_T2/LM03_L1GS_001001_19780510_02_T2.json
@@ -509,7 +509,7 @@
     "https://stac-extensions.github.io/eo/v1.0.0/schema.json",
     "https://stac-extensions.github.io/view/v1.0.0/schema.json",
     "https://stac-extensions.github.io/projection/v1.0.0/schema.json",
-    "https://landsat.usgs.gov/stac/landsat-extension/v1.1.1/schema.json",
+    "https://stac-extensions.github.io/landsat/v2.0.0/schema.json",
     "https://stac-extensions.github.io/classification/v1.0.0/schema.json",
     "https://stac-extensions.github.io/scientific/v1.0.0/schema.json"
   ],

diff --git a/examples/landsat-c2-l1/LM04_L1GS_001001_19830527_02_T2/LM04_L1GS_001001_19830527_02_T2.json b/examples/landsat-c2-l1/LM04_L1GS_001001_19830527_02_T2/LM04_L1GS_001001_19830527_02_T2.json
@@ -509,7 +509,7 @@
     "https://stac-extensions.github.io/eo/v1.0.0/schema.json",
     "https://stac-extensions.github.io/view/v1.0.0/schema.json",
     "https://stac-extensions.github.io/projection/v1.0.0/schema.json",
-    "https://landsat.usgs.gov/stac/landsat-extension/v1.1.1/schema.json",
+    "https://stac-extensions.github.io/landsat/v2.0.0/schema.json",
     "https://stac-extensions.github.io/classification/v1.0.0/schema.json",
     "https://stac-extensions.github.io/scientific/v1.0.0/schema.json"
   ],

diff --git a/examples/landsat-c2-l1/LM05_L1GS_001001_19850524_02_T2/LM05_L1GS_001001_19850524_02_T2.json b/examples/landsat-c2-l1/LM05_L1GS_001001_19850524_02_T2/LM05_L1GS_001001_19850524_02_T2.json
@@ -509,7 +509,7 @@
     "https://stac-extensions.github.io/eo/v1.0.0/schema.json",
     "https://stac-extensions.github.io/view/v1.0.0/schema.json",
     "https://stac-extensions.github.io/projection/v1.0.0/schema.json",
-    "https://landsat.usgs.gov/stac/landsat-extension/v1.1.1/schema.json",
+    "https://stac-extensions.github.io/landsat/v2.0.0/schema.json",
     "https://stac-extensions.github.io/classification/v1.0.0/schema.json",
     "https://stac-extensions.github.io/scientific/v1.0.0/schema.json"
   ],

diff --git a/examples/landsat-c2-l2/LC08_L2SP_047027_20201204_02_T1/LC08_L2SP_047027_20201204_02_T1.json b/examples/landsat-c2-l2/LC08_L2SP_047027_20201204_02_T1/LC08_L2SP_047027_20201204_02_T1.json
@@ -1115,7 +1115,7 @@
     "https://stac-extensions.github.io/eo/v1.0.0/schema.json",
     "https://stac-extensions.github.io/view/v1.0.0/schema.json",
     "https://stac-extensions.github.io/projection/v1.0.0/schema.json",
-    "https://landsat.usgs.gov/stac/landsat-extension/v1.1.1/schema.json",
+    "https://stac-extensions.github.io/landsat/v2.0.0/schema.json",
     "https://stac-extensions.github.io/classification/v1.0.0/schema.json",
     "https://stac-extensions.github.io/scientific/v1.0.0/schema.json"
   ],

diff --git a/examples/landsat-c2-l2/LC09_L2SP_010065_20220129_02_T1/LC09_L2SP_010065_20220129_02_T1.json b/examples/landsat-c2-l2/LC09_L2SP_010065_20220129_02_T1/LC09_L2SP_010065_20220129_02_T1.json
@@ -1115,7 +1115,7 @@
     "https://stac-extensions.github.io/eo/v1.0.0/schema.json",
     "https://stac-extensions.github.io/view/v1.0.0/schema.json",
     "https://stac-extensions.github.io/projection/v1.0.0/schema.json",
-    "https://landsat.usgs.gov/stac/landsat-extension/v1.1.1/schema.json",
+    "https://stac-extensions.github.io/landsat/v2.0.0/schema.json",
     "https://stac-extensions.github.io/classification/v1.0.0/schema.json",
     "https://stac-extensions.github.io/scientific/v1.0.0/schema.json"
   ],

diff --git a/examples/landsat-c2-l2/LE07_L2SP_021030_20100109_02_T1/LE07_L2SP_021030_20100109_02_T1.json b/examples/landsat-c2-l2/LE07_L2SP_021030_20100109_02_T1/LE07_L2SP_021030_20100109_02_T1.json
@@ -1066,7 +1066,7 @@
     "https://stac-extensions.github.io/eo/v1.0.0/schema.json",
     "https://stac-extensions.github.io/view/v1.0.0/schema.json",
     "https://stac-extensions.github.io/projection/v1.0.0/schema.json",
-    "https://landsat.usgs.gov/stac/landsat-extension/v1.1.1/schema.json",
+    "https://stac-extensions.github.io/landsat/v2.0.0/schema.json",
     "https://stac-extensions.github.io/classification/v1.0.0/schema.json",
     "https://stac-extensions.github.io/scientific/v1.0.0/schema.json"
   ],

diff --git a/examples/landsat-c2-l2/LT04_L2SP_002026_19830110_02_T1/LT04_L2SP_002026_19830110_02_T1.json b/examples/landsat-c2-l2/LT04_L2SP_002026_19830110_02_T1/LT04_L2SP_002026_19830110_02_T1.json
@@ -1048,7 +1048,7 @@
     "https://stac-extensions.github.io/eo/v1.0.0/schema.json",
     "https://stac-extensions.github.io/view/v1.0.0/schema.json",
     "https://stac-extensions.github.io/projection/v1.0.0/schema.json",
-    "https://landsat.usgs.gov/stac/landsat-extension/v1.1.1/schema.json",
+    "https://stac-extensions.github.io/landsat/v2.0.0/schema.json",
     "https://stac-extensions.github.io/classification/v1.0.0/schema.json",
     "https://stac-extensions.github.io/scientific/v1.0.0/schema.json"
   ],

diff --git a/examples/landsat-c2-l2/LT05_L2SP_010067_19860424_02_T2/LT05_L2SP_010067_19860424_02_T2.json b/examples/landsat-c2-l2/LT05_L2SP_010067_19860424_02_T2/LT05_L2SP_010067_19860424_02_T2.json
@@ -1048,7 +1048,7 @@
     "https://stac-extensions.github.io/eo/v1.0.0/schema.json",
     "https://stac-extensions.github.io/view/v1.0.0/schema.json",
     "https://stac-extensions.github.io/projection/v1.0.0/schema.json",
-    "https://landsat.usgs.gov/stac/landsat-extension/v1.1.1/schema.json",
+    "https://stac-extensions.github.io/landsat/v2.0.0/schema.json",
     "https://stac-extensions.github.io/classification/v1.0.0/schema.json",
     "https://stac-extensions.github.io/scientific/v1.0.0/schema.json"
   ],

diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -7,4 +7,7 @@ pre-commit
 pytest
 pytest-cov
 vcrpy
-types-python-dateutil
+types-python-dateutil
+# AttributeError: module 'urllib3.connectionpool' has no attribute 'VerifiedHTTPSConnection'
+# if urllib3 2.x is installed on python 3.9
+urllib3<2 
diff --git a/src/stactools/landsat/constants.py b/src/stactools/landsat/constants.py
@@ -10,8 +10,9 @@ class Sensor(Enum):
 
 
 LANDSAT_EXTENSION_SCHEMA = (
-    "https://landsat.usgs.gov/stac/landsat-extension/v1.1.1/schema.json"
+    "https://stac-extensions.github.io/landsat/v2.0.0/schema.json"
 )
+
 CLASSIFICATION_EXTENSION_SCHEMA = (
     "https://stac-extensions.github.io/classification/v1.0.0/schema.json"  # noqa
 )

diff --git a/src/stactools/landsat/mtl_metadata.py b/src/stactools/landsat/mtl_metadata.py
@@ -5,7 +5,7 @@
 from lxml import etree
 from lxml.etree import _Element as lxmlElement
 from pyproj import Geod
-from pystac.utils import map_opt, str_to_datetime
+from pystac.utils import datetime_to_str, map_opt, str_to_datetime
 from stactools.core.io import ReadHrefModifier, read_text
 from stactools.core.io.xml import XmlElement
 from stactools.core.projection import transform_from_bbox
@@ -273,9 +273,15 @@ def wrs_path(self) -> str:
     def wrs_row(self) -> str:
         return self._get_text("IMAGE_ATTRIBUTES/WRS_ROW").zfill(3)
 
+    @property
+    def product_generated(self) -> Optional[datetime]:
+        return self._product_generated(
+            "LEVEL2_PROCESSING_RECORD"
+        ) or self._product_generated("LEVEL1_PROCESSING_RECORD")
+
     @property
     def landsat_metadata(self) -> Dict[str, Any]:
-        return {
+        d = {
             "landsat:cloud_cover_land": self._get_float(
                 "IMAGE_ATTRIBUTES/CLOUD_COVER_LAND"
             ),
@@ -292,6 +298,12 @@ def landsat_metadata(self) -> Dict[str, Any]:
             "landsat:scene_id": self.scene_id,
         }
 
+        pg = self.product_generated
+        if pg:
+            d["landsat:product_generated"] = datetime_to_str(pg)
+
+        return d
+
     @property
     def level1_radiance(self) -> Dict[str, Dict[str, Optional[float]]]:
         """Gets the scale (mult) and offset (add) values for generating TOA
@@ -317,6 +329,13 @@ def level1_radiance(self) -> Dict[str, Dict[str, Optional[float]]]:
                 mult_add[band]["add"] = self._float_or_none(value)
         return mult_add
 
+    def _product_generated(self, path: str) -> Optional[datetime]:
+        v = self._root.find_text(f"{path}/DATE_PRODUCT_GENERATED")
+        if v:
+            return str_to_datetime(v)
+        else:
+            return None
+
     @classmethod
     def from_file(
         cls,