Merge pull request #35 from eea/develop

Release of CLMS STAC generator v0.1.0

.gitignore

@@ -100,3 +100,6 @@ target/
 
 # Pycopy
 __pycache__/
+
+# macOS
+.DS_Store

create_clc_collection.py

@@ -0,0 +1,15 @@
+import logging
+
+from scripts.clc.collection import create_collection, populate_collection
+
+LOGGER = logging.getLogger(__name__)
+
+
+def main():
+    logging.basicConfig(filename="create_clc_collection.log")
+    collection = create_collection()
+    populate_collection(collection, data_root="../CLC_100m")
+
+
+if __name__ == "__main__":
+    main()

create_euhydro_collection.py

@@ -0,0 +1,9 @@
+import logging
+
+from scripts.euhydro.collection import create_euhydro_collection
+
+LOGGER = logging.getLogger(__name__)
+
+if __name__ == "__main__":
+    logging.basicConfig(filename="create_euhydro_collection.log")
+    create_euhydro_collection("<euhydro-root>")

create_n2k_collection.py

@@ -0,0 +1,9 @@
+import logging
+
+from scripts.n2k.collection import create_n2k_collection
+
+LOGGER = logging.getLogger(__name__)
+
+if __name__ == "__main__":
+    logging.basicConfig(filename="create_n2k_collection.log")
+    create_n2k_collection("<path-to-n2k-root-directory>")

create_uabh_collection.py

@@ -0,0 +1,18 @@
+import logging
+from glob import glob
+
+from scripts.uabh.collection import create_uabh_collection, get_stac_validator
+from scripts.uabh.constants import COLLECTION_ID, STAC_DIR, WORKING_DIR
+
+LOGGER = logging.getLogger(__name__)
+
+
+def main():
+    logging.basicConfig(filename="create_uabh_collection.log")
+    item_list = glob(f"{WORKING_DIR}/{STAC_DIR}/{COLLECTION_ID}/**/*.json")
+    validator = get_stac_validator("schema/products/uabh.json")
+    create_uabh_collection(item_list, validator)
+
+
+if __name__ == "__main__":
+    main()

create_uabh_items.py

@@ -0,0 +1,18 @@
+import logging
+from glob import glob
+
+from scripts.uabh.item import create_uabh_item, get_stac_validator
+
+LOGGER = logging.getLogger(__name__)
+
+
+def main():
+    logging.basicConfig(filename="create_uabh_items.log")
+    validator = get_stac_validator("schema/products/uabh.json")
+    zip_list = glob("/Users/chung-xianghong/Downloads/uabh_samples/**/*.zip")
+    for zip_file in zip_list:
+        create_uabh_item(zip_file, validator)
+
+
+if __name__ == "__main__":
+    main()

create_vpp_collection.py

@@ -0,0 +1,18 @@
+import logging
+from glob import glob
+
+from scripts.vpp.collection import create_vpp_collection, get_stac_validator
+from scripts.vpp.constants import COLLECTION_ID, STAC_DIR, WORKING_DIR
+
+LOGGER = logging.getLogger(__name__)
+
+
+def main():
+    logging.basicConfig(filename="create_vpp_collection.log")
+    item_list = glob(f"{WORKING_DIR}/{STAC_DIR}/{COLLECTION_ID}/**/*.json")
+    validator = get_stac_validator("schema/products/vpp.json")
+    create_vpp_collection(item_list, validator)
+
+
+if __name__ == "__main__":
+    main()

create_vpp_items.py

@@ -0,0 +1,34 @@
+import itertools as it
+import logging
+from concurrent.futures import ThreadPoolExecutor
+
+from tqdm import tqdm
+
+from scripts.vpp.constants import AWS_SESSION, BUCKET
+from scripts.vpp.item import create_page_iterator, create_product_list, create_vpp_item, get_stac_validator
+
+LOGGER = logging.getLogger(__name__)
+
+
+def main():
+    logging.basicConfig(filename="create_vpp_items.log")
+    validator = get_stac_validator("schema/products/vpp.json")
+    product_list = create_product_list(2017, 2023)
+
+    for product in product_list:
+        page_iterator = create_page_iterator(AWS_SESSION, BUCKET, product)
+        for page in page_iterator:
+            tiles = [prefix["Prefix"] for prefix in page["CommonPrefixes"]]
+            with ThreadPoolExecutor(max_workers=100) as executor:
+                list(
+                    tqdm(
+                        executor.map(
+                            create_vpp_item, it.repeat(AWS_SESSION), it.repeat(BUCKET), it.repeat(validator), tiles
+                        ),
+                        total=len(tiles),
+                    )
+                )
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -1,3 +1,7 @@
 #Libraries that your project use
+boto3
+pyproj
 pystac
 pystac[validation]
+rasterio
+shapely

schema/products/eu-hydro.json

@@ -62,7 +62,7 @@
               "name": "Copernicus Land Monitoring Service",
               "description": "The Copernicus Land Monitoring Service provides geographical information on land cover and its changes, land use, ground motions, vegetation state, water cycle and Earth's surface energy variables to a broad range of users in Europe and across the World in the field of environmental terrestrial applications.",
               "roles": ["licensor", "host"],
-              "url": "https://land.copernicus.eu/en"
+              "url": "https://land.copernicus.eu"
             }
           ]
         },
@@ -79,9 +79,7 @@
             },
             "temporal": {
               "const": {
-                "interval": [
-                  ["2006-01-01T00:00:00.000Z", "2012-12-31T00:00:00.000Z"]
-                ]
+                "interval": [["2006-01-01T00:00:00Z", "2012-12-31T00:00:00Z"]]
               }
             }
           }
@@ -151,7 +149,10 @@
         },
         "assets": {
           "type": "object",
-          "required": ["eu-hydro_v1p3_user_guide", "how_use_esri_fgdb_in_qgis"],
+          "required": [
+            "EU-HYDRO_V1p3_User_Guide_pdf",
+            "How_use_ESRI_FGDB_in_QGIS_pdf"
+          ],
           "additionalProperties": {
             "type": "object",
             "properties": {

schema/products/n2k.json

@@ -54,7 +54,7 @@
               "name": "Copernicus Land Monitoring Service",
               "description": "The Copernicus Land Monitoring Service provides geographical information on land cover and its changes, land use, ground motions, vegetation state, water cycle and Earth's surface energy variables to a broad range of users in Europe and across the World in the field of environmental terrestrial applications.",
               "roles": ["licensor", "host"],
-              "url": "https://land.copernicus.eu/en"
+              "url": "https://land.copernicus.eu"
             }
           ]
         },
@@ -71,7 +71,7 @@
               ]
             },
             "temporal": {
-              "const": { "interval": [["2006-01-01T00:00:00.000Z", null]] }
+              "const": { "interval": [["2006-01-01T00:00:00Z", null]] }
             }
           }
         },
@@ -141,7 +141,7 @@
         "assets": {
           "type": "object",
           "propertyNames": {
-            "pattern": "gdb$|gpkg$|zip$|metadata$|arcgis_layer$|qgis_layer$|ogc_layer$"
+            "pattern": "gdb$|gpkg$|zip$|xml$|lyr$|qml$|sld$"
           }
         }
       }

schema/products/uabh.json

@@ -142,17 +142,8 @@
         },
         "assets": {
           "type": "object",
-          "minProperties": 4,
-          "maxProperties": 6,
           "propertyNames": {
-            "anyOf": [
-              { "const": "dataset" },
-              { "const": "quality_check_report" },
-              { "const": "metadata" },
-              { "const": "quality_control_report" },
-              { "type": "string", "pattern": "^pixel_based_info" },
-              { "const": "compressed_dataset" }
-            ]
+            "pattern": "^(A[LT]|B[AEG]|C[HYZ]|D[EK]|E[ELS]|F[IR]|H[RU]|I[EST]|L[TUV]|M[EKT]|N[LO]|P[LT]|R[OS]|S[EIK]|TR|UK|XK)[0-9]{3}"
           }
         }
       }
@@ -179,8 +170,8 @@
         "stac_version": { "const": "1.0.0" },
         "stac_extensions": {
           "const": [
-            "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json",
-            "https://stac-extensions.github.io/projection/v1.1.0/schema.json"
+            "https://stac-extensions.github.io/projection/v1.1.0/schema.json",
+            "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json"
           ]
         },
         "type": { "const": "Collection" },
@@ -201,7 +192,7 @@
               "name": "Copernicus Land Monitoring Service",
               "description": "The Copernicus Land Monitoring Service provides geographical information on land cover and its changes, land use, ground motions, vegetation state, water cycle and Earth's surface energy variables to a broad range of users in Europe and across the World in the field of environmental terrestrial applications.",
               "roles": ["licensor", "host"],
-              "url": "https://land.copernicus.eu/en"
+              "url": "https://land.copernicus.eu"
             }
           ]
         },
@@ -211,7 +202,7 @@
               "bbox": [[-22.13, 35.07, 33.48, 64.38]]
             },
             "temporal": {
-              "interval": [["2012-01-01T00:00:00.000Z", null]]
+              "interval": [["2012-01-01T00:00:00Z", null]]
             }
           }
         },

schema/products/vpp.json

@@ -37,50 +37,50 @@
           "oneOf": [
             {
               "items": [
-                { "type": "number", "minimum": -25, "maximum": 45 },
-                { "type": "number", "minimum": 26, "maximum": 72 },
-                { "type": "number", "minimum": -25, "maximum": 45 },
-                { "type": "number", "minimum": 26, "maximum": 72 }
+                { "type": "number", "minimum": -31.86, "maximum": 45.12 },
+                { "type": "number", "minimum": 26.99, "maximum": 72.09 },
+                { "type": "number", "minimum": -31.86, "maximum": 45.12 },
+                { "type": "number", "minimum": 26.99, "maximum": 72.09 }
               ]
             },
             {
               "items": [
-                { "type": "number", "minimum": -55.23, "maximum": -50.89 },
-                { "type": "number", "minimum": 1.69, "maximum": 6.34 },
-                { "type": "number", "minimum": -55.23, "maximum": -50.89 },
-                { "type": "number", "minimum": 1.69, "maximum": 6.34 }
+                { "type": "number", "minimum": -62.09, "maximum": -60.1 },
+                { "type": "number", "minimum": 13.45, "maximum": 15.38 },
+                { "type": "number", "minimum": -62.09, "maximum": -60.1 },
+                { "type": "number", "minimum": 13.45, "maximum": 15.38 }
               ]
             },
             {
               "items": [
-                { "type": "number", "minimum": -62.09, "maximum": -60.08 },
-                { "type": "number", "minimum": 15.26, "maximum": 17.19 },
-                { "type": "number", "minimum": -62.09, "maximum": -60.08 },
-                { "type": "number", "minimum": 15.26, "maximum": 17.19 }
+                { "type": "number", "minimum": 44.07, "maximum": 46.02 },
+                { "type": "number", "minimum": -13.67, "maximum": -11.75 },
+                { "type": "number", "minimum": 44.07, "maximum": 46.02 },
+                { "type": "number", "minimum": -13.67, "maximum": -11.75 }
               ]
             },
             {
               "items": [
-                { "type": "number", "minimum": -62.1, "maximum": -60.1 },
-                { "type": "number", "minimum": 13.45, "maximum": 15.38 },
-                { "type": "number", "minimum": -62.1, "maximum": -60.1 },
-                { "type": "number", "minimum": 13.45, "maximum": 15.38 }
+                { "type": "number", "minimum": 55.06, "maximum": 56.15 },
+                { "type": "number", "minimum": -21.8, "maximum": -19.88 },
+                { "type": "number", "minimum": 55.06, "maximum": 56.15 },
+                { "type": "number", "minimum": -21.8, "maximum": -19.88 }
               ]
             },
             {
               "items": [
-                { "type": "number", "minimum": 44.05, "maximum": 46.04 },
-                { "type": "number", "minimum": -13.68, "maximum": -11.73 },
-                { "type": "number", "minimum": 44.05, "maximum": 46.04 },
-                { "type": "number", "minimum": -13.68, "maximum": -11.73 }
+                { "type": "number", "minimum": -62.08, "maximum": -60.08 },
+                { "type": "number", "minimum": 15.26, "maximum": 17.19 },
+                { "type": "number", "minimum": -62.08, "maximum": -60.08 },
+                { "type": "number", "minimum": 15.26, "maximum": 17.19 }
               ]
             },
             {
               "items": [
-                { "type": "number", "minimum": 55.04, "maximum": 56.16 },
-                { "type": "number", "minimum": -21.81, "maximum": -19.87 },
-                { "type": "number", "minimum": 55.04, "maximum": 56.16 },
-                { "type": "number", "minimum": -21.81, "maximum": -19.87 }
+                { "type": "number", "minimum": -55.21, "maximum": -50.9 },
+                { "type": "number", "minimum": 1.71, "maximum": 6.34 },
+                { "type": "number", "minimum": -55.21, "maximum": -50.9 },
+                { "type": "number", "minimum": 1.71, "maximum": 6.34 }
               ]
             }
           ]
@@ -120,6 +120,12 @@
                   "description": "The Copernicus Land Monitoring Service provides geographical information on land cover and its changes, land use, ground motions, vegetation state, water cycle and Earth's surface energy variables to a broad range of users in Europe and across the World in the field of environmental terrestrial applications.",
                   "roles": ["licensor", "host"],
                   "url": "https://land.copernicus.eu"
+                },
+                {
+                  "name": "VITO NV",
+                  "description": "VITO is an independent Flemish research organisation in the area of cleantech and sustainable development.",
+                  "roles": ["processor", "producer"],
+                  "url": "https://vito.be"
                 }
               ]
             },
@@ -221,8 +227,8 @@
         "stac_version": { "const": "1.0.0" },
         "stac_extensions": {
           "const": [
-            "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json",
-            "https://stac-extensions.github.io/projection/v1.1.0/schema.json"
+            "https://stac-extensions.github.io/projection/v1.1.0/schema.json",
+            "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json"
           ]
         },
         "type": { "const": "Collection" },
@@ -251,7 +257,13 @@
               "name": "Copernicus Land Monitoring Service",
               "description": "The Copernicus Land Monitoring Service provides geographical information on land cover and its changes, land use, ground motions, vegetation state, water cycle and Earth's surface energy variables to a broad range of users in Europe and across the World in the field of environmental terrestrial applications.",
               "roles": ["licensor", "host"],
-              "url": "https://land.copernicus.eu/en"
+              "url": "https://land.copernicus.eu"
+            },
+            {
+              "name": "VITO NV",
+              "description": "VITO is an independent Flemish research organisation in the area of cleantech and sustainable development.",
+              "roles": ["processor", "producer"],
+              "url": "https://vito.be"
             }
           ]
         },
@@ -261,7 +273,7 @@
               "bbox": [[-25, 26, 45, 72]]
             },
             "temporal": {
-              "interval": [["2017-01-01T00:00:00.000Z", null]]
+              "interval": [["2017-01-01T00:00:00Z", null]]
             }
           }
         },