Merge pull request #1 from stacchain/nested-items

Add support for nested items
stacchain · Nov 18, 2024 · 90faced · 90faced
2 parents 9a75da2 + 310acbf
commit 90faced
Show file tree

Hide file tree

Showing 35 changed files with 756 additions and 62 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -17,6 +17,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+## [v0.2.0] - 2024-11-16
+
+### Added
+
+- Enhanced collection processing to support nested subdirectories for items [#1](https://github.com/stacchain/stac-merkle-tree-cli/pull/1)
+- Test to ensure `merkle:root` values remain the same whether items are nested or not [#1](https://github.com/stacchain/stac-merkle-tree-cli/pull/1)
+
 ## [v0.1.1] - 2024-11-16
 
 ### Fixed

diff --git a/README.md b/README.md
@@ -61,7 +61,37 @@ pip install stac-merkle-tree-cli
 
 ## Directory Structure
 
-Ensure your STAC catalog follows the directory structure below for optimal processing:
+Ensure your STAC catalog follows one of the directory structures below for optimal processing:
+
+### Standard Flat Structure
+
+In this structure, all items are at the same level as the `collection.json` file:
+
+```bash
+collection/
+├── collection.json
+├── item1.json
+├── item2.json
+└── ...
+```
+
+### Nested Structure
+
+In this structure, items can be nested inside their own subdirectories within a collection:
+
+```bash
+collection/
+├── collection.json
+├── item1/
+│   └── item1.json
+├── item2/
+│   └── item2.json
+└── ...
+```
+
+### Catalog with Collections and Nested Items
+
+A full STAC catalog with collections, where items can be either at the same level as the `collection.json` or nested within subdirectories:
 
 ```bash
 catalog/
@@ -70,13 +100,14 @@ catalog/
 │   ├── collection1/
 │   │   ├── collection.json
 │   │   ├── item1.json
-│   │   ├── item2.json
-│   │   └── ...
+│   │   ├── item2/
+│   │   │   └── item2.json
 │   ├── collection2/
 │   │   ├── collection.json
-│   │   ├── item1.json
-│   │   └── ...
-│   └── ...
+│   │   ├── item1/
+│   │   │   └── item1.json
+│   │   └── item2.json
+└── ...
 ```
 
 - **Catalog Level**:
@@ -86,7 +117,7 @@ catalog/
   - Each collection has its own directory inside `collections/`, named after the collection.
   - Inside each collection directory:
     - `collection.json`: Collection metadata.
-    - `item.json`, `item2.json`, ...: Items belonging to the collection.
+    - `item.json`, `item2.json`, ...: Items belonging to the collection, either at the same level or nested within subdirectories.
 
 ## Usage
 
@@ -97,7 +128,7 @@ After installing the package, you can use the `stac-merkle-tree-cli` command to
 Navigate to the directory containing your catalog.json file and run the command as follows:
 
 ```bash
-stac-merkle-cli path/to/catalog.json
+stac-merkle-tree-cli path/to/catalog.json
 ```
 
 ### Example
@@ -111,18 +142,19 @@ my_stac_catalog/
 │   ├── collection1/
 │   │   ├── collection.json
 │   │   ├── item1.json
-│   │   └── item2.json
+│   │   └── item2/
+│   │       └── item2.json
 │   └── collection2/
 │       ├── collection.json
-│       ├── item1.json
+│       ├── item1/
+│       │   └── item1.json
 │       └── item2.json
-
 ```
 
 Run the tool:
 
 ```bash
-stac-merkle-tree-cli ./my_stac_catalog/catalog.json
+stac-merkle-tree-cli my_stac_catalog/catalog.json
 ```
 
 Expected Output:

diff --git a/example_catalog/catalog.json b/example_catalog/catalog.json
@@ -55,14 +55,14 @@
   "stac_extensions": [
     "https://stacchain.github.io/merkle-tree/v1.0.0/schema.json"
   ],
-  "merkle:object_hash": "b14fd102417c1d673f481bc053d19946aefdc27d84c584989b23c676c897bd5a",
-  "merkle:root": "e3f79fe1a806f083fb6a3e5d7b052a0bed84f69c4b5c5969ddc4e39e0255f997",
+  "merkle:object_hash": "4adf81a9aba90fe1f483ec3bbe7c628309e598239cee45d4aeafa1bc8c767ae6",
+  "merkle:root": "6dcb155aa44f0384cf23af6843fb0505d752b11aaaa0778829889b9d598c176f",
   "merkle:hash_method": {
     "function": "sha256",
     "fields": [
       "*"
     ],
     "ordering": "ascending",
-    "description": "Computed by excluding Merkle fields and including merkle:object_hash values in ascending order to build the Merkle tree."
+    "description": "Computed by including the merkle:root of collections and the catalogs own merkle:object_hash."
   }
 }
diff --git a/.../collections/COP-DEM/DEM1_SAR_DGE_30_20101212T230244_20140325T230302_ADS_000000_1jTi.json b/.../collections/COP-DEM/DEM1_SAR_DGE_30_20101212T230244_20140325T230302_ADS_000000_1jTi.json
@@ -43,7 +43,7 @@
     "datetime": "2010-12-12T23:02:44.000000Z",
     "end_datetime": "2014-03-25T23:03:02.000000Z",
     "start_datetime": "2010-12-12T23:02:44.000000Z",
-    "merkle:object_hash": "ce9f56e695ab1751b8f0c8d9ef1f1ecedaf04574ec3077e70e7426ec9fc61ea4"
+    "merkle:object_hash": "32c896520ed1fc38a93e59877cb6a88a5e795721eeae3bf6e7bd29206efb87af"
   },
   "bbox": [
     99,

diff --git a/.../collections/COP-DEM/DEM1_SAR_DGE_30_20101212T230244_20140329T113710_ADS_000000_eAmG.json b/.../collections/COP-DEM/DEM1_SAR_DGE_30_20101212T230244_20140329T113710_ADS_000000_eAmG.json
@@ -43,7 +43,7 @@
     "datetime": "2010-12-12T23:02:44.000000Z",
     "end_datetime": "2014-03-29T11:37:10.000000Z",
     "start_datetime": "2010-12-12T23:02:44.000000Z",
-    "merkle:object_hash": "ac66e07717b56e8421c8fec00b2b300afd49d30a8ec9c6d505df3b0568de9c77"
+    "merkle:object_hash": "9c6b0b011ec783f4bcdec7c712c2443c3c6eca6d1a0d043842a7c8d587d767c4"
   },
   "bbox": [
     100,

diff --git a/.../collections/COP-DEM/DEM1_SAR_DGE_30_20101215T103647_20130405T103047_ADS_000000_RHJx.json b/.../collections/COP-DEM/DEM1_SAR_DGE_30_20101215T103647_20130405T103047_ADS_000000_RHJx.json
@@ -43,7 +43,7 @@
     "datetime": "2010-12-15T10:36:47.000000Z",
     "end_datetime": "2013-04-05T10:30:47.000000Z",
     "start_datetime": "2010-12-15T10:36:47.000000Z",
-    "merkle:object_hash": "39969fd5f4a3a170ff39df8f2c13ebca66aab40890275a94ce798e281b85d54d"
+    "merkle:object_hash": "762a6152760f8d4868e4036b2ad009a948485e2626c51c6db2a506fc2c7e491c"
   },
   "bbox": [
     98,

diff --git a/.../collections/COP-DEM/DEM1_SAR_DGE_30_20101215T103647_20130405T103047_ADS_000000_oCX9.json b/.../collections/COP-DEM/DEM1_SAR_DGE_30_20101215T103647_20130405T103047_ADS_000000_oCX9.json
@@ -43,7 +43,7 @@
     "datetime": "2010-12-15T10:36:47.000000Z",
     "end_datetime": "2013-04-05T10:30:47.000000Z",
     "start_datetime": "2010-12-15T10:36:47.000000Z",
-    "merkle:object_hash": "22a31ab988181f280a0bfc6872556ad1d670373bd997f711389d057e1b1f531a"
+    "merkle:object_hash": "b56b4a839810159bfbcd77fe458bcf12173e796a6750b31399166c67cfd29c33"
   },
   "bbox": [
     100,

diff --git a/.../collections/COP-DEM/DEM1_SAR_DGE_30_20101215T203649_20140913T090954_ADS_000000_4KBA.json b/.../collections/COP-DEM/DEM1_SAR_DGE_30_20101215T203649_20140913T090954_ADS_000000_4KBA.json
@@ -43,7 +43,7 @@
     "datetime": "2010-12-15T20:36:49.000000Z",
     "end_datetime": "2014-09-13T09:09:54.000000Z",
     "start_datetime": "2010-12-15T20:36:49.000000Z",
-    "merkle:object_hash": "3c22648957e7c76f75a9fbfdf7c164e7d745d9e4ebe809db15d2474e01be4764"
+    "merkle:object_hash": "df89c14ec6017be8ad68b8a51f1152671520a8959c65a7a377cfc2b789f30708"
   },
   "bbox": [
     135,

diff --git a/.../collections/COP-DEM/DEM1_SAR_DGE_30_20101215T203649_20140913T090954_ADS_000000_Quqd.json b/.../collections/COP-DEM/DEM1_SAR_DGE_30_20101215T203649_20140913T090954_ADS_000000_Quqd.json
@@ -43,7 +43,7 @@
     "datetime": "2010-12-15T20:36:49.000000Z",
     "end_datetime": "2014-09-13T09:09:54.000000Z",
     "start_datetime": "2010-12-15T20:36:49.000000Z",
-    "merkle:object_hash": "6d16f23e2fe2849ce0316c012e9284b413f7c086a11d6de421c0582b397f513e"
+    "merkle:object_hash": "7365039727dfa4456a355cd02585e713af706692716bf0b6050c1b8e82e1308f"
   },
   "bbox": [
     135,

diff --git a/.../collections/COP-DEM/DEM1_SAR_DGE_30_20101215T203914_20121226T204852_ADS_000000_Yjwo.json b/.../collections/COP-DEM/DEM1_SAR_DGE_30_20101215T203914_20121226T204852_ADS_000000_Yjwo.json
@@ -43,7 +43,7 @@
     "datetime": "2010-12-15T20:39:14.000000Z",
     "end_datetime": "2012-12-26T20:48:52.000000Z",
     "start_datetime": "2010-12-15T20:39:14.000000Z",
-    "merkle:object_hash": "ed99b4348ede45959a9cf471490f0f2c6a106bb4337a46d07d4fc4b4988f099f"
+    "merkle:object_hash": "b2bb980dc3b1d92106f009b6c732b7140533e622f3e06801dedd643eb9a8614c"
   },
   "bbox": [
     134,

diff --git a/.../collections/COP-DEM/DEM1_SAR_DGE_30_20101216T005027_20130331T005208_ADS_000000_J3m5.json b/.../collections/COP-DEM/DEM1_SAR_DGE_30_20101216T005027_20130331T005208_ADS_000000_J3m5.json
@@ -43,7 +43,7 @@
     "datetime": "2010-12-16T00:50:27.000000Z",
     "end_datetime": "2013-03-31T00:52:08.000000Z",
     "start_datetime": "2010-12-16T00:50:27.000000Z",
-    "merkle:object_hash": "90a0e5265d02fd58ab81437d14ef1c318ec1b86167ac935d6abb598602798690"
+    "merkle:object_hash": "1d3e8b68bfa0c90b75cbbddec7cccc674ee732d90e83dcb2afa7b97958096626"
   },
   "bbox": [
     -111,

diff --git a/.../collections/COP-DEM/DEM1_SAR_DGE_30_20101216T005027_20130411T005216_ADS_000000_Hbh2.json b/.../collections/COP-DEM/DEM1_SAR_DGE_30_20101216T005027_20130411T005216_ADS_000000_Hbh2.json
@@ -43,7 +43,7 @@
     "datetime": "2010-12-16T00:50:27.000000Z",
     "end_datetime": "2013-04-11T00:52:16.000000Z",
     "start_datetime": "2010-12-16T00:50:27.000000Z",
-    "merkle:object_hash": "176a6f0026a763a85909d64d3eeac37873cd7c970fae685a6f37bd16d806b3fe"
+    "merkle:object_hash": "4543f10e43c01172bfa78b0a822c5619e4d7cd6b711475355930ade977c92d23"
   },
   "bbox": [
     -112,

diff --git a/.../collections/COP-DEM/DEM1_SAR_DGE_90_20101212T102356_20130709T103701_ADS_000000_6275.json b/.../collections/COP-DEM/DEM1_SAR_DGE_90_20101212T102356_20130709T103701_ADS_000000_6275.json
@@ -43,7 +43,7 @@
     "datetime": "2010-12-12T10:24:16.000000Z",
     "end_datetime": "2010-12-12T10:24:17.000000Z",
     "start_datetime": "2010-12-12T10:24:16.000000Z",
-    "merkle:object_hash": "ffaee98a244aaad0f970100f0f3b11bf3ffd3f5de74fa473ed86cc851c73dbc9"
+    "merkle:object_hash": "c3c057f15200da00d392d8608a77190019a06c092bd7814670c3426653e1ca78"
   },
   "bbox": [
     -72,

diff --git a/.../collections/COP-DEM/DEM1_SAR_DGE_90_20101212T102356_20130709T103701_ADS_000000_8521.json b/.../collections/COP-DEM/DEM1_SAR_DGE_90_20101212T102356_20130709T103701_ADS_000000_8521.json
@@ -43,7 +43,7 @@
     "datetime": "2010-12-12T10:24:18.000000Z",
     "end_datetime": "2010-12-12T10:24:19.000000Z",
     "start_datetime": "2010-12-12T10:24:18.000000Z",
-    "merkle:object_hash": "6d7dca89eec5da5e31023b7e24b9527e33a021099c8e74791ecda860f96091ab"
+    "merkle:object_hash": "8c1277ae875d680ba189a884e63cd66ed9ac0c94b2ba5eb55d3ae11847814f6d"
   },
   "bbox": [
     -72,

diff --git a/.../collections/COP-DEM/DEM1_SAR_DTE_30_20101215T203649_20140913T090954_ADS_000000_kipX.json b/.../collections/COP-DEM/DEM1_SAR_DTE_30_20101215T203649_20140913T090954_ADS_000000_kipX.json
@@ -43,7 +43,7 @@
     "datetime": "2010-12-15T20:36:49.000000Z",
     "end_datetime": "2014-09-13T09:09:54.000000Z",
     "start_datetime": "2010-12-15T20:36:49.000000Z",
-    "merkle:object_hash": "88f4d9bb5b1512f214f1ca60e6ff2bac28640fd0a00339d01c453b6db9fd7f88"
+    "merkle:object_hash": "ded2686d7956fde26b4941fd1a4294d7822f54bd7bf21f6275eff176484757db"
   },
   "bbox": [
     135,

diff --git a/.../collections/COP-DEM/DEM1_SAR_DTE_30_20101215T203914_20130404T204908_ADS_000000_CURF.json b/.../collections/COP-DEM/DEM1_SAR_DTE_30_20101215T203914_20130404T204908_ADS_000000_CURF.json
@@ -43,7 +43,7 @@
     "datetime": "2010-12-15T20:39:14.000000Z",
     "end_datetime": "2013-04-04T20:49:08.000000Z",
     "start_datetime": "2010-12-15T20:39:14.000000Z",
-    "merkle:object_hash": "457aa1fbc4005627077b55cd673a4457f2afc8a15ef3b776af8df60e60e8e1ab"
+    "merkle:object_hash": "e6f9dfc50f4bda3d7d07448d15e6528cb34b1d64f5a95ac77058004356afe2d8"
   },
   "bbox": [
     133,

diff --git a/.../collections/COP-DEM/DEM1_SAR_DTE_30_20101216T100443_20140415T214254_ADS_000000_9Wqx.json b/.../collections/COP-DEM/DEM1_SAR_DTE_30_20101216T100443_20140415T214254_ADS_000000_9Wqx.json
@@ -43,7 +43,7 @@
     "datetime": "2010-12-16T10:04:43.000000Z",
     "end_datetime": "2014-04-15T21:42:54.000000Z",
     "start_datetime": "2010-12-16T10:04:43.000000Z",
-    "merkle:object_hash": "e7b65b76ed20f947d8a2b4ae126b832d9f07fb18822863556c9fa0ac40dae873"
+    "merkle:object_hash": "da40bec7fb828698829f9b69bcf2c926a8f7967044bc15cf049335bfa2bded3a"
   },
   "bbox": [
     122,

diff --git a/.../collections/COP-DEM/DEM1_SAR_DTE_30_20101217T200231_20140802T083458_ADS_000000_tFcK.json b/.../collections/COP-DEM/DEM1_SAR_DTE_30_20101217T200231_20140802T083458_ADS_000000_tFcK.json
@@ -43,7 +43,7 @@
     "datetime": "2010-12-17T20:02:31.000000Z",
     "end_datetime": "2014-08-02T08:34:58.000000Z",
     "start_datetime": "2010-12-17T20:02:31.000000Z",
-    "merkle:object_hash": "d0114075968ff1a3860eaf90d85317304f96757d6313f396eb556546a9c95006"
+    "merkle:object_hash": "09eae78bd6d1a6385d330d96832329c30c27868321a886220ec8d4eb7f729b3f"
   },
   "bbox": [
     144,

diff --git a/.../collections/COP-DEM/DEM1_SAR_DTE_90_20101212T084914_20130418T085214_ADS_000000_5545.json b/.../collections/COP-DEM/DEM1_SAR_DTE_90_20101212T084914_20130418T085214_ADS_000000_5545.json
@@ -43,7 +43,7 @@
     "datetime": "2010-12-12T08:49:36.000000Z",
     "end_datetime": "2010-12-12T08:49:37.000000Z",
     "start_datetime": "2010-12-12T08:49:36.000000Z",
-    "merkle:object_hash": "7611aa1e37ee256bb346d0405c210c9e2256c43e7ed9854b4a91ef75caf59d09"
+    "merkle:object_hash": "718637ce64200c90a8981916961cbaa994684aa362208ae4c4588f99dafa3834"
   },
   "bbox": [
     -49,

diff --git a/.../collections/COP-DEM/DEM1_SAR_DTE_90_20101212T084914_20140212T212323_ADS_000000_1370.json b/.../collections/COP-DEM/DEM1_SAR_DTE_90_20101212T084914_20140212T212323_ADS_000000_1370.json
@@ -43,7 +43,7 @@
     "datetime": "2010-12-12T08:49:42.000000Z",
     "end_datetime": "2010-12-12T08:49:43.000000Z",
     "start_datetime": "2010-12-12T08:49:42.000000Z",
-    "merkle:object_hash": "c4a241c5917f7fcb82bedca10127ad22504b3462c0d3704d76a5f549e2c24010"
+    "merkle:object_hash": "81a60b7a33c7c17b486a4b3e77b2a74211aa2a4c868d1db8a39f8442800cdb7c"
   },
   "bbox": [
     -48,

diff --git a/.../collections/COP-DEM/DEM1_SAR_DTE_90_20101212T102356_20130628T103333_ADS_000000_1705.json b/.../collections/COP-DEM/DEM1_SAR_DTE_90_20101212T102356_20130628T103333_ADS_000000_1705.json
@@ -43,7 +43,7 @@
     "datetime": "2010-12-12T10:24:00.000000Z",
     "end_datetime": "2010-12-12T10:24:01.000000Z",
     "start_datetime": "2010-12-12T10:24:00.000000Z",
-    "merkle:object_hash": "ae063aa10078d5316f36718b92f966c9f96f63f621839bd031730eb068c1c265"
+    "merkle:object_hash": "0372d094f1c112b8df328070bcabdc34d3bbe043803f89fb259c0d1510a5bee5"
   },
   "bbox": [
     -72,

diff --git a/.../collections/COP-DEM/DEM1_SAR_DTE_90_20101213T034716_20130408T035028_ADS_000000_5033.json b/.../collections/COP-DEM/DEM1_SAR_DTE_90_20101213T034716_20130408T035028_ADS_000000_5033.json
@@ -43,7 +43,7 @@
     "datetime": "2010-12-13T03:47:46.000000Z",
     "end_datetime": "2010-12-13T03:47:47.000000Z",
     "start_datetime": "2010-12-13T03:47:46.000000Z",
-    "merkle:object_hash": "c1511bd4889e8078010287a820d3fcc216cad1ab48374e324fb7098fd1c3f882"
+    "merkle:object_hash": "550175301f559ef22b03c179a2ece08cde642c5502bbeccc40dd55b6526fb6e2"
   },
   "bbox": [
     26,

diff --git a/.../collections/COP-DEM/DEM1_SAR_DTE_90_20101217T224141_20140127T121413_ADS_000000_0611.json b/.../collections/COP-DEM/DEM1_SAR_DTE_90_20101217T224141_20140127T121413_ADS_000000_0611.json
@@ -43,7 +43,7 @@
     "datetime": "2010-12-17T22:42:03.000000Z",
     "end_datetime": "2010-12-17T22:42:04.000000Z",
     "start_datetime": "2010-12-17T22:42:03.000000Z",
-    "merkle:object_hash": "9c33c4c49913483588fb7e2aa8084e675649ed253083ab62ac1ebebcfc16a944"
+    "merkle:object_hash": "176aa74d27edd9327ada0801fa79ccba1348e69d2f82d0bed44eda5b59908a31"
   },
   "bbox": [
     -84,

diff --git a/example_catalog/collections/COP-DEM/collection.json b/example_catalog/collections/COP-DEM/collection.json
@@ -50,7 +50,7 @@
       "href": "https://catalogue.dataspace.copernicus.eu/stac/collections/COP-DEM/items"
     }
   ],
-  "merkle:root": "aa7f89b29cb339032ec86d81d4090bdbd52199152fb657f50b08eec1b3234ee2",
+  "merkle:root": "458a7fa66b5a2e0a4993f648813eaff0879a54ffdece2b2d58c546d177925174",
   "merkle:hash_method": {
     "function": "sha256",
     "fields": [
@@ -59,5 +59,5 @@
     "ordering": "ascending",
     "description": "Computed by including merkle:object_hash values in ascending order and building the Merkle tree."
   },
-  "merkle:object_hash": "17789b31f8ae304de8dbe2350a15263dbf5e31adfc0d17a997e7e55f4cfc2f53"
+  "merkle:object_hash": "a3606ee70addcac399baa161aeb6b5f92c2f56759e19a0670f7d352e921aa4ee"
 }
diff --git a/example_catalog/collections/S2GLC/S2GLC_T30TWT_2017.json b/example_catalog/collections/S2GLC/S2GLC_T30TWT_2017.json
@@ -41,7 +41,7 @@
     "datetime": "2019-07-15T00:00:00.000000Z",
     "end_datetime": "2019-07-15T00:00:00.000000Z",
     "start_datetime": "2019-07-15T00:00:00.000000Z",
-    "merkle:object_hash": "3a3803a0dae5dbaf9561aeb4cce2770bf38b5da4b71ca67398fb24d48c43a68f"
+    "merkle:object_hash": "1d7db82d0d699e04c7470451ae682e161824d34e610566499be35e8417d74f43"
   },
   "bbox": [
     -3.00026750564575,

diff --git a/example_catalog/collections/S2GLC/collection.json b/example_catalog/collections/S2GLC/collection.json
@@ -50,8 +50,8 @@
       "href": "https://catalogue.dataspace.copernicus.eu/stac/collections/S2GLC/items"
     }
   ],
-  "merkle:object_hash": "84ab0e102924c012d4cf2a3b3e10ed4f768f695001174cfd5d9c75d4335b7a48",
-  "merkle:root": "33631c1a3d9339ffc66b3f3a3eb3de8f558bcabe4900494b55ca17aff851e661",
+  "merkle:object_hash": "164a3ea0c299caad2fed2e7799d5c5e2caef872efb41ca56c1e0fc627ecacc4a",
+  "merkle:root": "29e0391249b04fdf945b03ae3cd01662665cd6e29471ffa51e2d43e75d201112",
   "merkle:hash_method": {
     "function": "sha256",
     "fields": [