[u] Define sources within catalog JSON (#3114, PR #3476)

DataBiosphere · Oct 15, 2021 · f2a7e05 · f2a7e05
2 parents f5e686b + b4075d4
commit f2a7e05
Show file tree

Hide file tree

Showing 14 changed files with 320 additions and 360 deletions.
diff --git a/UPGRADING.rst b/UPGRADING.rst
@@ -11,6 +11,15 @@ reverted. This is all fairly informal and loosely defined. Hopefully we won't
 have too many entries in this file.
 
 
+#3114 Define sources within catalog JSON
+========================================
+
+The ``AZUL_TDR_SOURCES`` and ``AZUL_…_SOURCES`` environment variables have been
+removed. Sources must be defined within the catalog configuration as a list of
+sources. As always, use the sandbox deployment's ``environment.py`` as a model
+when upgrading personal deployments.
+
+
 HumanCellAtlas/dcp2#17 TDR dev dataset is stale
 ===============================================
 

diff --git a/deployments/dev/environment.py b/deployments/dev/environment.py
diff --git a/deployments/prod/environment.py b/deployments/prod/environment.py
@@ -40,48 +40,47 @@ def env() -> Mapping[str, Optional[str]]:
         'AZUL_S3_BUCKET': 'edu-ucsc-gi-azul-dcp2-prod-storage-{AZUL_DEPLOYMENT_STAGE}',
 
         'AZUL_CATALOGS': json.dumps({
-            **{
-                f'{name}{rel}': dict(atlas='hca',
-                                     internal=internal,
-                                     plugins=dict(metadata=dict(name='hca'),
-                                                  repository=dict(name='tdr')))
-                for rel in (9, 10, 1)
-                for name, internal in (('dcp', False), ('it', True))
-            },
-            **{
-                name: dict(atlas='lungmap',
-                           internal=internal,
-                           plugins=dict(metadata=dict(name='hca'),
-                                        repository=dict(name='tdr')))
-                for name, internal in (('lungmap', False), ('it0lungmap', True))
-            }
+            f'{name}': dict(atlas=atlas,
+                            internal=bool(i),
+                            plugins=dict(metadata=dict(name='hca'),
+                                         repository=dict(name='tdr')),
+                            sources=sources)
+            for atlas, names, sources in [
+                (
+                    'hca',
+                    ['dcp9', 'it9'],
+                    [
+                        'tdr:tdr-fp-546ade29:snapshot/hca_prod_20201120_dcp2___20210910_dcp9:',
+                    ]
+                ),
+                (
+                    'hca',
+                    ['dcp10', 'it10'],
+                    [
+                        'tdr:tdr-fp-43194825:snapshot/hca_prod_20201120_dcp2___20211004_dcp10:'
+                    ]
+                ),
+                (
+                    'hca',
+                    ['dcp1', 'it1'],
+                    [
+                        'tdr:broad-datarepo-terra-prod-hca2:snapshot/hca_prod_20201118_dcp1___20201209:',
+                    ]
+                ),
+                (
+                    'lungmap',
+                    ['lungmap', 'it0lungmap'],
+                    [
+                        'tdr:tdr-fp-a02eee6b:snapshot/hca_prod_1bdcecde16be420888f478cd2133d11d__20211013_20211013_lungmap:',
+                        'tdr:tdr-fp-9f58860b:snapshot/hca_prod_00f056f273ff43ac97ff69ca10e38c89__20211004_lungmap_20211005:',
+                        'tdr:tdr-fp-ad968fd8:snapshot/hca_prod_2620497955a349b28d2b53e0bdfcb176__20211012_lungmap_20211013_lungmap:'
+                    ]
+                )
+            ] for i, name in enumerate(names)
         }),
 
         'AZUL_PARTITION_PREFIX_LENGTH': '2',
 
-        'AZUL_TDR_SOURCES': ','.join([
-            'tdr:broad-datarepo-terra-prod-hca2:snapshot/hca_prod_20201118_dcp1___20201209:',
-        ]),
-        **{
-            f'AZUL_TDR_{catalog.upper()}_SOURCES': ','.join([
-                'tdr:tdr-fp-546ade29:snapshot/hca_prod_20201120_dcp2___20210910_dcp9:',
-            ])
-            for catalog in ('dcp9', 'it9')
-        },
-        **{
-            f'AZUL_TDR_{catalog.upper()}_SOURCES': ','.join([
-                'tdr:tdr-fp-43194825:snapshot/hca_prod_20201120_dcp2___20211004_dcp10:'
-            ])
-            for catalog in ('dcp10', 'it10')
-        },
-        **{
-            f'AZUL_TDR_{catalog.upper()}_SOURCES': ','.join([
-                'tdr:tdr-fp-a02eee6b:snapshot/hca_prod_1bdcecde16be420888f478cd2133d11d__20211013_20211013_lungmap:',
-                'tdr:tdr-fp-9f58860b:snapshot/hca_prod_00f056f273ff43ac97ff69ca10e38c89__20211004_lungmap_20211005:',
-                'tdr:tdr-fp-ad968fd8:snapshot/hca_prod_2620497955a349b28d2b53e0bdfcb176__20211012_lungmap_20211013_lungmap:'
-            ])
-            for catalog in ('lungmap', 'it0lungmap')
-        },
         'AZUL_TDR_SOURCE_LOCATION': 'US',
         'AZUL_TDR_SERVICE_URL': 'https://jade-terra.datarepo-prod.broadinstitute.org',
         'AZUL_SAM_SERVICE_URL': 'https://sam.dsde-prod.broadinstitute.org',