From b4075d44e4117b0594b478b42f0abf1d49ed5833 Mon Sep 17 00:00:00 2001 From: amar jandu Date: Wed, 25 Aug 2021 17:29:54 -0700 Subject: [PATCH] [u] Define sources within catalog JSON (#3114) --- UPGRADING.rst | 9 + deployments/dev/environment.py | 243 +++++++++--------- deployments/prod/environment.py | 75 +++--- deployments/sandbox/environment.py | 239 ++++++++--------- environment.py | 52 +--- scripts/generate_openapi_document.py | 3 +- scripts/post_deploy.py | 2 +- src/azul/__init__.py | 19 +- .../plugins/repository/canned/__init__.py | 2 +- src/azul/plugins/repository/tdr/__init__.py | 2 +- src/azul/portal_service.py | 2 +- test/azul_test_case.py | 3 +- test/integration_test.py | 10 +- test/service/test_repository_proxy.py | 19 +- 14 files changed, 320 insertions(+), 360 deletions(-) diff --git a/UPGRADING.rst b/UPGRADING.rst index a937c5a5af..2f2a26e46d 100644 --- a/UPGRADING.rst +++ b/UPGRADING.rst @@ -11,6 +11,15 @@ reverted. This is all fairly informal and loosely defined. Hopefully we won't have too many entries in this file. +#3114 Define sources within catalog JSON +======================================== + +The ``AZUL_TDR_SOURCES`` and ``AZUL_…_SOURCES`` environment variables have been +removed. Sources must be defined within the catalog configuration as a list of +sources. As always, use the sandbox deployment's ``environment.py`` as a model +when upgrading personal deployments. + + HumanCellAtlas/dcp2#17 TDR dev dataset is stale =============================================== diff --git a/deployments/dev/environment.py b/deployments/dev/environment.py index 43fbcab4e4..c0c3bb20ce 100644 --- a/deployments/dev/environment.py +++ b/deployments/dev/environment.py @@ -25,12 +25,121 @@ def mksrc(project, snapshot, subgraphs, ma: int = 0): return f'tdr:{project}:snapshot/{snapshot}:/{prefix(subgraphs)}' +dcp2_sources = [ + mksrc('datarepo-dev-a9252919', 'hca_dev_005d611a14d54fbf846e571a1f874f70__20210827_20210903', 7), + mksrc('datarepo-dev-c148d39c', 'hca_dev_027c51c60719469fa7f5640fe57cbece__20210827_20210902', 8), + mksrc('datarepo-dev-e2ab8487', 'hca_dev_03c6fce7789e4e78a27a664d562bb738__20210902_20210907', 1530), + mksrc('datarepo-dev-37639c56', 'hca_dev_05657a599f9d4bb9b77b24be13aa5cea__20210827_20210928', 185), + mksrc('datarepo-dev-9f4012c9', 'hca_dev_05be4f374506429bb112506444507d62__20210827_20210902', 1544), + mksrc('datarepo-dev-baa2812f', 'hca_dev_0792db3480474e62802c9177c9cd8e28__20210827_20210903', 1450), + mksrc('datarepo-dev-38e08b5c', 'hca_dev_08b794a0519c4516b184c583746254c5__20210901_20210903', 2), + mksrc('datarepo-dev-2749da57', 'hca_dev_091cf39b01bc42e59437f419a66c8a45__20210830_20210903', 20), + mksrc('datarepo-dev-eab7fa76', 'hca_dev_0c3b7785f74d40918616a68757e4c2a8__20210827_20210903', 177), + mksrc('datarepo-dev-fef02a92', 'hca_dev_0d4b87ea6e9e456982e41343e0e3259f__20210827_20210903', 8), + mksrc('datarepo-dev-78bae095', 'hca_dev_0fd8f91862d64b8bac354c53dd601f71__20210830_20210903', 10), + mksrc('datarepo-dev-ff0a2fe7', 'hca_dev_116965f3f09447699d28ae675c1b569c__20210827_20210903', 8), + mksrc('datarepo-dev-4de1b9fd', 'hca_dev_16ed4ad8731946b288596fe1c1d73a82__20210830_20210903', 28), + mksrc('datarepo-dev-135f340c', 'hca_dev_1c6a960d52ac44eab728a59c7ab9dc8e__20210827_20210928', 10), + mksrc('datarepo-dev-86c60513', 'hca_dev_1cd1f41ff81a486ba05b66ec60f81dcf__20210901_20210903', 18), + mksrc('datarepo-dev-f5321179', 'hca_dev_1ce3b3dc02f244a896dad6d107b27a76__20210827_20210903', 421), + mksrc('datarepo-dev-76de829d', 'hca_dev_2043c65a1cf84828a6569e247d4e64f1__20210831_20210907', 1734), + mksrc('datarepo-dev-c808badb', 'hca_dev_2086eb0510b9432bb7f0169ccc49d270__20210827_20210903', 10), + mksrc('datarepo-dev-23782220', 'hca_dev_23587fb31a4a4f58ad74cc9a4cb4c254__20210827_20210909', 1476), + mksrc('datarepo-dev-4c3e6011', 'hca_dev_248fcf0316c64a41b6ccaad4d894ca42__20210907_20210907', 2958), + mksrc('datarepo-dev-1c2c69d9', 'hca_dev_24c654a5caa5440a8f02582921f2db4a__20210830_20210903', 3), + mksrc('datarepo-dev-38f08cd8', 'hca_dev_2a64db431b554639aabb8dba0145689d__20210830_20210903', 10), + mksrc('datarepo-dev-4cf05ce2', 'hca_dev_2a72a4e566b2405abb7c1e463e8febb0__20210901_20210903', 2290), + mksrc('datarepo-dev-3041c2cf', 'hca_dev_2af52a1365cb4973b51339be38f2df3f__20210830_20210903', 10), + mksrc('datarepo-dev-311340f6', 'hca_dev_2d8460958a334f3c97d4585bafac13b4__20210902_20210907', 3589), + mksrc('datarepo-dev-766bfb76', 'hca_dev_2ef3655a973d4d699b4121fa4041eed7__20210827_20210903', 8), + mksrc('datarepo-dev-1720b3c5', 'hca_dev_379ed69ebe0548bcaf5ea7fc589709bf__20210827_20210902', 4), + mksrc('datarepo-dev-ac6efd3f', 'hca_dev_38449aea70b540db84b31e08f32efe34__20210830_20210903', 42), + mksrc('datarepo-dev-40283c27', 'hca_dev_3a69470330844ece9abed935fd5f6748__20210901_20210903', 125), + mksrc('datarepo-dev-b08233fa', 'hca_dev_3cfcdff5dee14a7ba591c09c6e850b11__20210827_20210903', 8), + mksrc('datarepo-dev-bdc9f342', 'hca_dev_3e329187a9c448ec90e3cc45f7c2311c__20210901_20210903', 1001), + mksrc('datarepo-dev-ec07c8d8', 'hca_dev_4037007b0eff4e6db7bd8dd8eec80143__20210831_20210903', 39), + mksrc('datarepo-dev-c0ec174a', 'hca_dev_403c3e7668144a2da5805dd5de38c7ff__20210827_20210903', 63), + mksrc('datarepo-dev-31b3553a', 'hca_dev_414accedeba0440fb721befbc5642bef__20210827_20210903', 4), + mksrc('datarepo-dev-b4789901', 'hca_dev_41fb1734a121461695c73b732c9433c7__20210830_20210903', 12), + mksrc('datarepo-dev-4e5ffd52', 'hca_dev_42d4f8d454224b78adaee7c3c2ef511c__20210830_20210903', 9), + mksrc('datarepo-dev-5ef7f2e2', 'hca_dev_455b46e6d8ea4611861ede720a562ada__20210901_20210903', 74), + mksrc('datarepo-dev-a6c6b953', 'hca_dev_4bec484dca7a47b48d488830e06ad6db__20210830_20210903', 14), + mksrc('datarepo-dev-f31edbc2', 'hca_dev_4d6f6c962a8343d88fe10f53bffd4674__20210901_20210903', 12), + mksrc('datarepo-dev-bb8fbae4', 'hca_dev_51f02950ee254f4b8d0759aa99bb3498__20210827_20210928', 6), + mksrc('datarepo-dev-71de019e', 'hca_dev_520afa10f9d24e93ab7a26c4c863ce18__20210827_20210928', 649), + mksrc('datarepo-dev-ffcf8b00', 'hca_dev_52b29aa4c8d642b4807ab35be94469ca__20210830_20210903', 467), + mksrc('datarepo-dev-f76414c8', 'hca_dev_52d10a60c8d14d068a5eaf0d5c0d5034__20210827_20210902', 176), + mksrc('datarepo-dev-319b80f7', 'hca_dev_577c946d6de54b55a854cd3fde40bff2__20210827_20210903', 7), + mksrc('datarepo-dev-279f1986', 'hca_dev_5ee710d7e2d54fe2818d15f5e31dae32__20210901_20210903', 41), + mksrc('datarepo-dev-0abea017', 'hca_dev_6072616c87944b208f52fb15992ea5a4__20210827_20210902', 603), + mksrc('datarepo-dev-da221b1a', 'hca_dev_60ea42e1af4942f58164d641fdb696bc__20210827_20210903', 1145), + mksrc('datarepo-dev-c3d623dc', 'hca_dev_63b5b6c1bbcd487d8c2e0095150c1ecd__20210830_20210903', 11), + mksrc('datarepo-dev-ecb9c129', 'hca_dev_67a3de0945b949c3a068ff4665daa50e__20210827_20210903', 732), + mksrc('datarepo-dev-3545971c', 'hca_dev_71436067ac414acebe1b2fbcc2cb02fa__20210827_20210928', 4), + mksrc('datarepo-dev-12b7a9e1', 'hca_dev_7880637a35a14047b422b5eac2a2a358__20210901_20210903', 366), + mksrc('datarepo-dev-7913b094', 'hca_dev_78b2406dbff246fc8b6120690e602227__20210827_20210902', 216), + mksrc('datarepo-dev-4747d8fe', 'hca_dev_7adede6a0ab745e69b67ffe7466bec1f__20210830_20210903', 1601), + mksrc('datarepo-dev-aba01389', 'hca_dev_7b947aa243a74082afff222a3e3a4635__20210831_20210907', 7), + mksrc('datarepo-dev-bf3a4c8a', 'hca_dev_8185730f411340d39cc3929271784c2b__20210830_20210903', 12), + mksrc('datarepo-dev-560ee3d1', 'hca_dev_83f5188e3bf749569544cea4f8997756__20210929_20211007', 1612), + mksrc('datarepo-dev-bd995e95', 'hca_dev_842605c7375a47c59e2ca71c2c00fcad__20210830_20210903', 8), + mksrc('datarepo-dev-d0772077', 'hca_dev_8787c23889ef4636a57d3167e8b54a80__20210827_20210903', 3), + mksrc('datarepo-dev-8eb2ffd1', 'hca_dev_87d52a86bdc7440cb84d170f7dc346d9__20210830_20210903', 16), + mksrc('datarepo-dev-0c5c20b5', 'hca_dev_8c3c290ddfff4553886854ce45f4ba7f__20210902_20210907', 6640), + mksrc('datarepo-dev-a198b032', 'hca_dev_90bd693340c048d48d76778c103bf545__20210827_20210903', 2245), + mksrc('datarepo-dev-59d37b9a', 'hca_dev_946c5add47d1402a97bba5af97e8bce7__20210831_20210903', 149), + mksrc('datarepo-dev-788c3b52', 'hca_dev_955dfc2ca8c64d04aa4d907610545d11__20210831_20210903', 13), + mksrc('datarepo-dev-4b88b45b', 'hca_dev_962bd805eb894c54bad2008e497d1307__20210830_20210903', 28), + mksrc('datarepo-dev-02c59b72', 'hca_dev_99101928d9b14aafb759e97958ac7403__20210830_20210903', 1190, ma=1), + mksrc('datarepo-dev-a6312a94', 'hca_dev_992aad5e7fab46d9a47ddf715e8cfd24__20210830_20210903', 41), + mksrc('datarepo-dev-75589244', 'hca_dev_996120f9e84f409fa01e732ab58ca8b9__20210827_20210903', 26), + mksrc('datarepo-dev-d4b988d6', 'hca_dev_a004b1501c364af69bbd070c06dbc17d__20210830_20210903', 16, ma=1), + mksrc('datarepo-dev-9ec7beb6', 'hca_dev_a29952d9925e40f48a1c274f118f1f51__20210827_20210902', 26), + mksrc('datarepo-dev-d3d5bbfa', 'hca_dev_a39728aa70a04201b0a281b7badf3e71__20210830_20210903', 33), + mksrc('datarepo-dev-7b7daff7', 'hca_dev_a96b71c078a742d188ce83c78925cfeb__20210827_20210902', 6), + mksrc('datarepo-dev-58610528', 'hca_dev_a9c022b4c7714468b769cabcf9738de3__20210827_20210903', 23), + mksrc('datarepo-dev-1dce87e5', 'hca_dev_ae71be1dddd84feb9bed24c3ddb6e1ad__20210916_20210916', 3515), + mksrc('datarepo-dev-b2004d1c', 'hca_dev_b4a7d12f6c2f40a39e359756997857e3__20210831_20210903', 24), + mksrc('datarepo-dev-0b465564', 'hca_dev_b51f49b40d2e4cbdbbd504cd171fc2fa__20210830_20210903', 193), + mksrc('datarepo-dev-cd97e83a', 'hca_dev_b963bd4b4bc14404842569d74bc636b8__20210827_20210928', 2), + mksrc('datarepo-dev-376d3f4a', 'hca_dev_bd40033154b94fccbff66bb8b079ee1f__20210901_20210903', 18), + mksrc('datarepo-dev-aa783adb', 'hca_dev_c5f4661568de4cf4bbc2a0ae10f08243__20210827_20210928', 1), + mksrc('datarepo-dev-990234a8', 'hca_dev_c6ad8f9bd26a4811b2ba93d487978446__20210827_20210903', 639), + mksrc('datarepo-dev-61b8d081', 'hca_dev_c715cd2fdc7c44a69cd5b6a6d9f075ae__20210827_20210902', 9), + mksrc('datarepo-dev-71926fdc', 'hca_dev_c893cb575c9f4f26931221b85be84313__20210901_20210903', 20), + mksrc('datarepo-dev-2f4bfe5d', 'hca_dev_ccd1f1ba74ce469b9fc9f6faea623358__20210827_20210902', 222), + mksrc('datarepo-dev-aa8357fb', 'hca_dev_ccef38d7aa9240109621c4c7b1182647__20210831_20210903', 1314), + mksrc('datarepo-dev-24e672db', 'hca_dev_cddab57b68684be4806f395ed9dd635a__20210831_20210907', 2545), + mksrc('datarepo-dev-f4cb2365', 'hca_dev_ce33dde2382d448cb6acbfb424644f23__20210827_20210928', 189), + mksrc('datarepo-dev-0d6f73ac', 'hca_dev_d012d4768f8c4ff389d6ebbe22c1b5c1__20210827_20210903', 8), + mksrc('datarepo-dev-5674b4eb', 'hca_dev_d2111fac3fc44f429b6d32cd6a828267__20210830_20210903', 735), + mksrc('datarepo-dev-b3632667', 'hca_dev_d3446f0c30f34a12b7c36af877c7bb2d__20210901_20210903', 40), + mksrc('datarepo-dev-92c3a1de', 'hca_dev_d3a4ceac4d66498497042570c0647a56__20210831_20210903', 14), + mksrc('datarepo-dev-4e88d60b', 'hca_dev_d3ac7c1b53024804b611dad9f89c049d__20210827_20210928', 11), + mksrc('datarepo-dev-dbc582d9', 'hca_dev_dbcd4b1d31bd4eb594e150e8706fa192__20210827_20210902', 84), + mksrc('datarepo-dev-848e2d4f', 'hca_dev_dbd836cfbfc241f0983441cc6c0b235a__20210827_20210902', 1), + mksrc('datarepo-dev-d7517bce', 'hca_dev_dc1a41f69e0942a6959e3be23db6da56__20210827_20210902', 5), + mksrc('datarepo-dev-27ad01e5', 'hca_dev_df88f39f01a84b5b92f43177d6c0f242__20210827_20210928', 1), + mksrc('datarepo-dev-b839d6c7', 'hca_dev_e526d91dcf3a44cb80c5fd7676b55a1d__20210902_20210907', 606), + mksrc('datarepo-dev-3faef568', 'hca_dev_e5d455791f5b48c3b568320d93e7ca72__20210827_20210903', 8), + mksrc('datarepo-dev-e304a8fe', 'hca_dev_e77fed30959d4fadbc15a0a5a85c21d2__20210830_20210903', 333), + mksrc('datarepo-dev-6fdac3db', 'hca_dev_e8808cc84ca0409680f2bba73600cba6__20210902_20210907', 898), + mksrc('datarepo-dev-dbc3e131', 'hca_dev_eaefa1b6dae14414953b17b0427d061e__20210827_20210903', 385), + mksrc('datarepo-dev-b51e6694', 'hca_dev_f48e7c39cc6740559d79bc437892840c__20210830_20211007', 14), + mksrc('datarepo-dev-10f0610a', 'hca_dev_f81efc039f564354aabb6ce819c3d414__20210827_20210903', 4), + mksrc('datarepo-dev-24e9529e', 'hca_dev_f83165c5e2ea4d15a5cf33f3550bffde__20210901_20210908', 7663), + mksrc('datarepo-dev-67240cf2', 'hca_dev_f86f1ab41fbb4510ae353ffd752d4dfc__20210901_20210903', 20), + mksrc('datarepo-dev-e8e0a59a', 'hca_dev_f8aa201c4ff145a4890e840d63459ca2__20210901_20210903', 384), + mksrc('datarepo-dev-96d8e08c', 'hca_dev_faeedcb0e0464be7b1ad80a3eeabb066__20210831_20210903', 62), +] + + def env() -> Mapping[str, Optional[str]]: """ - Returns a dictionary that maps environment variable names to values. The - values are either None or strings. String values can contain references to - other environment variables in the form `{FOO}` where FOO is the name of an - environment variable. See + Returns a dictionary that maps environment variable names to values. The + values are either None or strings. String values can contain references to + other environment variables in the form `{FOO}` where FOO is the name of an + environment variable. See https://docs.python.org/3.8/library/string.html#format-string-syntax @@ -55,125 +164,19 @@ def env() -> Mapping[str, Optional[str]]: 'AZUL_DEPLOYMENT_STAGE': 'dev', 'AZUL_CATALOGS': json.dumps({ - name: dict(atlas=atlas, - internal=internal, - plugins=dict(metadata=dict(name='hca'), - repository=dict(name='tdr'))) - for name, atlas, internal in [ - ('dcp2', 'hca', False), - ('it2', 'hca', True), + f'{name}': dict(atlas='hca', + internal=internal, + plugins=dict(metadata=dict(name='hca'), + repository=dict(name='tdr')), + sources=dcp2_sources) + for name, internal in [ + ('dcp2', False), + ('it2', True) ] }), - 'AZUL_PARTITION_PREFIX_LENGTH': '0', - - 'AZUL_TDR_SOURCES': ','.join(filter(None, [ - mksrc('datarepo-dev-a9252919', 'hca_dev_005d611a14d54fbf846e571a1f874f70__20210827_20210903', 7), - mksrc('datarepo-dev-c148d39c', 'hca_dev_027c51c60719469fa7f5640fe57cbece__20210827_20210902', 8), - mksrc('datarepo-dev-e2ab8487', 'hca_dev_03c6fce7789e4e78a27a664d562bb738__20210902_20210907', 1530), - mksrc('datarepo-dev-37639c56', 'hca_dev_05657a599f9d4bb9b77b24be13aa5cea__20210827_20210928', 185), - mksrc('datarepo-dev-9f4012c9', 'hca_dev_05be4f374506429bb112506444507d62__20210827_20210902', 1544), - mksrc('datarepo-dev-baa2812f', 'hca_dev_0792db3480474e62802c9177c9cd8e28__20210827_20210903', 1450), - mksrc('datarepo-dev-38e08b5c', 'hca_dev_08b794a0519c4516b184c583746254c5__20210901_20210903', 2), - mksrc('datarepo-dev-2749da57', 'hca_dev_091cf39b01bc42e59437f419a66c8a45__20210830_20210903', 20), - mksrc('datarepo-dev-eab7fa76', 'hca_dev_0c3b7785f74d40918616a68757e4c2a8__20210827_20210903', 177), - mksrc('datarepo-dev-fef02a92', 'hca_dev_0d4b87ea6e9e456982e41343e0e3259f__20210827_20210903', 8), - mksrc('datarepo-dev-78bae095', 'hca_dev_0fd8f91862d64b8bac354c53dd601f71__20210830_20210903', 10), - mksrc('datarepo-dev-ff0a2fe7', 'hca_dev_116965f3f09447699d28ae675c1b569c__20210827_20210903', 8), - mksrc('datarepo-dev-4de1b9fd', 'hca_dev_16ed4ad8731946b288596fe1c1d73a82__20210830_20210903', 28), - mksrc('datarepo-dev-135f340c', 'hca_dev_1c6a960d52ac44eab728a59c7ab9dc8e__20210827_20210928', 10), - mksrc('datarepo-dev-86c60513', 'hca_dev_1cd1f41ff81a486ba05b66ec60f81dcf__20210901_20210903', 18), - mksrc('datarepo-dev-f5321179', 'hca_dev_1ce3b3dc02f244a896dad6d107b27a76__20210827_20210903', 421), - mksrc('datarepo-dev-76de829d', 'hca_dev_2043c65a1cf84828a6569e247d4e64f1__20210831_20210907', 1734), - mksrc('datarepo-dev-c808badb', 'hca_dev_2086eb0510b9432bb7f0169ccc49d270__20210827_20210903', 10), - mksrc('datarepo-dev-23782220', 'hca_dev_23587fb31a4a4f58ad74cc9a4cb4c254__20210827_20210909', 1476), - mksrc('datarepo-dev-4c3e6011', 'hca_dev_248fcf0316c64a41b6ccaad4d894ca42__20210907_20210907', 2958), - mksrc('datarepo-dev-1c2c69d9', 'hca_dev_24c654a5caa5440a8f02582921f2db4a__20210830_20210903', 3), - mksrc('datarepo-dev-38f08cd8', 'hca_dev_2a64db431b554639aabb8dba0145689d__20210830_20210903', 10), - mksrc('datarepo-dev-4cf05ce2', 'hca_dev_2a72a4e566b2405abb7c1e463e8febb0__20210901_20210903', 2290), - mksrc('datarepo-dev-3041c2cf', 'hca_dev_2af52a1365cb4973b51339be38f2df3f__20210830_20210903', 10), - mksrc('datarepo-dev-311340f6', 'hca_dev_2d8460958a334f3c97d4585bafac13b4__20210902_20210907', 3589), - mksrc('datarepo-dev-766bfb76', 'hca_dev_2ef3655a973d4d699b4121fa4041eed7__20210827_20210903', 8), - mksrc('datarepo-dev-1720b3c5', 'hca_dev_379ed69ebe0548bcaf5ea7fc589709bf__20210827_20210902', 4), - mksrc('datarepo-dev-ac6efd3f', 'hca_dev_38449aea70b540db84b31e08f32efe34__20210830_20210903', 42), - mksrc('datarepo-dev-40283c27', 'hca_dev_3a69470330844ece9abed935fd5f6748__20210901_20210903', 125), - mksrc('datarepo-dev-b08233fa', 'hca_dev_3cfcdff5dee14a7ba591c09c6e850b11__20210827_20210903', 8), - mksrc('datarepo-dev-bdc9f342', 'hca_dev_3e329187a9c448ec90e3cc45f7c2311c__20210901_20210903', 1001), - mksrc('datarepo-dev-ec07c8d8', 'hca_dev_4037007b0eff4e6db7bd8dd8eec80143__20210831_20210903', 39), - mksrc('datarepo-dev-c0ec174a', 'hca_dev_403c3e7668144a2da5805dd5de38c7ff__20210827_20210903', 63), - mksrc('datarepo-dev-31b3553a', 'hca_dev_414accedeba0440fb721befbc5642bef__20210827_20210903', 4), - mksrc('datarepo-dev-b4789901', 'hca_dev_41fb1734a121461695c73b732c9433c7__20210830_20210903', 12), - mksrc('datarepo-dev-4e5ffd52', 'hca_dev_42d4f8d454224b78adaee7c3c2ef511c__20210830_20210903', 9), - mksrc('datarepo-dev-5ef7f2e2', 'hca_dev_455b46e6d8ea4611861ede720a562ada__20210901_20210903', 74), - mksrc('datarepo-dev-a6c6b953', 'hca_dev_4bec484dca7a47b48d488830e06ad6db__20210830_20210903', 14), - mksrc('datarepo-dev-f31edbc2', 'hca_dev_4d6f6c962a8343d88fe10f53bffd4674__20210901_20210903', 12), - mksrc('datarepo-dev-bb8fbae4', 'hca_dev_51f02950ee254f4b8d0759aa99bb3498__20210827_20210928', 6), - mksrc('datarepo-dev-71de019e', 'hca_dev_520afa10f9d24e93ab7a26c4c863ce18__20210827_20210928', 649), - mksrc('datarepo-dev-ffcf8b00', 'hca_dev_52b29aa4c8d642b4807ab35be94469ca__20210830_20210903', 467), - mksrc('datarepo-dev-f76414c8', 'hca_dev_52d10a60c8d14d068a5eaf0d5c0d5034__20210827_20210902', 176), - mksrc('datarepo-dev-319b80f7', 'hca_dev_577c946d6de54b55a854cd3fde40bff2__20210827_20210903', 7), - mksrc('datarepo-dev-279f1986', 'hca_dev_5ee710d7e2d54fe2818d15f5e31dae32__20210901_20210903', 41), - mksrc('datarepo-dev-0abea017', 'hca_dev_6072616c87944b208f52fb15992ea5a4__20210827_20210902', 603), - mksrc('datarepo-dev-da221b1a', 'hca_dev_60ea42e1af4942f58164d641fdb696bc__20210827_20210903', 1145), - mksrc('datarepo-dev-c3d623dc', 'hca_dev_63b5b6c1bbcd487d8c2e0095150c1ecd__20210830_20210903', 11), - mksrc('datarepo-dev-ecb9c129', 'hca_dev_67a3de0945b949c3a068ff4665daa50e__20210827_20210903', 732), - mksrc('datarepo-dev-3545971c', 'hca_dev_71436067ac414acebe1b2fbcc2cb02fa__20210827_20210928', 4), - mksrc('datarepo-dev-12b7a9e1', 'hca_dev_7880637a35a14047b422b5eac2a2a358__20210901_20210903', 366), - mksrc('datarepo-dev-7913b094', 'hca_dev_78b2406dbff246fc8b6120690e602227__20210827_20210902', 216), - mksrc('datarepo-dev-4747d8fe', 'hca_dev_7adede6a0ab745e69b67ffe7466bec1f__20210830_20210903', 1601), - mksrc('datarepo-dev-aba01389', 'hca_dev_7b947aa243a74082afff222a3e3a4635__20210831_20210907', 7), - mksrc('datarepo-dev-bf3a4c8a', 'hca_dev_8185730f411340d39cc3929271784c2b__20210830_20210903', 12), - mksrc('datarepo-dev-560ee3d1', 'hca_dev_83f5188e3bf749569544cea4f8997756__20210929_20211007', 1612), - mksrc('datarepo-dev-bd995e95', 'hca_dev_842605c7375a47c59e2ca71c2c00fcad__20210830_20210903', 8), - mksrc('datarepo-dev-d0772077', 'hca_dev_8787c23889ef4636a57d3167e8b54a80__20210827_20210903', 3), - mksrc('datarepo-dev-8eb2ffd1', 'hca_dev_87d52a86bdc7440cb84d170f7dc346d9__20210830_20210903', 16), - mksrc('datarepo-dev-0c5c20b5', 'hca_dev_8c3c290ddfff4553886854ce45f4ba7f__20210902_20210907', 6640), - mksrc('datarepo-dev-a198b032', 'hca_dev_90bd693340c048d48d76778c103bf545__20210827_20210903', 2245), - mksrc('datarepo-dev-59d37b9a', 'hca_dev_946c5add47d1402a97bba5af97e8bce7__20210831_20210903', 149), - mksrc('datarepo-dev-788c3b52', 'hca_dev_955dfc2ca8c64d04aa4d907610545d11__20210831_20210903', 13), - mksrc('datarepo-dev-4b88b45b', 'hca_dev_962bd805eb894c54bad2008e497d1307__20210830_20210903', 28), - mksrc('datarepo-dev-02c59b72', 'hca_dev_99101928d9b14aafb759e97958ac7403__20210830_20210903', 1190, ma=1), - mksrc('datarepo-dev-a6312a94', 'hca_dev_992aad5e7fab46d9a47ddf715e8cfd24__20210830_20210903', 41), - mksrc('datarepo-dev-75589244', 'hca_dev_996120f9e84f409fa01e732ab58ca8b9__20210827_20210903', 26), - mksrc('datarepo-dev-d4b988d6', 'hca_dev_a004b1501c364af69bbd070c06dbc17d__20210830_20210903', 16, ma=1), - mksrc('datarepo-dev-9ec7beb6', 'hca_dev_a29952d9925e40f48a1c274f118f1f51__20210827_20210902', 26), - mksrc('datarepo-dev-d3d5bbfa', 'hca_dev_a39728aa70a04201b0a281b7badf3e71__20210830_20210903', 33), - mksrc('datarepo-dev-7b7daff7', 'hca_dev_a96b71c078a742d188ce83c78925cfeb__20210827_20210902', 6), - mksrc('datarepo-dev-58610528', 'hca_dev_a9c022b4c7714468b769cabcf9738de3__20210827_20210903', 23), - mksrc('datarepo-dev-1dce87e5', 'hca_dev_ae71be1dddd84feb9bed24c3ddb6e1ad__20210916_20210916', 3515), - mksrc('datarepo-dev-b2004d1c', 'hca_dev_b4a7d12f6c2f40a39e359756997857e3__20210831_20210903', 24), - mksrc('datarepo-dev-0b465564', 'hca_dev_b51f49b40d2e4cbdbbd504cd171fc2fa__20210830_20210903', 193), - mksrc('datarepo-dev-cd97e83a', 'hca_dev_b963bd4b4bc14404842569d74bc636b8__20210827_20210928', 2), - mksrc('datarepo-dev-376d3f4a', 'hca_dev_bd40033154b94fccbff66bb8b079ee1f__20210901_20210903', 18), - mksrc('datarepo-dev-aa783adb', 'hca_dev_c5f4661568de4cf4bbc2a0ae10f08243__20210827_20210928', 1), - mksrc('datarepo-dev-990234a8', 'hca_dev_c6ad8f9bd26a4811b2ba93d487978446__20210827_20210903', 639), - mksrc('datarepo-dev-61b8d081', 'hca_dev_c715cd2fdc7c44a69cd5b6a6d9f075ae__20210827_20210902', 9), - mksrc('datarepo-dev-71926fdc', 'hca_dev_c893cb575c9f4f26931221b85be84313__20210901_20210903', 20), - mksrc('datarepo-dev-2f4bfe5d', 'hca_dev_ccd1f1ba74ce469b9fc9f6faea623358__20210827_20210902', 222), - mksrc('datarepo-dev-aa8357fb', 'hca_dev_ccef38d7aa9240109621c4c7b1182647__20210831_20210903', 1314), - mksrc('datarepo-dev-24e672db', 'hca_dev_cddab57b68684be4806f395ed9dd635a__20210831_20210907', 2545), - mksrc('datarepo-dev-f4cb2365', 'hca_dev_ce33dde2382d448cb6acbfb424644f23__20210827_20210928', 189), - mksrc('datarepo-dev-0d6f73ac', 'hca_dev_d012d4768f8c4ff389d6ebbe22c1b5c1__20210827_20210903', 8), - mksrc('datarepo-dev-5674b4eb', 'hca_dev_d2111fac3fc44f429b6d32cd6a828267__20210830_20210903', 735), - mksrc('datarepo-dev-b3632667', 'hca_dev_d3446f0c30f34a12b7c36af877c7bb2d__20210901_20210903', 40), - mksrc('datarepo-dev-92c3a1de', 'hca_dev_d3a4ceac4d66498497042570c0647a56__20210831_20210903', 14), - mksrc('datarepo-dev-4e88d60b', 'hca_dev_d3ac7c1b53024804b611dad9f89c049d__20210827_20210928', 11), - mksrc('datarepo-dev-dbc582d9', 'hca_dev_dbcd4b1d31bd4eb594e150e8706fa192__20210827_20210902', 84), - mksrc('datarepo-dev-848e2d4f', 'hca_dev_dbd836cfbfc241f0983441cc6c0b235a__20210827_20210902', 1), - mksrc('datarepo-dev-d7517bce', 'hca_dev_dc1a41f69e0942a6959e3be23db6da56__20210827_20210902', 5), - mksrc('datarepo-dev-27ad01e5', 'hca_dev_df88f39f01a84b5b92f43177d6c0f242__20210827_20210928', 1), - mksrc('datarepo-dev-b839d6c7', 'hca_dev_e526d91dcf3a44cb80c5fd7676b55a1d__20210902_20210907', 606), - mksrc('datarepo-dev-3faef568', 'hca_dev_e5d455791f5b48c3b568320d93e7ca72__20210827_20210903', 8), - mksrc('datarepo-dev-e304a8fe', 'hca_dev_e77fed30959d4fadbc15a0a5a85c21d2__20210830_20210903', 333), - mksrc('datarepo-dev-6fdac3db', 'hca_dev_e8808cc84ca0409680f2bba73600cba6__20210902_20210907', 898), - mksrc('datarepo-dev-dbc3e131', 'hca_dev_eaefa1b6dae14414953b17b0427d061e__20210827_20210903', 385), - mksrc('datarepo-dev-b51e6694', 'hca_dev_f48e7c39cc6740559d79bc437892840c__20210830_20211007', 14), - mksrc('datarepo-dev-10f0610a', 'hca_dev_f81efc039f564354aabb6ce819c3d414__20210827_20210903', 4), - mksrc('datarepo-dev-24e9529e', 'hca_dev_f83165c5e2ea4d15a5cf33f3550bffde__20210901_20210908', 7663), - mksrc('datarepo-dev-67240cf2', 'hca_dev_f86f1ab41fbb4510ae353ffd752d4dfc__20210901_20210903', 20), - mksrc('datarepo-dev-e8e0a59a', 'hca_dev_f8aa201c4ff145a4890e840d63459ca2__20210901_20210903', 384), - mksrc('datarepo-dev-96d8e08c', 'hca_dev_faeedcb0e0464be7b1ad80a3eeabb066__20210831_20210903', 62), - ])), + 'AZUL_PARTITION_PREFIX_LENGTH': '2', + 'AZUL_TDR_SOURCE_LOCATION': 'us-central1', 'AZUL_TDR_SERVICE_URL': 'https://jade.datarepo-dev.broadinstitute.org', 'AZUL_SAM_SERVICE_URL': 'https://sam.dsde-dev.broadinstitute.org', diff --git a/deployments/prod/environment.py b/deployments/prod/environment.py index 5b57ff4c40..bb8aa4bf0d 100644 --- a/deployments/prod/environment.py +++ b/deployments/prod/environment.py @@ -40,48 +40,47 @@ def env() -> Mapping[str, Optional[str]]: 'AZUL_S3_BUCKET': 'edu-ucsc-gi-azul-dcp2-prod-storage-{AZUL_DEPLOYMENT_STAGE}', 'AZUL_CATALOGS': json.dumps({ - **{ - f'{name}{rel}': dict(atlas='hca', - internal=internal, - plugins=dict(metadata=dict(name='hca'), - repository=dict(name='tdr'))) - for rel in (9, 10, 1) - for name, internal in (('dcp', False), ('it', True)) - }, - **{ - name: dict(atlas='lungmap', - internal=internal, - plugins=dict(metadata=dict(name='hca'), - repository=dict(name='tdr'))) - for name, internal in (('lungmap', False), ('it0lungmap', True)) - } + f'{name}': dict(atlas=atlas, + internal=bool(i), + plugins=dict(metadata=dict(name='hca'), + repository=dict(name='tdr')), + sources=sources) + for atlas, names, sources in [ + ( + 'hca', + ['dcp9', 'it9'], + [ + 'tdr:tdr-fp-546ade29:snapshot/hca_prod_20201120_dcp2___20210910_dcp9:', + ] + ), + ( + 'hca', + ['dcp10', 'it10'], + [ + 'tdr:tdr-fp-43194825:snapshot/hca_prod_20201120_dcp2___20211004_dcp10:' + ] + ), + ( + 'hca', + ['dcp1', 'it1'], + [ + 'tdr:broad-datarepo-terra-prod-hca2:snapshot/hca_prod_20201118_dcp1___20201209:', + ] + ), + ( + 'lungmap', + ['lungmap', 'it0lungmap'], + [ + 'tdr:tdr-fp-a02eee6b:snapshot/hca_prod_1bdcecde16be420888f478cd2133d11d__20211013_20211013_lungmap:', + 'tdr:tdr-fp-9f58860b:snapshot/hca_prod_00f056f273ff43ac97ff69ca10e38c89__20211004_lungmap_20211005:', + 'tdr:tdr-fp-ad968fd8:snapshot/hca_prod_2620497955a349b28d2b53e0bdfcb176__20211012_lungmap_20211013_lungmap:' + ] + ) + ] for i, name in enumerate(names) }), 'AZUL_PARTITION_PREFIX_LENGTH': '2', - 'AZUL_TDR_SOURCES': ','.join([ - 'tdr:broad-datarepo-terra-prod-hca2:snapshot/hca_prod_20201118_dcp1___20201209:', - ]), - **{ - f'AZUL_TDR_{catalog.upper()}_SOURCES': ','.join([ - 'tdr:tdr-fp-546ade29:snapshot/hca_prod_20201120_dcp2___20210910_dcp9:', - ]) - for catalog in ('dcp9', 'it9') - }, - **{ - f'AZUL_TDR_{catalog.upper()}_SOURCES': ','.join([ - 'tdr:tdr-fp-43194825:snapshot/hca_prod_20201120_dcp2___20211004_dcp10:' - ]) - for catalog in ('dcp10', 'it10') - }, - **{ - f'AZUL_TDR_{catalog.upper()}_SOURCES': ','.join([ - 'tdr:tdr-fp-a02eee6b:snapshot/hca_prod_1bdcecde16be420888f478cd2133d11d__20211013_20211013_lungmap:', - 'tdr:tdr-fp-9f58860b:snapshot/hca_prod_00f056f273ff43ac97ff69ca10e38c89__20211004_lungmap_20211005:', - 'tdr:tdr-fp-ad968fd8:snapshot/hca_prod_2620497955a349b28d2b53e0bdfcb176__20211012_lungmap_20211013_lungmap:' - ]) - for catalog in ('lungmap', 'it0lungmap') - }, 'AZUL_TDR_SOURCE_LOCATION': 'US', 'AZUL_TDR_SERVICE_URL': 'https://jade-terra.datarepo-prod.broadinstitute.org', 'AZUL_SAM_SERVICE_URL': 'https://sam.dsde-prod.broadinstitute.org', diff --git a/deployments/sandbox/environment.py b/deployments/sandbox/environment.py index 7281823a36..042edea4f8 100644 --- a/deployments/sandbox/environment.py +++ b/deployments/sandbox/environment.py @@ -28,12 +28,121 @@ def mksrc(project, snapshot, subgraphs, ma: int = 0): return f'tdr:{project}:snapshot/{snapshot}:{prefix(subgraphs)}' +dcp2_sources = [ + mksrc('datarepo-dev-a9252919', 'hca_dev_005d611a14d54fbf846e571a1f874f70__20210827_20210903', 7), + mksrc('datarepo-dev-c148d39c', 'hca_dev_027c51c60719469fa7f5640fe57cbece__20210827_20210902', 8), + mksrc('datarepo-dev-e2ab8487', 'hca_dev_03c6fce7789e4e78a27a664d562bb738__20210902_20210907', 1530), + mksrc('datarepo-dev-37639c56', 'hca_dev_05657a599f9d4bb9b77b24be13aa5cea__20210827_20210928', 185), + mksrc('datarepo-dev-9f4012c9', 'hca_dev_05be4f374506429bb112506444507d62__20210827_20210902', 1544), + mksrc('datarepo-dev-baa2812f', 'hca_dev_0792db3480474e62802c9177c9cd8e28__20210827_20210903', 1450), + mksrc('datarepo-dev-38e08b5c', 'hca_dev_08b794a0519c4516b184c583746254c5__20210901_20210903', 2), + mksrc('datarepo-dev-2749da57', 'hca_dev_091cf39b01bc42e59437f419a66c8a45__20210830_20210903', 20), + mksrc('datarepo-dev-eab7fa76', 'hca_dev_0c3b7785f74d40918616a68757e4c2a8__20210827_20210903', 177), + mksrc('datarepo-dev-fef02a92', 'hca_dev_0d4b87ea6e9e456982e41343e0e3259f__20210827_20210903', 8), + mksrc('datarepo-dev-78bae095', 'hca_dev_0fd8f91862d64b8bac354c53dd601f71__20210830_20210903', 10), + mksrc('datarepo-dev-ff0a2fe7', 'hca_dev_116965f3f09447699d28ae675c1b569c__20210827_20210903', 8), + mksrc('datarepo-dev-4de1b9fd', 'hca_dev_16ed4ad8731946b288596fe1c1d73a82__20210830_20210903', 28), + mksrc('datarepo-dev-135f340c', 'hca_dev_1c6a960d52ac44eab728a59c7ab9dc8e__20210827_20210928', 10), + mksrc('datarepo-dev-86c60513', 'hca_dev_1cd1f41ff81a486ba05b66ec60f81dcf__20210901_20210903', 18), + mksrc('datarepo-dev-f5321179', 'hca_dev_1ce3b3dc02f244a896dad6d107b27a76__20210827_20210903', 421), + mksrc('datarepo-dev-76de829d', 'hca_dev_2043c65a1cf84828a6569e247d4e64f1__20210831_20210907', 1734), + mksrc('datarepo-dev-c808badb', 'hca_dev_2086eb0510b9432bb7f0169ccc49d270__20210827_20210903', 10), + mksrc('datarepo-dev-23782220', 'hca_dev_23587fb31a4a4f58ad74cc9a4cb4c254__20210827_20210909', 1476), + mksrc('datarepo-dev-4c3e6011', 'hca_dev_248fcf0316c64a41b6ccaad4d894ca42__20210907_20210907', 2958), + mksrc('datarepo-dev-1c2c69d9', 'hca_dev_24c654a5caa5440a8f02582921f2db4a__20210830_20210903', 3), + mksrc('datarepo-dev-38f08cd8', 'hca_dev_2a64db431b554639aabb8dba0145689d__20210830_20210903', 10), + mksrc('datarepo-dev-4cf05ce2', 'hca_dev_2a72a4e566b2405abb7c1e463e8febb0__20210901_20210903', 2290), + mksrc('datarepo-dev-3041c2cf', 'hca_dev_2af52a1365cb4973b51339be38f2df3f__20210830_20210903', 10), + mksrc('datarepo-dev-311340f6', 'hca_dev_2d8460958a334f3c97d4585bafac13b4__20210902_20210907', 3589), + mksrc('datarepo-dev-766bfb76', 'hca_dev_2ef3655a973d4d699b4121fa4041eed7__20210827_20210903', 8), + mksrc('datarepo-dev-1720b3c5', 'hca_dev_379ed69ebe0548bcaf5ea7fc589709bf__20210827_20210902', 4), + mksrc('datarepo-dev-ac6efd3f', 'hca_dev_38449aea70b540db84b31e08f32efe34__20210830_20210903', 42), + mksrc('datarepo-dev-40283c27', 'hca_dev_3a69470330844ece9abed935fd5f6748__20210901_20210903', 125), + mksrc('datarepo-dev-b08233fa', 'hca_dev_3cfcdff5dee14a7ba591c09c6e850b11__20210827_20210903', 8), + mksrc('datarepo-dev-bdc9f342', 'hca_dev_3e329187a9c448ec90e3cc45f7c2311c__20210901_20210903', 1001), + mksrc('datarepo-dev-ec07c8d8', 'hca_dev_4037007b0eff4e6db7bd8dd8eec80143__20210831_20210903', 39), + mksrc('datarepo-dev-c0ec174a', 'hca_dev_403c3e7668144a2da5805dd5de38c7ff__20210827_20210903', 63), + mksrc('datarepo-dev-31b3553a', 'hca_dev_414accedeba0440fb721befbc5642bef__20210827_20210903', 4), + mksrc('datarepo-dev-b4789901', 'hca_dev_41fb1734a121461695c73b732c9433c7__20210830_20210903', 12), + mksrc('datarepo-dev-4e5ffd52', 'hca_dev_42d4f8d454224b78adaee7c3c2ef511c__20210830_20210903', 9), + mksrc('datarepo-dev-5ef7f2e2', 'hca_dev_455b46e6d8ea4611861ede720a562ada__20210901_20210903', 74), + mksrc('datarepo-dev-a6c6b953', 'hca_dev_4bec484dca7a47b48d488830e06ad6db__20210830_20210903', 14), + mksrc('datarepo-dev-f31edbc2', 'hca_dev_4d6f6c962a8343d88fe10f53bffd4674__20210901_20210903', 12), + mksrc('datarepo-dev-bb8fbae4', 'hca_dev_51f02950ee254f4b8d0759aa99bb3498__20210827_20210928', 6), + mksrc('datarepo-dev-71de019e', 'hca_dev_520afa10f9d24e93ab7a26c4c863ce18__20210827_20210928', 649), + mksrc('datarepo-dev-ffcf8b00', 'hca_dev_52b29aa4c8d642b4807ab35be94469ca__20210830_20210903', 467), + mksrc('datarepo-dev-f76414c8', 'hca_dev_52d10a60c8d14d068a5eaf0d5c0d5034__20210827_20210902', 176), + mksrc('datarepo-dev-319b80f7', 'hca_dev_577c946d6de54b55a854cd3fde40bff2__20210827_20210903', 7), + mksrc('datarepo-dev-279f1986', 'hca_dev_5ee710d7e2d54fe2818d15f5e31dae32__20210901_20210903', 41), + mksrc('datarepo-dev-0abea017', 'hca_dev_6072616c87944b208f52fb15992ea5a4__20210827_20210902', 603), + mksrc('datarepo-dev-da221b1a', 'hca_dev_60ea42e1af4942f58164d641fdb696bc__20210827_20210903', 1145), + mksrc('datarepo-dev-c3d623dc', 'hca_dev_63b5b6c1bbcd487d8c2e0095150c1ecd__20210830_20210903', 11), + mksrc('datarepo-dev-ecb9c129', 'hca_dev_67a3de0945b949c3a068ff4665daa50e__20210827_20210903', 732), + mksrc('datarepo-dev-3545971c', 'hca_dev_71436067ac414acebe1b2fbcc2cb02fa__20210827_20210928', 4), + mksrc('datarepo-dev-12b7a9e1', 'hca_dev_7880637a35a14047b422b5eac2a2a358__20210901_20210903', 366), + mksrc('datarepo-dev-7913b094', 'hca_dev_78b2406dbff246fc8b6120690e602227__20210827_20210902', 216), + mksrc('datarepo-dev-4747d8fe', 'hca_dev_7adede6a0ab745e69b67ffe7466bec1f__20210830_20210903', 1601), + mksrc('datarepo-dev-aba01389', 'hca_dev_7b947aa243a74082afff222a3e3a4635__20210831_20210907', 7), + mksrc('datarepo-dev-bf3a4c8a', 'hca_dev_8185730f411340d39cc3929271784c2b__20210830_20210903', 12), + mksrc('datarepo-dev-560ee3d1', 'hca_dev_83f5188e3bf749569544cea4f8997756__20210929_20211007', 1612), + mksrc('datarepo-dev-bd995e95', 'hca_dev_842605c7375a47c59e2ca71c2c00fcad__20210830_20210903', 8), + mksrc('datarepo-dev-d0772077', 'hca_dev_8787c23889ef4636a57d3167e8b54a80__20210827_20210903', 3), + mksrc('datarepo-dev-8eb2ffd1', 'hca_dev_87d52a86bdc7440cb84d170f7dc346d9__20210830_20210903', 16), + mksrc('datarepo-dev-0c5c20b5', 'hca_dev_8c3c290ddfff4553886854ce45f4ba7f__20210902_20210907', 6640), + mksrc('datarepo-dev-a198b032', 'hca_dev_90bd693340c048d48d76778c103bf545__20210827_20210903', 2245), + mksrc('datarepo-dev-59d37b9a', 'hca_dev_946c5add47d1402a97bba5af97e8bce7__20210831_20210903', 149), + mksrc('datarepo-dev-788c3b52', 'hca_dev_955dfc2ca8c64d04aa4d907610545d11__20210831_20210903', 13), + mksrc('datarepo-dev-4b88b45b', 'hca_dev_962bd805eb894c54bad2008e497d1307__20210830_20210903', 28), + mksrc('datarepo-dev-02c59b72', 'hca_dev_99101928d9b14aafb759e97958ac7403__20210830_20210903', 1190, ma=1), + mksrc('datarepo-dev-a6312a94', 'hca_dev_992aad5e7fab46d9a47ddf715e8cfd24__20210830_20210903', 41), + mksrc('datarepo-dev-75589244', 'hca_dev_996120f9e84f409fa01e732ab58ca8b9__20210827_20210903', 26), + mksrc('datarepo-dev-d4b988d6', 'hca_dev_a004b1501c364af69bbd070c06dbc17d__20210830_20210903', 16, ma=1), + mksrc('datarepo-dev-9ec7beb6', 'hca_dev_a29952d9925e40f48a1c274f118f1f51__20210827_20210902', 26), + mksrc('datarepo-dev-d3d5bbfa', 'hca_dev_a39728aa70a04201b0a281b7badf3e71__20210830_20210903', 33), + mksrc('datarepo-dev-7b7daff7', 'hca_dev_a96b71c078a742d188ce83c78925cfeb__20210827_20210902', 6), + mksrc('datarepo-dev-58610528', 'hca_dev_a9c022b4c7714468b769cabcf9738de3__20210827_20210903', 23), + mksrc('datarepo-dev-1dce87e5', 'hca_dev_ae71be1dddd84feb9bed24c3ddb6e1ad__20210916_20210916', 3515), + mksrc('datarepo-dev-b2004d1c', 'hca_dev_b4a7d12f6c2f40a39e359756997857e3__20210831_20210903', 24), + mksrc('datarepo-dev-0b465564', 'hca_dev_b51f49b40d2e4cbdbbd504cd171fc2fa__20210830_20210903', 193), + mksrc('datarepo-dev-cd97e83a', 'hca_dev_b963bd4b4bc14404842569d74bc636b8__20210827_20210928', 2), + mksrc('datarepo-dev-376d3f4a', 'hca_dev_bd40033154b94fccbff66bb8b079ee1f__20210901_20210903', 18), + mksrc('datarepo-dev-aa783adb', 'hca_dev_c5f4661568de4cf4bbc2a0ae10f08243__20210827_20210928', 1), + mksrc('datarepo-dev-990234a8', 'hca_dev_c6ad8f9bd26a4811b2ba93d487978446__20210827_20210903', 639), + mksrc('datarepo-dev-61b8d081', 'hca_dev_c715cd2fdc7c44a69cd5b6a6d9f075ae__20210827_20210902', 9), + mksrc('datarepo-dev-71926fdc', 'hca_dev_c893cb575c9f4f26931221b85be84313__20210901_20210903', 20), + mksrc('datarepo-dev-2f4bfe5d', 'hca_dev_ccd1f1ba74ce469b9fc9f6faea623358__20210827_20210902', 222), + mksrc('datarepo-dev-aa8357fb', 'hca_dev_ccef38d7aa9240109621c4c7b1182647__20210831_20210903', 1314), + mksrc('datarepo-dev-24e672db', 'hca_dev_cddab57b68684be4806f395ed9dd635a__20210831_20210907', 2545), + mksrc('datarepo-dev-f4cb2365', 'hca_dev_ce33dde2382d448cb6acbfb424644f23__20210827_20210928', 189), + mksrc('datarepo-dev-0d6f73ac', 'hca_dev_d012d4768f8c4ff389d6ebbe22c1b5c1__20210827_20210903', 8), + mksrc('datarepo-dev-5674b4eb', 'hca_dev_d2111fac3fc44f429b6d32cd6a828267__20210830_20210903', 735), + mksrc('datarepo-dev-b3632667', 'hca_dev_d3446f0c30f34a12b7c36af877c7bb2d__20210901_20210903', 40), + mksrc('datarepo-dev-92c3a1de', 'hca_dev_d3a4ceac4d66498497042570c0647a56__20210831_20210903', 14), + mksrc('datarepo-dev-4e88d60b', 'hca_dev_d3ac7c1b53024804b611dad9f89c049d__20210827_20210928', 11), + mksrc('datarepo-dev-dbc582d9', 'hca_dev_dbcd4b1d31bd4eb594e150e8706fa192__20210827_20210902', 84), + mksrc('datarepo-dev-848e2d4f', 'hca_dev_dbd836cfbfc241f0983441cc6c0b235a__20210827_20210902', 1), + mksrc('datarepo-dev-d7517bce', 'hca_dev_dc1a41f69e0942a6959e3be23db6da56__20210827_20210902', 5), + mksrc('datarepo-dev-27ad01e5', 'hca_dev_df88f39f01a84b5b92f43177d6c0f242__20210827_20210928', 1), + mksrc('datarepo-dev-b839d6c7', 'hca_dev_e526d91dcf3a44cb80c5fd7676b55a1d__20210902_20210907', 606), + mksrc('datarepo-dev-3faef568', 'hca_dev_e5d455791f5b48c3b568320d93e7ca72__20210827_20210903', 8), + mksrc('datarepo-dev-e304a8fe', 'hca_dev_e77fed30959d4fadbc15a0a5a85c21d2__20210830_20210903', 333), + mksrc('datarepo-dev-6fdac3db', 'hca_dev_e8808cc84ca0409680f2bba73600cba6__20210902_20210907', 898), + mksrc('datarepo-dev-dbc3e131', 'hca_dev_eaefa1b6dae14414953b17b0427d061e__20210827_20210903', 385), + mksrc('datarepo-dev-b51e6694', 'hca_dev_f48e7c39cc6740559d79bc437892840c__20210830_20211007', 14), + mksrc('datarepo-dev-10f0610a', 'hca_dev_f81efc039f564354aabb6ce819c3d414__20210827_20210903', 4), + mksrc('datarepo-dev-24e9529e', 'hca_dev_f83165c5e2ea4d15a5cf33f3550bffde__20210901_20210908', 7663), + mksrc('datarepo-dev-67240cf2', 'hca_dev_f86f1ab41fbb4510ae353ffd752d4dfc__20210901_20210903', 20), + mksrc('datarepo-dev-e8e0a59a', 'hca_dev_f8aa201c4ff145a4890e840d63459ca2__20210901_20210903', 384), + mksrc('datarepo-dev-96d8e08c', 'hca_dev_faeedcb0e0464be7b1ad80a3eeabb066__20210831_20210903', 62), +] + + def env() -> Mapping[str, Optional[str]]: """ - Returns a dictionary that maps environment variable names to values. The - values are either None or strings. String values can contain references to - other environment variables in the form `{FOO}` where FOO is the name of an - environment variable. See + Returns a dictionary that maps environment variable names to values. The + values are either None or strings. String values can contain references to + other environment variables in the form `{FOO}` where FOO is the name of an + environment variable. See https://docs.python.org/3.8/library/string.html#format-string-syntax @@ -66,125 +175,19 @@ def env() -> Mapping[str, Optional[str]]: 'AZUL_DEPLOYMENT_STAGE': 'sandbox' if is_sandbox else None, 'AZUL_CATALOGS': json.dumps({ - name: dict(atlas=atlas, - internal=internal, - plugins=dict(metadata=dict(name='hca'), - repository=dict(name='tdr'))) - for name, atlas, internal in [ - ('dcp2', 'hca', False), - ('it2', 'hca', True), + f'{name}': dict(atlas='hca', + internal=internal, + plugins=dict(metadata=dict(name='hca'), + repository=dict(name='tdr')), + sources=dcp2_sources) + for name, internal in [ + ('dcp2', False), + ('it2', True) ] }), 'AZUL_PARTITION_PREFIX_LENGTH': '0', - 'AZUL_TDR_SOURCES': ','.join(filter(None, [ - mksrc('datarepo-dev-a9252919', 'hca_dev_005d611a14d54fbf846e571a1f874f70__20210827_20210903', 7), - mksrc('datarepo-dev-c148d39c', 'hca_dev_027c51c60719469fa7f5640fe57cbece__20210827_20210902', 8), - mksrc('datarepo-dev-e2ab8487', 'hca_dev_03c6fce7789e4e78a27a664d562bb738__20210902_20210907', 1530), - mksrc('datarepo-dev-37639c56', 'hca_dev_05657a599f9d4bb9b77b24be13aa5cea__20210827_20210928', 185), - mksrc('datarepo-dev-9f4012c9', 'hca_dev_05be4f374506429bb112506444507d62__20210827_20210902', 1544), - mksrc('datarepo-dev-baa2812f', 'hca_dev_0792db3480474e62802c9177c9cd8e28__20210827_20210903', 1450), - mksrc('datarepo-dev-38e08b5c', 'hca_dev_08b794a0519c4516b184c583746254c5__20210901_20210903', 2), - mksrc('datarepo-dev-2749da57', 'hca_dev_091cf39b01bc42e59437f419a66c8a45__20210830_20210903', 20), - mksrc('datarepo-dev-eab7fa76', 'hca_dev_0c3b7785f74d40918616a68757e4c2a8__20210827_20210903', 177), - mksrc('datarepo-dev-fef02a92', 'hca_dev_0d4b87ea6e9e456982e41343e0e3259f__20210827_20210903', 8), - mksrc('datarepo-dev-78bae095', 'hca_dev_0fd8f91862d64b8bac354c53dd601f71__20210830_20210903', 10), - mksrc('datarepo-dev-ff0a2fe7', 'hca_dev_116965f3f09447699d28ae675c1b569c__20210827_20210903', 8), - mksrc('datarepo-dev-4de1b9fd', 'hca_dev_16ed4ad8731946b288596fe1c1d73a82__20210830_20210903', 28), - mksrc('datarepo-dev-135f340c', 'hca_dev_1c6a960d52ac44eab728a59c7ab9dc8e__20210827_20210928', 10), - mksrc('datarepo-dev-86c60513', 'hca_dev_1cd1f41ff81a486ba05b66ec60f81dcf__20210901_20210903', 18), - mksrc('datarepo-dev-f5321179', 'hca_dev_1ce3b3dc02f244a896dad6d107b27a76__20210827_20210903', 421), - mksrc('datarepo-dev-76de829d', 'hca_dev_2043c65a1cf84828a6569e247d4e64f1__20210831_20210907', 1734), - mksrc('datarepo-dev-c808badb', 'hca_dev_2086eb0510b9432bb7f0169ccc49d270__20210827_20210903', 10), - mksrc('datarepo-dev-23782220', 'hca_dev_23587fb31a4a4f58ad74cc9a4cb4c254__20210827_20210909', 1476), - mksrc('datarepo-dev-4c3e6011', 'hca_dev_248fcf0316c64a41b6ccaad4d894ca42__20210907_20210907', 2958), - mksrc('datarepo-dev-1c2c69d9', 'hca_dev_24c654a5caa5440a8f02582921f2db4a__20210830_20210903', 3), - mksrc('datarepo-dev-38f08cd8', 'hca_dev_2a64db431b554639aabb8dba0145689d__20210830_20210903', 10), - mksrc('datarepo-dev-4cf05ce2', 'hca_dev_2a72a4e566b2405abb7c1e463e8febb0__20210901_20210903', 2290), - mksrc('datarepo-dev-3041c2cf', 'hca_dev_2af52a1365cb4973b51339be38f2df3f__20210830_20210903', 10), - mksrc('datarepo-dev-311340f6', 'hca_dev_2d8460958a334f3c97d4585bafac13b4__20210902_20210907', 3589), - mksrc('datarepo-dev-766bfb76', 'hca_dev_2ef3655a973d4d699b4121fa4041eed7__20210827_20210903', 8), - mksrc('datarepo-dev-1720b3c5', 'hca_dev_379ed69ebe0548bcaf5ea7fc589709bf__20210827_20210902', 4), - mksrc('datarepo-dev-ac6efd3f', 'hca_dev_38449aea70b540db84b31e08f32efe34__20210830_20210903', 42), - mksrc('datarepo-dev-40283c27', 'hca_dev_3a69470330844ece9abed935fd5f6748__20210901_20210903', 125), - mksrc('datarepo-dev-b08233fa', 'hca_dev_3cfcdff5dee14a7ba591c09c6e850b11__20210827_20210903', 8), - mksrc('datarepo-dev-bdc9f342', 'hca_dev_3e329187a9c448ec90e3cc45f7c2311c__20210901_20210903', 1001), - mksrc('datarepo-dev-ec07c8d8', 'hca_dev_4037007b0eff4e6db7bd8dd8eec80143__20210831_20210903', 39), - mksrc('datarepo-dev-c0ec174a', 'hca_dev_403c3e7668144a2da5805dd5de38c7ff__20210827_20210903', 63), - mksrc('datarepo-dev-31b3553a', 'hca_dev_414accedeba0440fb721befbc5642bef__20210827_20210903', 4), - mksrc('datarepo-dev-b4789901', 'hca_dev_41fb1734a121461695c73b732c9433c7__20210830_20210903', 12), - mksrc('datarepo-dev-4e5ffd52', 'hca_dev_42d4f8d454224b78adaee7c3c2ef511c__20210830_20210903', 9), - mksrc('datarepo-dev-5ef7f2e2', 'hca_dev_455b46e6d8ea4611861ede720a562ada__20210901_20210903', 74), - mksrc('datarepo-dev-a6c6b953', 'hca_dev_4bec484dca7a47b48d488830e06ad6db__20210830_20210903', 14), - mksrc('datarepo-dev-f31edbc2', 'hca_dev_4d6f6c962a8343d88fe10f53bffd4674__20210901_20210903', 12), - mksrc('datarepo-dev-bb8fbae4', 'hca_dev_51f02950ee254f4b8d0759aa99bb3498__20210827_20210928', 6), - mksrc('datarepo-dev-71de019e', 'hca_dev_520afa10f9d24e93ab7a26c4c863ce18__20210827_20210928', 649), - mksrc('datarepo-dev-ffcf8b00', 'hca_dev_52b29aa4c8d642b4807ab35be94469ca__20210830_20210903', 467), - mksrc('datarepo-dev-f76414c8', 'hca_dev_52d10a60c8d14d068a5eaf0d5c0d5034__20210827_20210902', 176), - mksrc('datarepo-dev-319b80f7', 'hca_dev_577c946d6de54b55a854cd3fde40bff2__20210827_20210903', 7), - mksrc('datarepo-dev-279f1986', 'hca_dev_5ee710d7e2d54fe2818d15f5e31dae32__20210901_20210903', 41), - mksrc('datarepo-dev-0abea017', 'hca_dev_6072616c87944b208f52fb15992ea5a4__20210827_20210902', 603), - mksrc('datarepo-dev-da221b1a', 'hca_dev_60ea42e1af4942f58164d641fdb696bc__20210827_20210903', 1145), - mksrc('datarepo-dev-c3d623dc', 'hca_dev_63b5b6c1bbcd487d8c2e0095150c1ecd__20210830_20210903', 11), - mksrc('datarepo-dev-ecb9c129', 'hca_dev_67a3de0945b949c3a068ff4665daa50e__20210827_20210903', 732), - mksrc('datarepo-dev-3545971c', 'hca_dev_71436067ac414acebe1b2fbcc2cb02fa__20210827_20210928', 4), - mksrc('datarepo-dev-12b7a9e1', 'hca_dev_7880637a35a14047b422b5eac2a2a358__20210901_20210903', 366), - mksrc('datarepo-dev-7913b094', 'hca_dev_78b2406dbff246fc8b6120690e602227__20210827_20210902', 216), - mksrc('datarepo-dev-4747d8fe', 'hca_dev_7adede6a0ab745e69b67ffe7466bec1f__20210830_20210903', 1601), - mksrc('datarepo-dev-aba01389', 'hca_dev_7b947aa243a74082afff222a3e3a4635__20210831_20210907', 7), - mksrc('datarepo-dev-bf3a4c8a', 'hca_dev_8185730f411340d39cc3929271784c2b__20210830_20210903', 12), - mksrc('datarepo-dev-560ee3d1', 'hca_dev_83f5188e3bf749569544cea4f8997756__20210929_20211007', 1612), - mksrc('datarepo-dev-bd995e95', 'hca_dev_842605c7375a47c59e2ca71c2c00fcad__20210830_20210903', 8), - mksrc('datarepo-dev-d0772077', 'hca_dev_8787c23889ef4636a57d3167e8b54a80__20210827_20210903', 3), - mksrc('datarepo-dev-8eb2ffd1', 'hca_dev_87d52a86bdc7440cb84d170f7dc346d9__20210830_20210903', 16), - mksrc('datarepo-dev-0c5c20b5', 'hca_dev_8c3c290ddfff4553886854ce45f4ba7f__20210902_20210907', 6640), - mksrc('datarepo-dev-a198b032', 'hca_dev_90bd693340c048d48d76778c103bf545__20210827_20210903', 2245), - mksrc('datarepo-dev-59d37b9a', 'hca_dev_946c5add47d1402a97bba5af97e8bce7__20210831_20210903', 149), - mksrc('datarepo-dev-788c3b52', 'hca_dev_955dfc2ca8c64d04aa4d907610545d11__20210831_20210903', 13), - mksrc('datarepo-dev-4b88b45b', 'hca_dev_962bd805eb894c54bad2008e497d1307__20210830_20210903', 28), - mksrc('datarepo-dev-02c59b72', 'hca_dev_99101928d9b14aafb759e97958ac7403__20210830_20210903', 1190, ma=1), - mksrc('datarepo-dev-a6312a94', 'hca_dev_992aad5e7fab46d9a47ddf715e8cfd24__20210830_20210903', 41), - mksrc('datarepo-dev-75589244', 'hca_dev_996120f9e84f409fa01e732ab58ca8b9__20210827_20210903', 26), - mksrc('datarepo-dev-d4b988d6', 'hca_dev_a004b1501c364af69bbd070c06dbc17d__20210830_20210903', 16, ma=1), - mksrc('datarepo-dev-9ec7beb6', 'hca_dev_a29952d9925e40f48a1c274f118f1f51__20210827_20210902', 26), - mksrc('datarepo-dev-d3d5bbfa', 'hca_dev_a39728aa70a04201b0a281b7badf3e71__20210830_20210903', 33), - mksrc('datarepo-dev-7b7daff7', 'hca_dev_a96b71c078a742d188ce83c78925cfeb__20210827_20210902', 6), - mksrc('datarepo-dev-58610528', 'hca_dev_a9c022b4c7714468b769cabcf9738de3__20210827_20210903', 23), - mksrc('datarepo-dev-1dce87e5', 'hca_dev_ae71be1dddd84feb9bed24c3ddb6e1ad__20210916_20210916', 3515), - mksrc('datarepo-dev-b2004d1c', 'hca_dev_b4a7d12f6c2f40a39e359756997857e3__20210831_20210903', 24), - mksrc('datarepo-dev-0b465564', 'hca_dev_b51f49b40d2e4cbdbbd504cd171fc2fa__20210830_20210903', 193), - mksrc('datarepo-dev-cd97e83a', 'hca_dev_b963bd4b4bc14404842569d74bc636b8__20210827_20210928', 2), - mksrc('datarepo-dev-376d3f4a', 'hca_dev_bd40033154b94fccbff66bb8b079ee1f__20210901_20210903', 18), - mksrc('datarepo-dev-aa783adb', 'hca_dev_c5f4661568de4cf4bbc2a0ae10f08243__20210827_20210928', 1), - mksrc('datarepo-dev-990234a8', 'hca_dev_c6ad8f9bd26a4811b2ba93d487978446__20210827_20210903', 639), - mksrc('datarepo-dev-61b8d081', 'hca_dev_c715cd2fdc7c44a69cd5b6a6d9f075ae__20210827_20210902', 9), - mksrc('datarepo-dev-71926fdc', 'hca_dev_c893cb575c9f4f26931221b85be84313__20210901_20210903', 20), - mksrc('datarepo-dev-2f4bfe5d', 'hca_dev_ccd1f1ba74ce469b9fc9f6faea623358__20210827_20210902', 222), - mksrc('datarepo-dev-aa8357fb', 'hca_dev_ccef38d7aa9240109621c4c7b1182647__20210831_20210903', 1314), - mksrc('datarepo-dev-24e672db', 'hca_dev_cddab57b68684be4806f395ed9dd635a__20210831_20210907', 2545), - mksrc('datarepo-dev-f4cb2365', 'hca_dev_ce33dde2382d448cb6acbfb424644f23__20210827_20210928', 189), - mksrc('datarepo-dev-0d6f73ac', 'hca_dev_d012d4768f8c4ff389d6ebbe22c1b5c1__20210827_20210903', 8), - mksrc('datarepo-dev-5674b4eb', 'hca_dev_d2111fac3fc44f429b6d32cd6a828267__20210830_20210903', 735), - mksrc('datarepo-dev-b3632667', 'hca_dev_d3446f0c30f34a12b7c36af877c7bb2d__20210901_20210903', 40), - mksrc('datarepo-dev-92c3a1de', 'hca_dev_d3a4ceac4d66498497042570c0647a56__20210831_20210903', 14), - mksrc('datarepo-dev-4e88d60b', 'hca_dev_d3ac7c1b53024804b611dad9f89c049d__20210827_20210928', 11), - mksrc('datarepo-dev-dbc582d9', 'hca_dev_dbcd4b1d31bd4eb594e150e8706fa192__20210827_20210902', 84), - mksrc('datarepo-dev-848e2d4f', 'hca_dev_dbd836cfbfc241f0983441cc6c0b235a__20210827_20210902', 1), - mksrc('datarepo-dev-d7517bce', 'hca_dev_dc1a41f69e0942a6959e3be23db6da56__20210827_20210902', 5), - mksrc('datarepo-dev-27ad01e5', 'hca_dev_df88f39f01a84b5b92f43177d6c0f242__20210827_20210928', 1), - mksrc('datarepo-dev-b839d6c7', 'hca_dev_e526d91dcf3a44cb80c5fd7676b55a1d__20210902_20210907', 606), - mksrc('datarepo-dev-3faef568', 'hca_dev_e5d455791f5b48c3b568320d93e7ca72__20210827_20210903', 8), - mksrc('datarepo-dev-e304a8fe', 'hca_dev_e77fed30959d4fadbc15a0a5a85c21d2__20210830_20210903', 333), - mksrc('datarepo-dev-6fdac3db', 'hca_dev_e8808cc84ca0409680f2bba73600cba6__20210902_20210907', 898), - mksrc('datarepo-dev-dbc3e131', 'hca_dev_eaefa1b6dae14414953b17b0427d061e__20210827_20210903', 385), - mksrc('datarepo-dev-b51e6694', 'hca_dev_f48e7c39cc6740559d79bc437892840c__20210830_20211007', 14), - mksrc('datarepo-dev-10f0610a', 'hca_dev_f81efc039f564354aabb6ce819c3d414__20210827_20210903', 4), - mksrc('datarepo-dev-24e9529e', 'hca_dev_f83165c5e2ea4d15a5cf33f3550bffde__20210901_20210908', 7663), - mksrc('datarepo-dev-67240cf2', 'hca_dev_f86f1ab41fbb4510ae353ffd752d4dfc__20210901_20210903', 20), - mksrc('datarepo-dev-e8e0a59a', 'hca_dev_f8aa201c4ff145a4890e840d63459ca2__20210901_20210903', 384), - mksrc('datarepo-dev-96d8e08c', 'hca_dev_faeedcb0e0464be7b1ad80a3eeabb066__20210831_20210903', 62), - ])), 'AZUL_TDR_SOURCE_LOCATION': 'us-central1', 'AZUL_TDR_SERVICE_URL': 'https://jade.datarepo-dev.broadinstitute.org', 'AZUL_SAM_SERVICE_URL': 'https://sam.dsde-dev.broadinstitute.org', diff --git a/environment.py b/environment.py index cc3359963c..99421f60af 100644 --- a/environment.py +++ b/environment.py @@ -39,6 +39,10 @@ def env() -> Mapping[str, Optional[str]]: # plugin_type: {'name'=plugin_package}, # ... # } + # 'sources': [ + # source, + # ... + # ] # } # } # @@ -373,54 +377,6 @@ def env() -> Mapping[str, Optional[str]]: # 'TF_DATA_DIR': '{project_root}/deployments/.active/.terraform.{AWS_PROFILE}', - # Identifies the canned staging area to index. - # - # The syntax in EBNF is: - # - # sources = source (',', source )* ; - # - # source = GitHub URL, - # ':', [ UUID prefix [ '/', Partition prefix length ] ] ; - # - # Example: - # - # https://github.com/HumanCellAtlas/schema-test-data/tree/de355ca/tests: - # - # The GitHub URL must have the syntax - # - # 'https://github.com/', , '/', , '/tree/', , ['/', ] ; - # - # `ref` can be a branch, tag, or commit SHA. If `ref` contains special - # characters like `/`, '?` or `#` they must be URL-encoded. - # - 'azul_canned_sources': None, - - # Identifies the Terra Data Repository datasets or snapshots to index. - # - # The syntax in EBNF is: - # - # sources = source (',', source )* ; - # - # source = 'tdr:', Google Cloud project name, - # ':', ( 'dataset' | 'snapshot' ), - # '/', TDR dataset or snapshot name, - # ':', [ UUID prefix [ '/', Partition prefix length ] ] ; - # - # Example: - # - # tdr:broad-jade-dev-data:dataset/hca_mvp:2/1,tdr:broad-jade-dev-data:snapshot/hca_mvp:2 - # - # To configure a source specific to a particular catalog, include the - # catalog name in all upper case in the variable name. For example, to - # configure a source specific to a catalog named `bar0`, use - # - # AZUL_TDR_BAR0_SOURCES - # - # If defined, the catalog-specific variable will be used, otherwise this - # variable will be used. - # - 'AZUL_TDR_SOURCES': None, - # BigQuery dataset location of the TDR snapshots the deployment is # configured to index. All configured snapshots must reside in the same # location. diff --git a/scripts/generate_openapi_document.py b/scripts/generate_openapi_document.py index a70f9eacff..7e98ee2147 100644 --- a/scripts/generate_openapi_document.py +++ b/scripts/generate_openapi_document.py @@ -22,7 +22,8 @@ def main(): atlas='hca', internal=False, plugins=dict(metadata=config.Catalog.Plugin(name='hca'), - repository=config.Catalog.Plugin(name='tdr'))) + repository=config.Catalog.Plugin(name='tdr')), + sources=set()) } # To create a normalized OpenAPI document, we patch any diff --git a/scripts/post_deploy.py b/scripts/post_deploy.py index d1bf83592b..2a22724815 100644 --- a/scripts/post_deploy.py +++ b/scripts/post_deploy.py @@ -36,7 +36,7 @@ def verify_sources(): if catalog.plugins[RepositoryPlugin.type_name()].name == 'tdr' } assert tdr_catalogs, tdr_catalogs - for source in set(chain(*map(config.tdr_sources, tdr_catalogs))): + for source in set(chain(*map(config.sources, tdr_catalogs))): source = TDRSourceSpec.parse(source) verify_source(source) diff --git a/src/azul/__init__.py b/src/azul/__init__.py index 5bfbddb5e2..9ce941dc28 100644 --- a/src/azul/__init__.py +++ b/src/azul/__init__.py @@ -189,19 +189,8 @@ def data_portal_name(self): def dss_endpoint(self) -> Optional[str]: return self.environ.get('AZUL_DSS_ENDPOINT') - def canned_sources(self, catalog: CatalogName) -> AbstractSet[str]: - try: - sources = self.environ[f'azul_canned_{catalog.lower()}_sources'] - except KeyError: - sources = self.environ['azul_canned_sources'] - return frozenset(sources.split(',')) - - def tdr_sources(self, catalog: CatalogName) -> AbstractSet[str]: - try: - sources = self.environ[f'AZUL_TDR_{catalog.upper()}_SOURCES'] - except KeyError: - sources = self.environ['AZUL_TDR_SOURCES'] - return frozenset(sources.split(',')) + def sources(self, catalog: CatalogName) -> AbstractSet[str]: + return config.catalogs[catalog].sources @property def tdr_source_location(self) -> str: @@ -529,6 +518,7 @@ class Plugin: atlas: str internal: bool plugins: Mapping[str, Plugin] + sources: set _it_catalog_re: ClassVar[re.Pattern] = re.compile(r'it[\d]+') @@ -560,7 +550,8 @@ def from_json(cls, name: str, spec: JSON) -> 'Config.Catalog': return cls(name=name, atlas=spec['atlas'], internal=spec['internal'], - plugins=plugins) + plugins=plugins, + sources=set(spec['sources'])) @cached_property def catalogs(self) -> Mapping[CatalogName, Catalog]: diff --git a/src/azul/plugins/repository/canned/__init__.py b/src/azul/plugins/repository/canned/__init__.py index eb4accef30..83bb849e01 100644 --- a/src/azul/plugins/repository/canned/__init__.py +++ b/src/azul/plugins/repository/canned/__init__.py @@ -79,7 +79,7 @@ def create(cls, catalog: CatalogName) -> RepositoryPlugin: return cls( frozenset( SimpleSourceSpec.parse(name).effective - for name in config.canned_sources(catalog) + for name in config.sources(catalog) ) ) diff --git a/src/azul/plugins/repository/tdr/__init__.py b/src/azul/plugins/repository/tdr/__init__.py index 0dd9281e7e..89fe2b7c63 100644 --- a/src/azul/plugins/repository/tdr/__init__.py +++ b/src/azul/plugins/repository/tdr/__init__.py @@ -169,7 +169,7 @@ class Plugin(RepositoryPlugin[TDRSourceSpec, TDRSourceRef]): def create(cls, catalog: CatalogName) -> 'RepositoryPlugin': return cls(sources=frozenset( TDRSourceSpec.parse(spec).effective - for spec in config.tdr_sources(catalog)) + for spec in config.sources(catalog)) ) @property diff --git a/src/azul/portal_service.py b/src/azul/portal_service.py index aa80df44cd..1ae352b374 100644 --- a/src/azul/portal_service.py +++ b/src/azul/portal_service.py @@ -65,7 +65,7 @@ def catalog_source(self): # https://github.com/DataBiosphere/azul/issues/2716 catalog = config.default_catalog md5 = hashlib.md5() - for source in sorted(config.tdr_sources(catalog)): + for source in sorted(config.sources(catalog)): md5.update(source.encode()) return md5.hexdigest() diff --git a/test/azul_test_case.py b/test/azul_test_case.py index 9539d2ec47..0dcf6b73cc 100644 --- a/test/azul_test_case.py +++ b/test/azul_test_case.py @@ -196,7 +196,8 @@ def _reset_moto(self): atlas='hca', internal=False, plugins=dict(metadata=config.Catalog.Plugin(name='hca'), - repository=config.Catalog.Plugin(name='dss'))) + repository=config.Catalog.Plugin(name='dss')), + sources=set()) } _catalog_mock = None diff --git a/test/integration_test.py b/test/integration_test.py index 9e4e70123b..f2bf6d9fc0 100644 --- a/test/integration_test.py +++ b/test/integration_test.py @@ -792,7 +792,7 @@ def _test_managed_access(self, sourceId=self._tdr_client.lookup_source(source_spec).id) for source_spec in ( TDRSourceSpec.parse(source).effective - for source in config.tdr_sources(catalog) + for source in config.sources(catalog) ) } configured_source_ids = {source['sourceId'] for source in configured_sources} @@ -1220,16 +1220,16 @@ def test_can_bundle_configured_catalogs(self): repository=catalog.plugins['repository']): self._test_catalog(catalog) - canned_repo = 'https://github.com/HumanCellAtlas/schema-test-data/tree/master/tests' - - @mock.patch.dict(os.environ, azul_canned_sources=f'{canned_repo}:') def test_can_bundle_canned_repository(self): mock_catalog = config.Catalog(name='testcanned', atlas='hca', internal=True, plugins={ 'metadata': config.Catalog.Plugin(name='hca'), - 'repository': config.Catalog.Plugin(name='canned') + 'repository': config.Catalog.Plugin(name='canned'), + }, + sources={ + 'https://github.com/HumanCellAtlas/schema-test-data/tree/master/tests:' }) with mock.patch.object(azul.Config, diff --git a/test/service/test_repository_proxy.py b/test/service/test_repository_proxy.py index 12f8f83326..4042041e34 100644 --- a/test/service/test_repository_proxy.py +++ b/test/service/test_repository_proxy.py @@ -100,23 +100,22 @@ def assertUrlEqual(self, a: furl, b: furl): @mock.patch.object(SourceService, '_put', new=MagicMock()) @mock.patch.object(SourceService, '_get') class TestTDRRepositoryProxy(RepositoryPluginTestCase): + mock_service_url = f'https://serpentine.datarepo-dev.broadinstitute.net.test.{config.domain_name}' + mock_source_names = ['mock_snapshot_1', 'mock_snapshot_2'] + make_mock_source_spec = 'tdr:mock:snapshot/{}:'.format + mock_sources = set(map(make_mock_source_spec, mock_source_names)) + catalog = 'testtdr' catalog_config = { catalog: config.Catalog(name=catalog, atlas='hca', internal=False, plugins=dict(metadata=config.Catalog.Plugin(name='hca'), - repository=config.Catalog.Plugin(name='tdr'))) + repository=config.Catalog.Plugin(name='tdr')), + sources=mock_sources) } - mock_service_url = f'https://serpentine.datarepo-dev.broadinstitute.net.test.{config.domain_name}' - mock_source_names = ['mock_snapshot_1', 'mock_snapshot_2'] - make_mock_source_spec = 'tdr:mock:snapshot/{}:'.format - mock_source_specs = ','.join(map(make_mock_source_spec, mock_source_names)) - - @mock.patch.dict(os.environ, - AZUL_TDR_SERVICE_URL=mock_service_url, - AZUL_TDR_SOURCES=mock_source_specs) + @mock.patch.dict(os.environ, AZUL_TDR_SERVICE_URL=mock_service_url) @mock.patch.object(TerraClient, '_http_client', AuthorizedHttp(MagicMock(), @@ -174,8 +173,6 @@ def test_repository_files_proxy(self, mock_get_cached_sources): response = dict(response.headers) self.assertUrlEqual(pre_signed_gs, response['Location']) - @mock.patch.dict(os.environ, - {f'AZUL_TDR_{catalog.upper()}_SOURCES': mock_source_specs}) def test_list_sources(self, mock_get_cached_sources, ):