diff --git a/.github/workflows/commit_manifest.yml b/.github/workflows/commit_manifest.yml index fbc98d75178..53596568d7a 100644 --- a/.github/workflows/commit_manifest.yml +++ b/.github/workflows/commit_manifest.yml @@ -1,3 +1,5 @@ +name: Commit Manifest + on: workflow_dispatch: push: @@ -10,7 +12,11 @@ concurrency: jobs: commit_manifest: - runs-on: [ self-hosted, linux, spellbook ] + runs-on: [ self-hosted, linux, spellbook-trino ] + strategy: + matrix: + engine: [ 'dunesql', 'spark' ] + max-parallel: 1 steps: - uses: actions/setup-python@v3 @@ -21,25 +27,40 @@ jobs: - name: Add git_sha to schema run: "/runner/change_schema.sh wizard" - + + - name: Setup variables + run: | + if [[ "${{ matrix.engine }}" == "dunesql" ]]; then + printf "Using dunesql engine\n" + echo "PROFILE=--profile dunesql" >> $GITHUB_ENV + echo "COMPILE_TAG=--select tag:dunesql" >> $GITHUB_ENV + echo "S3_LOCATION=manifest-spellbook-dunesql" >> $GITHUB_ENV + elif [[ "${{ matrix.engine }}" == "spark" ]]; then + printf "Using spark engine\n" + echo "PROFILE=--profile spark" >> $GITHUB_ENV + echo "COMPILE_TAG=--exclude tag:dunesql" >> $GITHUB_ENV + echo "S3_LOCATION=manifest-spellbook" >> $GITHUB_ENV + echo + else + echo "Unknown engine: ${{ matrix.engine }}" + exit 1 + fi + - name: dbt dependencies run: "dbt deps" - name: dbt compile to create prod manifest from main - run: "dbt compile --target-path ." + run: "dbt compile --target-path . $PROFILE $COMPILE_TAG" - name: copy old manifest locally - run: "aws s3 cp s3://manifest-spellbook/manifest.json previous_manifest.json" - - - name: copy old manifest to s3 - run: "aws s3 cp previous_manifest.json s3://manifest-spellbook/previous_manifest.json" + run: "aws s3 cp s3://$S3_LOCATION/manifest.json s3://$S3_LOCATION/previous_manifest.json" - name: Set environment variables run: | echo "GIT_SHA=$(echo ${{ github.sha }} | tr - _ | cut -c1-7)" >> $GITHUB_ENV - name: upload git_sha versioned previous manifest (for manual catchup if jobs fail) - run: "aws s3 cp previous_manifest.json s3://manifest-spellbook/manifest_$GIT_SHA.json" + run: "aws s3 cp s3://$S3_LOCATION/manifest.json s3://$S3_LOCATION/manifest_$GIT_SHA.json" - name: upload manifest - run: "aws s3 cp manifest.json s3://manifest-spellbook/manifest.json" \ No newline at end of file + run: "aws s3 cp manifest.json s3://$S3_LOCATION/manifest.json" \ No newline at end of file diff --git a/.github/workflows/dbt_slim_ci.yml b/.github/workflows/dbt_slim_ci.yml index 1ab20d4d459..9713def25c6 100644 --- a/.github/workflows/dbt_slim_ci.yml +++ b/.github/workflows/dbt_slim_ci.yml @@ -16,49 +16,72 @@ concurrency: jobs: dbt-test: - runs-on: [ self-hosted, linux, spellbook ] + runs-on: [ self-hosted, linux, spellbook-trino ] + strategy: + matrix: + engine: [ 'dunesql', 'spark' ] + max-parallel: 1 timeout-minutes: 90 steps: - name: Check out repository code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - - name: Set environment variables + - name: Setup variables run: | + if [[ "${{ matrix.engine }}" == "dunesql" ]]; then + printf "Using dunesql engine\n" + echo "PROFILE=--profile dunesql" >> $GITHUB_ENV + echo "TAG=,tag:dunesql" >> $GITHUB_ENV + echo "COMPILE_TAG=--select tag:dunesql" >> $GITHUB_ENV + echo "S3_LOCATION=manifest-spellbook-dunesql" >> $GITHUB_ENV + elif [[ "${{ matrix.engine }}" == "spark" ]]; then + printf "Using spark engine\n" + echo "PROFILE=--profile spark" >> $GITHUB_ENV + echo "EXCLUDE=tag:dunesql" >> $GITHUB_ENV + echo "SINGLE_EXCLUDE=--exclude tag:dunesql" >> $GITHUB_ENV + echo "COMPILE_TAG=--exclude tag:dunesql" >> $GITHUB_ENV + echo "S3_LOCATION=manifest-spellbook" >> $GITHUB_ENV + echo + else + echo "Unknown engine: ${{ matrix.engine }}" + exit 1 + fi echo "GIT_SHA=$(echo ${{ github.sha }} | tr - _ | cut -c1-8)" >> $GITHUB_ENV - name: Add git_sha to schema - run: "/runner/change_schema.sh git_$GIT_SHA" + run: "/runner/change_schema.sh git_${{ matrix.engine }}_$GIT_SHA" - name: Get latest manifest - run: "aws s3 cp s3://manifest-spellbook/manifest.json manifest.json" + run: "aws s3 cp s3://$S3_LOCATION/manifest.json manifest.json" - name: dbt dependencies run: "dbt deps" - name: dbt compile to create manifest to compare to - run: "dbt compile" + run: "dbt compile $PROFILE $COMPILE_TAG" - name: dbt seed - run: "dbt seed --select state:modified --state ." + run: "dbt seed $PROFILE --select state:modified$TAG --state ." - name: dbt run initial model(s) - run: "dbt -x run --select state:modified --exclude tag:prod_exclude --defer --state ." + run: "dbt -x run $PROFILE --select state:modified$TAG --exclude $EXCLUDE tag:prod_exclude --defer --state ." - name: dbt test initial model(s) - run: "dbt test --select state:new state:modified --exclude tag:prod_exclude --defer --state ." + run: "dbt test $PROFILE --select state:new$TAG state:modified$TAG --exclude $EXCLUDE tag:prod_exclude --defer --state ." - name: Set environment variable for incremental model count run: | - echo "INC_MODEL_COUNT=$(echo dbt ls --select state:modified,config.materialized:incremental --state . --resource-type model | wc -l)" >> $GITHUB_ENV + echo "INC_MODEL_COUNT=$(echo dbt ls $PROFILE --select state:modified,config.materialized:incremental$TAG -SINGLE_EXCLUDE --state . --resource-type model | wc -l)" >> $GITHUB_ENV - name: dbt run incremental model(s) if applicable if: env.INC_MODEL_COUNT > 0 - run: "dbt run --select state:modified,config.materialized:incremental --exclude tag:prod_exclude --defer --state ." + run: "dbt run $PROFILE --select state:modified,config.materialized:incremental$TAG --exclude $EXCLUDE tag:prod_exclude --defer --state ." - name: dbt test incremental model(s) if applicable if: env.INC_MODEL_COUNT > 0 - run: "dbt test --select state:modified,config.materialized:incremental --exclude tag:prod_exclude --defer --state ." + run: "dbt test $PROFILE --select state:modified,config.materialized:incremental$TAG --exclude $EXCLUDE tag:prod_exclude --defer --state ." - name: Run DuneSQL Check + if: matrix.engine != 'dunesql' run: "/runner/dunesql_check.py --schema test_schema --pr_schema git_$GIT_SHA" diff --git a/Pipfile b/Pipfile index 265a3d4a68f..57c114c9dc0 100644 --- a/Pipfile +++ b/Pipfile @@ -8,7 +8,7 @@ dbt-databricks = "===1.4.1" numpy = "2.0.12" pre-commit = "2.20.0" pytest = "7.1.3" - +dbt-trino = "1.4.2" [requires] python_version = "3.9" diff --git a/Pipfile.lock b/Pipfile.lock index 78734066ecc..efe355a0112 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "730b48a7623517788166f4fa3e97af4ed6e0a2d99f899af68b5351e6b59615b7" + "sha256": "00675d66cae3371e3015454a288b28d400b32b69ce8b9e084b1cd6d01dbd4b75" }, "pipfile-spec": 6, "requires": { @@ -217,7 +217,7 @@ "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df", "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab" ], - "markers": "python_version >= '3.7'", + "markers": "python_full_version >= '3.7.0'", "version": "==3.1.0" }, "click": { @@ -238,11 +238,11 @@ }, "databricks-sql-connector": { "hashes": [ - "sha256:24fa7bee68b4d7c3e7abfd374e2579d1533da82d7806db5fab93d6f70fc9cf2a", - "sha256:d791a8a5eb8ced1669139e4dce290f1a3667b980e23d640bda44aae71fff53a4" + "sha256:0658a48662b166382f004b5cfb96d133ec5d51f5d3ea1e917f8e5df7bc9afddf", + "sha256:e814c835827d12e84e401d0ad6824b427e9abeedd1fc800950e7b428d2145f29" ], "markers": "python_full_version >= '3.7.1' and python_full_version < '4.0.0'", - "version": "==2.5.2" + "version": "==2.6.2" }, "dbt-core": { "hashes": [ @@ -290,6 +290,14 @@ "markers": "python_version >= '3.7'", "version": "==1.4.2" }, + "dbt-trino": { + "hashes": [ + "sha256:b50a7108641460ddc5dd91c1e46fd160763debf1fc3acd2a6c9134cf06afd97d", + "sha256:c1e7db0c8c35fe6d6d974f1a6e83169535a2f3aa5072e18eb2a7f4c03423a0ab" + ], + "index": "pypi", + "version": "==1.4.2" + }, "distlib": { "hashes": [ "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46", @@ -315,11 +323,11 @@ }, "filelock": { "hashes": [ - "sha256:ad98852315c2ab702aeb628412cbf7e95b7ce8c3bf9565670b4eaecf1db370a9", - "sha256:fc03ae43288c013d2ea83c8597001b1129db351aad9c57fe2409327916b8e718" + "sha256:002740518d8aa59a26b0c76e10fb8c6e15eae825d34b6fdf670333fd7b938d81", + "sha256:cbb791cdea2a72f23da6ac5b5269ab0a0d161e9ef0100e653b69049a7706d1ec" ], "markers": "python_version >= '3.7'", - "version": "==3.12.0" + "version": "==3.12.2" }, "future": { "hashes": [ @@ -328,72 +336,6 @@ "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.18.3" }, - "greenlet": { - "hashes": [ - "sha256:03a8f4f3430c3b3ff8d10a2a86028c660355ab637cee9333d63d66b56f09d52a", - "sha256:0bf60faf0bc2468089bdc5edd10555bab6e85152191df713e2ab1fcc86382b5a", - "sha256:18a7f18b82b52ee85322d7a7874e676f34ab319b9f8cce5de06067384aa8ff43", - "sha256:18e98fb3de7dba1c0a852731c3070cf022d14f0d68b4c87a19cc1016f3bb8b33", - "sha256:1a819eef4b0e0b96bb0d98d797bef17dc1b4a10e8d7446be32d1da33e095dbb8", - "sha256:26fbfce90728d82bc9e6c38ea4d038cba20b7faf8a0ca53a9c07b67318d46088", - "sha256:2780572ec463d44c1d3ae850239508dbeb9fed38e294c68d19a24d925d9223ca", - "sha256:283737e0da3f08bd637b5ad058507e578dd462db259f7f6e4c5c365ba4ee9343", - "sha256:2d4686f195e32d36b4d7cf2d166857dbd0ee9f3d20ae349b6bf8afc8485b3645", - "sha256:2dd11f291565a81d71dab10b7033395b7a3a5456e637cf997a6f33ebdf06f8db", - "sha256:30bcf80dda7f15ac77ba5af2b961bdd9dbc77fd4ac6105cee85b0d0a5fcf74df", - "sha256:32e5b64b148966d9cccc2c8d35a671409e45f195864560829f395a54226408d3", - "sha256:36abbf031e1c0f79dd5d596bfaf8e921c41df2bdf54ee1eed921ce1f52999a86", - "sha256:3a06ad5312349fec0ab944664b01d26f8d1f05009566339ac6f63f56589bc1a2", - "sha256:3a51c9751078733d88e013587b108f1b7a1fb106d402fb390740f002b6f6551a", - "sha256:3c9b12575734155d0c09d6c3e10dbd81665d5c18e1a7c6597df72fd05990c8cf", - "sha256:3f6ea9bd35eb450837a3d80e77b517ea5bc56b4647f5502cd28de13675ee12f7", - "sha256:4b58adb399c4d61d912c4c331984d60eb66565175cdf4a34792cd9600f21b394", - "sha256:4d2e11331fc0c02b6e84b0d28ece3a36e0548ee1a1ce9ddde03752d9b79bba40", - "sha256:5454276c07d27a740c5892f4907c86327b632127dd9abec42ee62e12427ff7e3", - "sha256:561091a7be172ab497a3527602d467e2b3fbe75f9e783d8b8ce403fa414f71a6", - "sha256:6c3acb79b0bfd4fe733dff8bc62695283b57949ebcca05ae5c129eb606ff2d74", - "sha256:703f18f3fda276b9a916f0934d2fb6d989bf0b4fb5a64825260eb9bfd52d78f0", - "sha256:7492e2b7bd7c9b9916388d9df23fa49d9b88ac0640db0a5b4ecc2b653bf451e3", - "sha256:76ae285c8104046b3a7f06b42f29c7b73f77683df18c49ab5af7983994c2dd91", - "sha256:7cafd1208fdbe93b67c7086876f061f660cfddc44f404279c1585bbf3cdc64c5", - "sha256:7efde645ca1cc441d6dc4b48c0f7101e8d86b54c8530141b09fd31cef5149ec9", - "sha256:88d9ab96491d38a5ab7c56dd7a3cc37d83336ecc564e4e8816dbed12e5aaefc8", - "sha256:8eab883b3b2a38cc1e050819ef06a7e6344d4a990d24d45bc6f2cf959045a45b", - "sha256:910841381caba4f744a44bf81bfd573c94e10b3045ee00de0cbf436fe50673a6", - "sha256:9190f09060ea4debddd24665d6804b995a9c122ef5917ab26e1566dcc712ceeb", - "sha256:937e9020b514ceedb9c830c55d5c9872abc90f4b5862f89c0887033ae33c6f73", - "sha256:94c817e84245513926588caf1152e3b559ff794d505555211ca041f032abbb6b", - "sha256:971ce5e14dc5e73715755d0ca2975ac88cfdaefcaab078a284fea6cfabf866df", - "sha256:9d14b83fab60d5e8abe587d51c75b252bcc21683f24699ada8fb275d7712f5a9", - "sha256:9f35ec95538f50292f6d8f2c9c9f8a3c6540bbfec21c9e5b4b751e0a7c20864f", - "sha256:a1846f1b999e78e13837c93c778dcfc3365902cfb8d1bdb7dd73ead37059f0d0", - "sha256:acd2162a36d3de67ee896c43effcd5ee3de247eb00354db411feb025aa319857", - "sha256:b0ef99cdbe2b682b9ccbb964743a6aca37905fda5e0452e5ee239b1654d37f2a", - "sha256:b80f600eddddce72320dbbc8e3784d16bd3fb7b517e82476d8da921f27d4b249", - "sha256:b864ba53912b6c3ab6bcb2beb19f19edd01a6bfcbdfe1f37ddd1778abfe75a30", - "sha256:b9ec052b06a0524f0e35bd8790686a1da006bd911dd1ef7d50b77bfbad74e292", - "sha256:ba2956617f1c42598a308a84c6cf021a90ff3862eddafd20c3333d50f0edb45b", - "sha256:bdfea8c661e80d3c1c99ad7c3ff74e6e87184895bbaca6ee8cc61209f8b9b85d", - "sha256:be4ed120b52ae4d974aa40215fcdfde9194d63541c7ded40ee12eb4dda57b76b", - "sha256:c4302695ad8027363e96311df24ee28978162cdcdd2006476c43970b384a244c", - "sha256:c48f54ef8e05f04d6eff74b8233f6063cb1ed960243eacc474ee73a2ea8573ca", - "sha256:c9c59a2120b55788e800d82dfa99b9e156ff8f2227f07c5e3012a45a399620b7", - "sha256:cd021c754b162c0fb55ad5d6b9d960db667faad0fa2ff25bb6e1301b0b6e6a75", - "sha256:d27ec7509b9c18b6d73f2f5ede2622441de812e7b1a80bbd446cb0633bd3d5ae", - "sha256:d5508f0b173e6aa47273bdc0a0b5ba055b59662ba7c7ee5119528f466585526b", - "sha256:d75209eed723105f9596807495d58d10b3470fa6732dd6756595e89925ce2470", - "sha256:db1a39669102a1d8d12b57de2bb7e2ec9066a6f2b3da35ae511ff93b01b5d564", - "sha256:dbfcfc0218093a19c252ca8eb9aee3d29cfdcb586df21049b9d777fd32c14fd9", - "sha256:e0f72c9ddb8cd28532185f54cc1453f2c16fb417a08b53a855c4e6a418edd099", - "sha256:e7c8dc13af7db097bed64a051d2dd49e9f0af495c26995c00a9ee842690d34c0", - "sha256:ea9872c80c132f4663822dd2a08d404073a5a9b5ba6155bea72fb2a79d1093b5", - "sha256:eff4eb9b7eb3e4d0cae3d28c283dc16d9bed6b193c2e1ace3ed86ce48ea8df19", - "sha256:f82d4d717d8ef19188687aa32b8363e96062911e63ba22a0cff7802a8e58e5f1", - "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526" - ], - "markers": "python_version >= '3' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))", - "version": "==2.0.2" - }, "grpclib": { "hashes": [ "sha256:80c7f5179b8d05d1192a092db1b60b4934a0d52e9919358de75c912c269485e5" @@ -551,59 +493,59 @@ }, "markupsafe": { "hashes": [ - "sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed", - "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc", - "sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2", - "sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460", - "sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7", - "sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0", - "sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1", - "sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa", - "sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03", - "sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323", - "sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65", - "sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013", - "sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036", - "sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f", - "sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4", - "sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419", - "sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2", - "sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619", - "sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a", - "sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a", - "sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd", - "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7", - "sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666", - "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65", - "sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859", - "sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625", - "sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff", - "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156", - "sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd", - "sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba", - "sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f", - "sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1", - "sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094", - "sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a", - "sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513", - "sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed", - "sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d", - "sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3", - "sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147", - "sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c", - "sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603", - "sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601", - "sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a", - "sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1", - "sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d", - "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3", - "sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54", - "sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2", - "sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6", - "sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58" + "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e", + "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e", + "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431", + "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686", + "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559", + "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc", + "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c", + "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0", + "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4", + "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9", + "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575", + "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba", + "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d", + "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3", + "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00", + "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155", + "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac", + "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52", + "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f", + "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8", + "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b", + "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24", + "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea", + "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198", + "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0", + "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee", + "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be", + "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2", + "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707", + "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6", + "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58", + "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779", + "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636", + "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c", + "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad", + "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee", + "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc", + "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2", + "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48", + "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7", + "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e", + "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b", + "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa", + "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5", + "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e", + "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb", + "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9", + "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57", + "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc", + "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2" ], "markers": "python_version >= '3.7'", - "version": "==2.1.2" + "version": "==2.1.3" }, "mashumaro": { "extras": [ @@ -788,37 +730,34 @@ }, "numpy": { "hashes": [ - "sha256:0ec87a7084caa559c36e0a2309e4ecb1baa03b687201d0a847c8b0ed476a7187", - "sha256:1a7d6acc2e7524c9955e5c903160aa4ea083736fde7e91276b0e5d98e6332812", - "sha256:202de8f38fc4a45a3eea4b63e2f376e5f2dc64ef0fa692838e31a808520efaf7", - "sha256:210461d87fb02a84ef243cac5e814aad2b7f4be953b32cb53327bb49fd77fbb4", - "sha256:2d926b52ba1367f9acb76b0df6ed21f0b16a1ad87c6720a1121674e5cf63e2b6", - "sha256:352ee00c7f8387b44d19f4cada524586f07379c0d49270f87233983bc5087ca0", - "sha256:35400e6a8d102fd07c71ed7dcadd9eb62ee9a6e84ec159bd48c28235bbb0f8e4", - "sha256:3c1104d3c036fb81ab923f507536daedc718d0ad5a8707c6061cdfd6d184e570", - "sha256:4719d5aefb5189f50887773699eaf94e7d1e02bf36c1a9d353d9f46703758ca4", - "sha256:4749e053a29364d3452c034827102ee100986903263e89884922ef01a0a6fd2f", - "sha256:5342cf6aad47943286afa6f1609cad9b4266a05e7f2ec408e2cf7aea7ff69d80", - "sha256:56e48aec79ae238f6e4395886b5eaed058abb7231fb3361ddd7bfdf4eed54289", - "sha256:76e3f4e85fc5d4fd311f6e9b794d0c00e7002ec122be271f2019d63376f1d385", - "sha256:7776ea65423ca6a15255ba1872d82d207bd1e09f6d0894ee4a64678dd2204078", - "sha256:784c6da1a07818491b0ffd63c6bbe5a33deaa0e25a20e1b3ea20cf0e43f8046c", - "sha256:8535303847b89aa6b0f00aa1dc62867b5a32923e4d1681a35b5eef2d9591a463", - "sha256:9a7721ec204d3a237225db3e194c25268faf92e19338a35f3a224469cb6039a3", - "sha256:a1d3c026f57ceaad42f8231305d4653d5f05dc6332a730ae5c0bea3513de0950", - "sha256:ab344f1bf21f140adab8e47fdbc7c35a477dc01408791f8ba00d018dd0bc5155", - "sha256:ab5f23af8c16022663a652d3b25dcdc272ac3f83c3af4c02eb8b824e6b3ab9d7", - "sha256:ae8d0be48d1b6ed82588934aaaa179875e7dc4f3d84da18d7eae6eb3f06c242c", - "sha256:c91c4afd8abc3908e00a44b2672718905b8611503f7ff87390cc0ac3423fb096", - "sha256:d5036197ecae68d7f491fcdb4df90082b0d4960ca6599ba2659957aafced7c17", - "sha256:d6cc757de514c00b24ae8cf5c876af2a7c3df189028d68c0cb4eaa9cd5afc2bf", - "sha256:d933fabd8f6a319e8530d0de4fcc2e6a61917e0b0c271fded460032db42a0fe4", - "sha256:ea8282b9bcfe2b5e7d491d0bf7f3e2da29700cec05b49e64d6246923329f2b02", - "sha256:ecde0f8adef7dfdec993fd54b0f78183051b6580f606111a6d789cd14c61ea0c", - "sha256:f21c442fdd2805e91799fbe044a7b999b8571bb0ab0f7850d0cb9641a687092b" + "sha256:0ac6edfb35d2a99aaf102b509c8e9319c499ebd4978df4971b94419a116d0790", + "sha256:26815c6c8498dc49d81faa76d61078c4f9f0859ce7817919021b9eba72b425e3", + "sha256:4aedd08f15d3045a4e9c648f1e04daca2ab1044256959f1f95aafeeb3d794c16", + "sha256:4c69fe5f05eea336b7a740e114dec995e2f927003c30702d896892403df6dbf0", + "sha256:5177310ac2e63d6603f659fadc1e7bab33dd5a8db4e0596df34214eeab0fee3b", + "sha256:5aa48bebfb41f93043a796128854b84407d4df730d3fb6e5dc36402f5cd594c0", + "sha256:5b1b90860bf7d8a8c313b372d4f27343a54f415b20fb69dd601b7efe1029c91e", + "sha256:6c284907e37f5e04d2412950960894b143a648dea3f79290757eb878b91acbd1", + "sha256:6d183b5c58513f74225c376643234c369468e02947b47942eacbb23c1671f25d", + "sha256:7412125b4f18aeddca2ecd7219ea2d2708f697943e6f624be41aa5f8a9852cc4", + "sha256:7cd981ccc0afe49b9883f14761bb57c964df71124dcd155b0cba2b591f0d64b9", + "sha256:85cdae87d8c136fd4da4dad1e48064d700f63e923d5af6c8c782ac0df8044542", + "sha256:8aa130c3042052d656751df5e81f6d61edff3e289b5994edcf77f54118a8d9f4", + "sha256:95367ccd88c07af21b379be1725b5322362bb83679d36691f124a16357390153", + "sha256:9c7211d7920b97aeca7b3773a6783492b5b93baba39e7c36054f6e749fc7490c", + "sha256:9e3f2b96e3b63c978bc29daaa3700c028fe3f049ea3031b58aa33fe2a5809d24", + "sha256:b76aa836a952059d70a2788a2d98cb2a533ccd46222558b6970348939e55fc24", + "sha256:b792164e539d99d93e4e5e09ae10f8cbe5466de7d759fc155e075237e0c274e4", + "sha256:c0dc071017bc00abb7d7201bac06fa80333c6314477b3d10b52b58fa6a6e38f6", + "sha256:cc3fda2b36482891db1060f00f881c77f9423eead4c3579629940a3e12095fe8", + "sha256:d6b267f349a99d3908b56645eebf340cb58f01bd1e773b4eea1a905b3f0e4208", + "sha256:d76a84998c51b8b68b40448ddd02bd1081bb33abcdc28beee6cd284fe11036c6", + "sha256:e559c6afbca484072a98a51b6fa466aae785cfe89b69e8b856c3191bc8872a82", + "sha256:ecc68f11404930e9c7ecfc937aa423e1e50158317bf67ca91736a9864eae0232", + "sha256:f1accae9a28dc3cda46a91de86acf69de0d1b5f4edd44a9b0c3ceb8036dfff19" ], "index": "pypi", - "version": "==1.24.3" + "version": "==1.25.0" }, "oauthlib": { "hashes": [ @@ -894,11 +833,11 @@ }, "platformdirs": { "hashes": [ - "sha256:412dae91f52a6f84830f39a8078cecd0e866cb72294a5c66808e74d5e88d251f", - "sha256:e2378146f1964972c03c085bb5662ae80b2b8c06226c54b2ff4aa9483e8a13a5" + "sha256:57e28820ca8094678b807ff529196506d7a21e17156cb1cddb3e74cebce54640", + "sha256:ffa199e3fbab8365778c4a10e1fbf1b9cd50707de826eb304b50e57ec0cc8d38" ], "markers": "python_version >= '3.7'", - "version": "==3.5.1" + "version": "==3.6.0" }, "pluggy": { "hashes": [ @@ -910,42 +849,42 @@ }, "pre-commit": { "hashes": [ - "sha256:66e37bec2d882de1f17f88075047ef8962581f83c234ac08da21a0c58953d1f0", - "sha256:8056bc52181efadf4aac792b1f4f255dfd2fb5a350ded7335d251a68561e8cb6" + "sha256:10badb65d6a38caff29703362271d7dca483d01da88f9d7e05d0b97171c136cb", + "sha256:a2256f489cd913d575c145132ae196fe335da32d91a8294b7afe6622335dd023" ], "index": "pypi", - "version": "==3.3.2" + "version": "==3.3.3" }, "pyarrow": { "hashes": [ - "sha256:0846ace49998825eda4722f8d7f83fa05601c832549c9087ea49d6d5397d8cec", - "sha256:0d8b90efc290e99a81d06015f3a46601c259ecc81ffb6d8ce288c91bd1b868c9", - "sha256:0e36425b1c1cbf5447718b3f1751bf86c58f2b3ad299f996cd9b1aa040967656", - "sha256:19c812d303610ab5d664b7b1de4051ae23565f9f94d04cbea9e50569746ae1ee", - "sha256:1b50bb9a82dca38a002d7cbd802a16b1af0f8c50ed2ec94a319f5f2afc047ee9", - "sha256:1d568acfca3faa565d663e53ee34173be8e23a95f78f2abfdad198010ec8f745", - "sha256:23a77d97f4d101ddfe81b9c2ee03a177f0e590a7e68af15eafa06e8f3cf05976", - "sha256:2466be046b81863be24db370dffd30a2e7894b4f9823fb60ef0a733c31ac6256", - "sha256:272f147d4f8387bec95f17bb58dcfc7bc7278bb93e01cb7b08a0e93a8921e18e", - "sha256:280289ebfd4ac3570f6b776515baa01e4dcbf17122c401e4b7170a27c4be63fd", - "sha256:2cc63e746221cddb9001f7281dee95fd658085dd5b717b076950e1ccc607059c", - "sha256:3b97649c8a9a09e1d8dc76513054f1331bd9ece78ee39365e6bf6bc7503c1e94", - "sha256:3d1733b1ea086b3c101427d0e57e2be3eb964686e83c2363862a887bb5c41fa8", - "sha256:5b0810864a593b89877120972d1f7af1d1c9389876dbed92b962ed81492d3ffc", - "sha256:7a7b6a765ee4f88efd7d8348d9a1f804487d60799d0428b6ddf3344eaef37282", - "sha256:7b5b9f60d9ef756db59bec8d90e4576b7df57861e6a3d6a8bf99538f68ca15b3", - "sha256:92fb031e6777847f5c9b01eaa5aa0c9033e853ee80117dce895f116d8b0c3ca3", - "sha256:993287136369aca60005ee7d64130f9466489c4f7425f5c284315b0a5401ccd9", - "sha256:a1c4fce253d5bdc8d62f11cfa3da5b0b34b562c04ce84abb8bd7447e63c2b327", - "sha256:a7cd32fe77f967fe08228bc100433273020e58dd6caced12627bcc0a7675a513", - "sha256:b99e559d27db36ad3a33868a475f03e3129430fc065accc839ef4daa12c6dab6", - "sha256:bc4ea634dacb03936f50fcf59574a8e727f90c17c24527e488d8ceb52ae284de", - "sha256:d8c26912607e26c2991826bbaf3cf2b9c8c3e17566598c193b492f058b40d3a4", - "sha256:e6be4d85707fc8e7a221c8ab86a40449ce62559ce25c94321df7c8500245888f", - "sha256:ea830d9f66bfb82d30b5794642f83dd0e4a718846462d22328981e9eb149cba8" + "sha256:051f9f5ccf585f12d7de836e50965b3c235542cc896959320d9776ab93f3b33d", + "sha256:1887bdae17ec3b4c046fcf19951e71b6a619f39fa674f9881216173566c8f718", + "sha256:2d3c4cbbf81e6dd23fe921bc91dc4619ea3b79bc58ef10bce0f49bdafb103daf", + "sha256:345e1828efdbd9aa4d4de7d5676778aba384a2c3add896d995b23d368e60e5af", + "sha256:3de26da901216149ce086920547dfff5cd22818c9eab67ebc41e863a5883bac7", + "sha256:43364daec02f69fec89d2315f7fbfbeec956e0d991cbbef471681bd77875c40f", + "sha256:459a1c0ed2d68671188b2118c63bac91eaef6fc150c77ddd8a583e3c795737bf", + "sha256:6251e38470da97a5b2e00de5c6a049149f7b2bd62f12fa5dbb9ac674119ba71a", + "sha256:6895b5fb74289d055c43db3af0de6e16b07586c45763cb5e558d38b86a91e3a7", + "sha256:6d288029a94a9bb5407ceebdd7110ba398a00412c5b0155ee9813a40d246c5df", + "sha256:749be7fd2ff260683f9cc739cb862fb11be376de965a2a8ccbf2693b098db6c7", + "sha256:85e705e33eaf666bbe508a16fd5ba27ca061e177916b7a317ba5a51bee43384c", + "sha256:8d6009fdf8986332b2169314da482baed47ac053311c8934ac6651e614deacd6", + "sha256:9120c3eb2b1f6f516a3b7a9714ed860882d9ef98c4b17edcdc91d95b7528db60", + "sha256:a3c63124fc26bf5f95f508f5d04e1ece8cc23a8b0af2a1e6ab2b1ec3fdc91b24", + "sha256:b13329f79fa4472324f8d32dc1b1216616d09bd1e77cfb13104dec5463632c36", + "sha256:bb656150d3d12ec1396f6dde542db1675a95c0cc8366d507347b0beed96e87ca", + "sha256:be2757e9275875d2a9c6e6052ac7957fbbfc7bc7370e4a036a9b893e96fedaba", + "sha256:c780f4dc40460015d80fcd6a6140de80b615349ed68ef9adb653fe351778c9b3", + "sha256:cce317fc96e5b71107bf1f9f184d5e54e2bd14bbf3f9a3d62819961f0af86fec", + "sha256:cdacf515ec276709ac8042c7d9bd5be83b4f5f39c6c037a17a60d7ebfd92c890", + "sha256:ce4aebdf412bd0eeb800d8e47db854f9f9f7e2f5a0220440acf219ddfddd4f63", + "sha256:cf812306d66f40f69e684300f7af5111c11f6e0d89d6b733e05a3de44961529d", + "sha256:e0d8730c7f6e893f6db5d5b86eda42c0a130842d101992b581e2138e4d5663d3", + "sha256:e2c9cb8eeabbadf5fcfc3d1ddea616c7ce893db2ce4dcef0ac13b099ad7ca082" ], "markers": "python_version < '3.11' and python_version >= '3.7'", - "version": "==12.0.0" + "version": "==12.0.1" }, "pycparser": { "hashes": [ @@ -989,11 +928,11 @@ }, "pytest": { "hashes": [ - "sha256:3799fa815351fea3a5e96ac7e503a96fa51cc9942c3753cda7651b93c1cfa362", - "sha256:434afafd78b1d78ed0addf160ad2b77a30d35d4bdf8af234fe621919d9ed15e3" + "sha256:cdcbd012c9312258922f8cd3f1b62a6580fdced17db6014896053d47cddf9295", + "sha256:ee990a3cc55ba808b80795a79944756f315c67c12b56abd3ac993a7b8c17030b" ], "index": "pypi", - "version": "==7.3.1" + "version": "==7.3.2" }, "python-dateutil": { "hashes": [ @@ -1081,11 +1020,11 @@ }, "setuptools": { "hashes": [ - "sha256:5df61bf30bb10c6f756eb19e7c9f3b473051f48db77fddbe06ff2ca307df9a6f", - "sha256:62642358adc77ffa87233bc4d2354c4b2682d214048f500964dbe760ccedf102" + "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f", + "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235" ], "markers": "python_version >= '3.7'", - "version": "==67.8.0" + "version": "==68.0.0" }, "six": { "hashes": [ @@ -1185,38 +1124,54 @@ "markers": "python_version < '3.11'", "version": "==2.0.1" }, + "trino": { + "hashes": [ + "sha256:3a9f1e48a302368560b4ad3f13d8145912802c59a3b26dea80edda5460843cd5", + "sha256:e12992811aef54645e589c671a722952ab00ac0160e8207ecb987254a1fe3d65" + ], + "markers": "python_version >= '3.7'", + "version": "==0.326.0" + }, "typing-extensions": { "hashes": [ - "sha256:06006244c70ac8ee83fa8282cb188f697b8db25bc8b4df07be1873c43897060c", - "sha256:3a8b36f13dd5fdc5d1b16fe317f5668545de77fa0b8e02006381fd49d731ab98" + "sha256:88a4153d8505aabbb4e13aacb7c486c2b4a33ca3b3f807914a9b4c844c471c26", + "sha256:d91d5919357fe7f681a9f2b5b4cb2a5f1ef0a1e9f59c4d8ff0d3491e05c0ffd5" + ], + "markers": "python_version >= '3.7'", + "version": "==4.6.3" + }, + "tzlocal": { + "hashes": [ + "sha256:46eb99ad4bdb71f3f72b7d24f4267753e240944ecfc16f25d2719ba89827a803", + "sha256:f3596e180296aaf2dbd97d124fe76ae3a0e3d32b258447de7b939b3fd4be992f" ], "markers": "python_version >= '3.7'", - "version": "==4.6.2" + "version": "==5.0.1" }, "urllib3": { "hashes": [ - "sha256:61717a1095d7e155cdb737ac7bb2f4324a858a1e2e6466f6d03ff630ca68d3cc", - "sha256:d055c2f9d38dc53c808f6fdc8eab7360b6fdbbde02340ed25cfbcd817c62469e" + "sha256:48e7fafa40319d358848e1bc6809b208340fafe2096f1725d05d67443d0483d1", + "sha256:bee28b5e56addb8226c96f7f13ac28cb4c301dd5ea8a6ca179c0b9835e032825" ], "markers": "python_version >= '3.7'", - "version": "==2.0.2" + "version": "==2.0.3" }, "virtualenv": { "hashes": [ - "sha256:6abec7670e5802a528357fdc75b26b9f57d5d92f29c5462ba0fbe45feacc685e", - "sha256:a85caa554ced0c0afbd0d638e7e2d7b5f92d23478d05d17a76daeac8f279f924" + "sha256:34da10f14fea9be20e0fd7f04aba9732f84e593dac291b757ce42e3368a39419", + "sha256:8ff19a38c1021c742148edc4f81cb43d7f8c6816d2ede2ab72af5b84c749ade1" ], "markers": "python_version >= '3.7'", - "version": "==20.23.0" + "version": "==20.23.1" }, "werkzeug": { "hashes": [ - "sha256:1d5a58e0377d1fe39d061a5de4469e414e78ccb1e1e59c0f5ad6fa1c36c52b76", - "sha256:48e5e61472fee0ddee27ebad085614ebedb7af41e88f687aaf881afb723a162f" + "sha256:935539fa1413afbb9195b24880778422ed620c0fc09670945185cce4d91a8890", + "sha256:98c774df2f91b05550078891dee5f0eb0cb797a522c757a2452b9cee5b202330" ], "markers": "python_version >= '3.8'", - "version": "==2.3.4" + "version": "==2.3.6" } }, "develop": {} -} \ No newline at end of file +} diff --git a/macros/dune/adapters.sql b/macros/dune/adapters.sql new file mode 100644 index 00000000000..6a94882c424 --- /dev/null +++ b/macros/dune/adapters.sql @@ -0,0 +1,24 @@ +{% macro trino__create_table_as(temporary, relation, sql) -%} + {%- set _properties = config.get('properties') -%} + create or replace table {{ relation }} + {{ create_table_properties(_properties, relation) }} + as ( + {{ sql }} + ); +{% endmacro %} + +{% macro create_table_properties(properties, relation) %} + {% set s3_bucket = var('DBT_ENV_CUSTOM_ENV_S3_BUCKET', 'local') %} + {% set modified_identifier = relation.identifier | replace("__dbt_tmp", "") %} + {%- set unique_location = modified_identifier ~ '_' ~ time_salted_md5_prefix() -%} + WITH ( + location = 's3a://{{s3_bucket}}/hive/{{relation.schema}}/{{unique_location}}' + ) +{%- endmacro -%} + +{% macro time_salted_md5_prefix(input_string=None) -%} + {% if input_string is none -%} + {% set input_string = modules.datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f') %} + {%- endif %} + {{- local_md5(input_string) -}} +{%- endmacro %} diff --git a/macros/dune/alias.sql b/macros/dune/alias.sql new file mode 100644 index 00000000000..aef4705fd6a --- /dev/null +++ b/macros/dune/alias.sql @@ -0,0 +1,10 @@ +{% macro alias(alias_name, legacy_model=False) %} + {% set legacy = target.type != 'trino' %} + {%- if legacy and not legacy_model -%} + {% do return(alias_name + '_dunesql') %} + {%- endif -%} + {%- if not legacy and legacy_model -%} + {% do return(alias_name + '_legacy') %} + {%- endif -%} + {% do return(alias_name) %} +{% endmacro %} \ No newline at end of file diff --git a/macros/dune/macros_schema.yml b/macros/dune/macros_schema.yml index 733f5d54967..a4d013e7663 100644 --- a/macros/dune/macros_schema.yml +++ b/macros/dune/macros_schema.yml @@ -48,4 +48,12 @@ macros: - name: mark_as_spell description: > - Catch all macro to mark all models as spells even if they are not exposed in the data explorer \ No newline at end of file + Catch all macro to mark all models as spells even if they are not exposed in the data explorer + + - name: source + description: > + Change source database. + + - name: alias + description: > + Add dummy alias \ No newline at end of file diff --git a/macros/dune/schema.sql b/macros/dune/schema.sql new file mode 100644 index 00000000000..76dfa6e2a8f --- /dev/null +++ b/macros/dune/schema.sql @@ -0,0 +1,10 @@ +{% macro create_schema(relation) -%} + {{ adapter.dispatch('create_schema', 'dbt')(relation) }} +{% endmacro %} + +{% macro default__create_schema(relation) -%} + {% set s3_bucket = 'prod-spellbook-trino-118330671040' %} + {%- call statement('create_schema') -%} + CREATE SCHEMA {{ relation }} WITH (location = 's3a://{{s3_bucket}}/hive/') + {% endcall %} +{% endmacro %} diff --git a/macros/dune/source.sql b/macros/dune/source.sql new file mode 100644 index 00000000000..e5269bb9e74 --- /dev/null +++ b/macros/dune/source.sql @@ -0,0 +1,9 @@ +{% macro source(source_name, table_name) %} + {% set rel = builtins.source(source_name, table_name) %} + {%- if 'dunesql' in model.config.get("tags") -%} + {% set newrel = rel.replace_path(database="delta_prod") %} + {% do return(newrel) %} + {%- else -%} + {% do return(rel) %} + {%- endif -%} +{% endmacro %} \ No newline at end of file diff --git a/macros/expose_spells.sql b/macros/expose_spells.sql index 1adf43d406f..b37790829fe 100644 --- a/macros/expose_spells.sql +++ b/macros/expose_spells.sql @@ -1,15 +1,29 @@ {% macro expose_spells(blockchains, spell_type, spell_name, contributors) %} {%- if target.name == 'prod'-%} - ALTER {{"view" if model.config.materialized == "view" else "table"}} {{ this }} - SET TBLPROPERTIES ( - 'dune.public'='true', - 'dune.data_explorer.blockchains'= '{{ blockchains }}', -- e.g., ["ethereum","solana"] - 'dune.data_explorer.category'='abstraction', - 'dune.data_explorer.abstraction.type'= '{{ spell_type }}', -- 'project' or 'sector' - 'dune.data_explorer.abstraction.name'= '{{ spell_name }}', -- 'aave' or 'uniswap' - 'dune.data_explorer.contributors'= '{{ contributors }}', -- e.g., ["soispoke","jeff_dude"] - 'dune.vacuum' = '{"enabled":true}' - ) -{%- else -%} + {%- if 'dunesql' not in model.config.get("tags") -%} + {# comment #} + ALTER {{"view" if model.config.materialized == "view" else "table"}} {{ this }} + SET TBLPROPERTIES ( + 'dune.public'='true', + 'dune.data_explorer.blockchains'= '{{ blockchains }}', -- e.g., ["ethereum","solana"] + 'dune.data_explorer.category'='abstraction', + 'dune.data_explorer.abstraction.type'= '{{ spell_type }}', -- 'project' or 'sector' + 'dune.data_explorer.abstraction.name'= '{{ spell_name }}', -- 'aave' or 'uniswap' + 'dune.data_explorer.contributors'= '{{ contributors }}', -- e.g., ["soispoke","jeff_dude"] + 'dune.vacuum' = '{"enabled":true}' + ) + {%- else -%} + {%- if model.config.materialized != "view" -%} + ALTER {{"view" if model.config.materialized == "view" else "table"}} {{ this }} + SET PROPERTIES extra_properties = map_from_entries(ARRAY[ + ROW('dune.public', 'true'), + ROW('dune.data_explorer.blockchains', '{{ blockchains }}'), -- e.g., ["ethereum","solana"] + ROW('dune.data_explorer.category', 'abstraction'), + ROW('dune.data_explorer.abstraction.type', '{{ spell_type }}'), -- 'project' or 'sector' + ROW('dune.data_explorer.abstraction.name', '{{ spell_name }}'), -- 'aave' or 'uniswap' + ROW('dune.data_explorer.contributors','{{ contributors }}') -- e.g., ["soispoke","jeff_dude"] + ]) + {%- endif -%} + {%- endif -%} {%- endif -%} {%- endmacro -%} diff --git a/macros/mark_as_spell.sql b/macros/mark_as_spell.sql index 4c1420f6a68..fd507a035b6 100644 --- a/macros/mark_as_spell.sql +++ b/macros/mark_as_spell.sql @@ -1,9 +1,17 @@ {% macro mark_as_spell(this, materialization) %} {%- if target.name == 'prod'-%} - ALTER {{"view" if materialization == "view" else "table"}} {{ this }} - SET TBLPROPERTIES ( - 'dune.data_explorer.category'='abstraction' - ) -{%- else -%} + {%- if 'dunesql' not in model.config.get("tags") -%} + ALTER {{"view" if materialization == "view" else "table"}} {{ this }} + SET TBLPROPERTIES ( + 'dune.data_explorer.category'='abstraction' + ) + {%- else -%} + {%- if model.config.materialized != "view" -%} + ALTER {{"view" if materialization == "view" else "table"}} {{ this }} + SET PROPERTIES extra_properties = map_from_entries (ARRAY[ + ROW('dune.data_explorer.category', 'abstraction') + ]) + {%- endif -%} + {%- endif -%} {%- endif -%} {%- endmacro -%} diff --git a/models/evms/evms_blocks.sql b/models/evms/evms_blocks.sql index 0e66576f566..89a358bbe44 100644 --- a/models/evms/evms_blocks.sql +++ b/models/evms/evms_blocks.sql @@ -1,5 +1,6 @@ {{ config( - alias ='blocks', + tags = ['dunesql'], + alias = alias('blocks'), unique_key=['blockchain', 'number'], post_hook='{{ expose_spells(\'["ethereum", "polygon", "bnb", "avalanche_c", "gnosis", "fantom", "optimism", "arbitrum"]\', "sector", diff --git a/models/evms/evms_blocks_legacy.sql b/models/evms/evms_blocks_legacy.sql new file mode 100644 index 00000000000..908b414cd43 --- /dev/null +++ b/models/evms/evms_blocks_legacy.sql @@ -0,0 +1,46 @@ +{{ config( + alias = alias('blocks', legacy_model=True), + unique_key=['blockchain', 'number'], + post_hook='{{ expose_spells(\'["ethereum", "polygon", "bnb", "avalanche_c", "gnosis", "fantom", "optimism", "arbitrum"]\', + "sector", + "evms", + \'["hildobby"]\') }}' + ) +}} + +{% set blocks_models = [ + ('ethereum', source('ethereum', 'blocks')) + , ('polygon', source('polygon', 'blocks')) + , ('bnb', source('bnb', 'blocks')) + , ('avalanche_c', source('avalanche_c', 'blocks')) + , ('gnosis', source('gnosis', 'blocks')) + , ('fantom', source('fantom', 'blocks')) + , ('optimism', source('optimism', 'blocks')) + , ('arbitrum', source('arbitrum', 'blocks')) +] %} + +SELECT * +FROM ( + {% for blocks_model in blocks_models %} + SELECT + '{{ blocks_model[0] }}' AS blockchain + , hash + , miner + , nonce + , parent_hash + , size + , time + , CAST(total_difficulty AS double) AS total_difficulty + , number + , base_fee_per_gas + , CAST(difficulty AS double) AS difficulty + , gas_limit + , gas_used + FROM {{ blocks_model[1] }} + {% if not loop.last %} + {% if is_incremental() %} + {% endif %} + UNION ALL + {% endif %} + {% endfor %} + ); \ No newline at end of file