diff --git a/.github/workflows/commit_manifest.yml b/.github/workflows/commit_manifest.yml index 2be7adc44ca..153383d9e67 100644 --- a/.github/workflows/commit_manifest.yml +++ b/.github/workflows/commit_manifest.yml @@ -15,7 +15,7 @@ jobs: runs-on: [ self-hosted, linux, spellbook-trino-ci ] strategy: matrix: - project: [ 'tokens', 'spellbook', 'daily_spellbook', 'nft', 'dex'] + project: [ 'tokens', 'spellbook', 'daily_spellbook', 'nft', 'dex', 'solana'] max-parallel: 1 steps: diff --git a/.github/workflows/solana.yml b/.github/workflows/solana.yml new file mode 100644 index 00000000000..867cc50deb3 --- /dev/null +++ b/.github/workflows/solana.yml @@ -0,0 +1,19 @@ +name: DBT Solana sector run + +on: + workflow_dispatch: + pull_request: + paths: + - dex/** + - .github/workflows/solana.yml + - .github/workflows/dbt_run.yml + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + dbt-run: + uses: ./.github/workflows/dbt_run.yml + with: + project: 'solana' diff --git a/.github/workflows/spellbook.yml b/.github/workflows/spellbook.yml index 919a25b1642..a8b1f62db1f 100644 --- a/.github/workflows/spellbook.yml +++ b/.github/workflows/spellbook.yml @@ -10,6 +10,7 @@ on: - 'tokens/**' - 'daily_spellbook/**' - 'nft/**' + - 'solana/**' concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/dbt_project.yml b/dbt_project.yml index fd0c179ce0d..56ebcd63efa 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -110,14 +110,6 @@ models: ethereum: +schema: addresses_ethereum - tokens: - +schema: tokens - +materialized: table - +file_format: delta - - solana_utils: - +schema: solana_utils - transfers: +schema: transfers ethereum: diff --git a/solana/README.md b/solana/README.md new file mode 100644 index 00000000000..2fee156a2d1 --- /dev/null +++ b/solana/README.md @@ -0,0 +1,3 @@ +## Solana subproject + +This is a DBT subproject for the main lineages of Solana data. \ No newline at end of file diff --git a/solana/dbt_project.yml b/solana/dbt_project.yml new file mode 100644 index 00000000000..cc557f8895d --- /dev/null +++ b/solana/dbt_project.yml @@ -0,0 +1,67 @@ +#Welcome to your Spellbook! + +name: "solana" +version: "1.0.0" +config-version: 2 + +quoting: + database: false + schema: false + identifier: false + +# This setting configures which "profile" dbt uses for this project. +profile: "spellbook-local" + +vars: + DBT_ENV_CUSTOM_ENV_S3_BUCKET: "{{ env_var('DBT_ENV_CUSTOM_ENV_S3_BUCKET', 'local') }}" + DBT_ENV_INCREMENTAL_TIME: "{{ env_var('DBT_ENV_INCREMENTAL_TIME', '1') }}" + DBT_ENV_INCREMENTAL_TIME_UNIT: "{{ env_var('DBT_ENV_INCREMENTAL_TIME_UNIT', 'day') }}" + +# These configurations specify where dbt should look for different types of files. +# You don't need to change these! +model-paths: ["models", "../sources"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros", "../macros"] +snapshot-paths: ["snapshots"] + +target-path: "target" # directory which will store compiled SQL files +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_packages" + +# Configuring tests +# Full documentation: https://docs.getdbt.com/reference/test-configs +tests: + solana: + +store_failures: true # store failures for all tests + +# Configuring seeds +# Full documentation: https://docs.getdbt.com/reference/seed-configs +# For configuring individual seeds (e.g. overriding column types) we recommend a yml file nested in the same folder as the seed +seeds: + solana: + +schema: test_data + +# defining search order of macro invokation +dispatch: + - macro_namespace: dbt_utils + search_order: ['trino_utils', 'dbt_utils'] + +# Configuring models +# Full documentation: https://docs.getdbt.com/docs/configuring-models +models: + solana: + +post-hook: + - sql: "{{ set_trino_session_property(is_materialized(model), 'writer_scaling_min_data_processed', model.config.get('writer_min_size', '500MB')) }}" + transaction: true + - sql: "{{ set_trino_session_property(is_materialized(model), 'task_scale_writers_enabled', false) }}" + transaction: true + - sql: "{{ optimize_spell(this, model.config.materialized) }}" + transaction: true + - sql: "{{ mark_as_spell(this, model.config.materialized) }}" + transaction: true + +materialized: view + +schema: no_schema # this should be overriden in model specific configs + +view_security: invoker \ No newline at end of file diff --git a/models/_sector/dex/bot_trades/solana/_schema.yml b/solana/models/_sector/dex/bot_trades/solana/_schema.yml similarity index 100% rename from models/_sector/dex/bot_trades/solana/_schema.yml rename to solana/models/_sector/dex/bot_trades/solana/_schema.yml diff --git a/models/_sector/dex/bot_trades/solana/dex_solana_bot_trades.sql b/solana/models/_sector/dex/bot_trades/solana/dex_solana_bot_trades.sql similarity index 100% rename from models/_sector/dex/bot_trades/solana/dex_solana_bot_trades.sql rename to solana/models/_sector/dex/bot_trades/solana/dex_solana_bot_trades.sql diff --git a/models/_sector/dex/bot_trades/solana/platforms/banana_gun_solana_bot_trades.sql b/solana/models/_sector/dex/bot_trades/solana/platforms/banana_gun_solana_bot_trades.sql similarity index 100% rename from models/_sector/dex/bot_trades/solana/platforms/banana_gun_solana_bot_trades.sql rename to solana/models/_sector/dex/bot_trades/solana/platforms/banana_gun_solana_bot_trades.sql diff --git a/models/_sector/dex/bot_trades/solana/platforms/bonkbot_solana_bot_trades.sql b/solana/models/_sector/dex/bot_trades/solana/platforms/bonkbot_solana_bot_trades.sql similarity index 100% rename from models/_sector/dex/bot_trades/solana/platforms/bonkbot_solana_bot_trades.sql rename to solana/models/_sector/dex/bot_trades/solana/platforms/bonkbot_solana_bot_trades.sql diff --git a/models/_sector/dex/bot_trades/solana/platforms/consortium_key_solana_bot_trades.sql b/solana/models/_sector/dex/bot_trades/solana/platforms/consortium_key_solana_bot_trades.sql similarity index 100% rename from models/_sector/dex/bot_trades/solana/platforms/consortium_key_solana_bot_trades.sql rename to solana/models/_sector/dex/bot_trades/solana/platforms/consortium_key_solana_bot_trades.sql diff --git a/models/_sector/dex/bot_trades/solana/platforms/looter_solana_bot_trades.sql b/solana/models/_sector/dex/bot_trades/solana/platforms/looter_solana_bot_trades.sql similarity index 100% rename from models/_sector/dex/bot_trades/solana/platforms/looter_solana_bot_trades.sql rename to solana/models/_sector/dex/bot_trades/solana/platforms/looter_solana_bot_trades.sql diff --git a/models/_sector/dex/bot_trades/solana/platforms/maestro_solana_bot_trades.sql b/solana/models/_sector/dex/bot_trades/solana/platforms/maestro_solana_bot_trades.sql similarity index 100% rename from models/_sector/dex/bot_trades/solana/platforms/maestro_solana_bot_trades.sql rename to solana/models/_sector/dex/bot_trades/solana/platforms/maestro_solana_bot_trades.sql diff --git a/models/_sector/dex/bot_trades/solana/platforms/magnum_solana_bot_trades.sql b/solana/models/_sector/dex/bot_trades/solana/platforms/magnum_solana_bot_trades.sql similarity index 100% rename from models/_sector/dex/bot_trades/solana/platforms/magnum_solana_bot_trades.sql rename to solana/models/_sector/dex/bot_trades/solana/platforms/magnum_solana_bot_trades.sql diff --git a/models/_sector/dex/bot_trades/solana/platforms/pepe_boost_solana_bot_trades.sql b/solana/models/_sector/dex/bot_trades/solana/platforms/pepe_boost_solana_bot_trades.sql similarity index 100% rename from models/_sector/dex/bot_trades/solana/platforms/pepe_boost_solana_bot_trades.sql rename to solana/models/_sector/dex/bot_trades/solana/platforms/pepe_boost_solana_bot_trades.sql diff --git a/models/_sector/dex/bot_trades/solana/platforms/readyswap_solana_bot_trades.sql b/solana/models/_sector/dex/bot_trades/solana/platforms/readyswap_solana_bot_trades.sql similarity index 100% rename from models/_sector/dex/bot_trades/solana/platforms/readyswap_solana_bot_trades.sql rename to solana/models/_sector/dex/bot_trades/solana/platforms/readyswap_solana_bot_trades.sql diff --git a/models/_sector/dex/bot_trades/solana/platforms/shuriken_solana_bot_trades.sql b/solana/models/_sector/dex/bot_trades/solana/platforms/shuriken_solana_bot_trades.sql similarity index 100% rename from models/_sector/dex/bot_trades/solana/platforms/shuriken_solana_bot_trades.sql rename to solana/models/_sector/dex/bot_trades/solana/platforms/shuriken_solana_bot_trades.sql diff --git a/models/_sector/dex/bot_trades/solana/platforms/sol_gun_solana_bot_trades.sql b/solana/models/_sector/dex/bot_trades/solana/platforms/sol_gun_solana_bot_trades.sql similarity index 100% rename from models/_sector/dex/bot_trades/solana/platforms/sol_gun_solana_bot_trades.sql rename to solana/models/_sector/dex/bot_trades/solana/platforms/sol_gun_solana_bot_trades.sql diff --git a/models/_sector/dex/bot_trades/solana/platforms/sol_trading_bot_solana_bot_trades.sql b/solana/models/_sector/dex/bot_trades/solana/platforms/sol_trading_bot_solana_bot_trades.sql similarity index 100% rename from models/_sector/dex/bot_trades/solana/platforms/sol_trading_bot_solana_bot_trades.sql rename to solana/models/_sector/dex/bot_trades/solana/platforms/sol_trading_bot_solana_bot_trades.sql diff --git a/models/_sector/dex/bot_trades/solana/platforms/soul_sniper_solana_bot_trades.sql b/solana/models/_sector/dex/bot_trades/solana/platforms/soul_sniper_solana_bot_trades.sql similarity index 100% rename from models/_sector/dex/bot_trades/solana/platforms/soul_sniper_solana_bot_trades.sql rename to solana/models/_sector/dex/bot_trades/solana/platforms/soul_sniper_solana_bot_trades.sql diff --git a/models/_sector/dex/bot_trades/solana/platforms/trojan_solana_bot_trades.sql b/solana/models/_sector/dex/bot_trades/solana/platforms/trojan_solana_bot_trades.sql similarity index 100% rename from models/_sector/dex/bot_trades/solana/platforms/trojan_solana_bot_trades.sql rename to solana/models/_sector/dex/bot_trades/solana/platforms/trojan_solana_bot_trades.sql diff --git a/models/_sector/dex/bot_trades/solana/platforms/wifbot_solana_bot_trades.sql b/solana/models/_sector/dex/bot_trades/solana/platforms/wifbot_solana_bot_trades.sql similarity index 100% rename from models/_sector/dex/bot_trades/solana/platforms/wifbot_solana_bot_trades.sql rename to solana/models/_sector/dex/bot_trades/solana/platforms/wifbot_solana_bot_trades.sql diff --git a/models/dex/solana/dex_solana_price_hour.sql b/solana/models/_sector/dex/dex_solana_price_hour.sql similarity index 100% rename from models/dex/solana/dex_solana_price_hour.sql rename to solana/models/_sector/dex/dex_solana_price_hour.sql diff --git a/models/dex/solana/dex_solana_schema.yml b/solana/models/_sector/dex/dex_solana_schema.yml similarity index 100% rename from models/dex/solana/dex_solana_schema.yml rename to solana/models/_sector/dex/dex_solana_schema.yml diff --git a/models/dex/solana/dex_solana_trades.sql b/solana/models/_sector/dex/dex_solana_trades.sql similarity index 100% rename from models/dex/solana/dex_solana_trades.sql rename to solana/models/_sector/dex/dex_solana_trades.sql diff --git a/models/goosefx/solana/_schema.yml b/solana/models/_sector/dex/goosefx/_schema.yml similarity index 100% rename from models/goosefx/solana/_schema.yml rename to solana/models/_sector/dex/goosefx/_schema.yml diff --git a/models/goosefx/solana/goosefx_ssl_v2_solana_trades.sql b/solana/models/_sector/dex/goosefx/goosefx_ssl_v2_solana_trades.sql similarity index 100% rename from models/goosefx/solana/goosefx_ssl_v2_solana_trades.sql rename to solana/models/_sector/dex/goosefx/goosefx_ssl_v2_solana_trades.sql diff --git a/models/jupiter/solana/jupiter_solana_aggregator_swaps.sql b/solana/models/_sector/dex/jupiter/jupiter_solana_aggregator_swaps.sql similarity index 95% rename from models/jupiter/solana/jupiter_solana_aggregator_swaps.sql rename to solana/models/_sector/dex/jupiter/jupiter_solana_aggregator_swaps.sql index 05974173d2c..e8a86bf5a0a 100644 --- a/models/jupiter/solana/jupiter_solana_aggregator_swaps.sql +++ b/solana/models/_sector/dex/jupiter/jupiter_solana_aggregator_swaps.sql @@ -15,7 +15,7 @@ \'["ilemi"]\') }}') }} -with +with amms as ( SELECT * @@ -65,13 +65,13 @@ with ('Jupiter Perps', 'PERPHjGBqRHArX4DySjwM6UJHiR3sWAatqfdBS2qQJu') ) as v(amm_name, amm) ) - + , jup_messages_logs as ( --SwapEvent through log messages - with + with hex_data as ( - SELECT + SELECT from_base64(split(l.logs, ' ')[3]) as hex_data -- , split(logs, ' ')[3] as base64_data , l.log_index_raw @@ -80,7 +80,7 @@ with , t.block_slot , t.block_time , t.signer as tx_signer - , case when contains(t.account_keys, 'JUP5pEAZeHdHrLxh5UCwAbpjGwYKKoquCpda2hfP4u8') then 5 + , case when contains(t.account_keys, 'JUP5pEAZeHdHrLxh5UCwAbpjGwYKKoquCpda2hfP4u8') then 5 when contains(t.account_keys, 'JUP4Fb2cqiRUcaTHdrPC8h2gNsA2ETXiPDD33WcGuJB') then 4 end as jup_version FROM {{ source('solana','transactions') }} t @@ -95,8 +95,8 @@ with {% endif %} -- and block_time >= now() - interval '7' day --shorten CI ) - - SELECT + + SELECT a.amm_name , toBase58(bytearray_substring(hex_data,1+8,32)) as amm , toBase58(bytearray_substring(hex_data,1+40,32)) as input_mint @@ -112,7 +112,7 @@ with FROM hex_data JOIN amms a ON a.amm = toBase58(bytearray_substring(hex_data,1+8,32)) --only include amms that we are tracking. ) - + , jup6_logs as ( --uses event CPI SELECT @@ -129,7 +129,7 @@ with , tx_signer , 6 as jup_version FROM ( - SELECT + SELECT * , row_number() over (partition by tx_id order by outer_instruction_index asc, COALESCE(inner_instruction_index,0) asc) as log_index FROM {{ source('solana','instruction_calls') }} @@ -168,22 +168,22 @@ l.amm , l.tx_signer , l.jup_version FROM ( - SELECT * FROM jup_messages_logs + SELECT * FROM jup_messages_logs WHERE input_amount > 0 and output_amount > 0 - UNION ALL + UNION ALL SELECT * FROM jup6_logs -) l +) l --tokens LEFT JOIN {{ ref('tokens_solana_fungible') }} tk_1 ON tk_1.token_mint_address = l.input_mint LEFT JOIN {{ ref('tokens_solana_fungible') }} tk_2 ON tk_2.token_mint_address = l.output_mint -LEFT JOIN {{ ref('prices_usd_forward_fill') }} p_1 ON p_1.blockchain = 'solana' - AND date_trunc('minute', l.block_time) = p_1.minute +LEFT JOIN {{ source('prices','usd_forward_fill') }} p_1 ON p_1.blockchain = 'solana' + AND date_trunc('minute', l.block_time) = p_1.minute AND l.input_mint = toBase58(p_1.contract_address) {% if is_incremental() %} AND {{ incremental_predicate('p_1.minute') }} {% endif %} -LEFT JOIN {{ ref('prices_usd_forward_fill') }} p_2 ON p_2.blockchain = 'solana' - AND date_trunc('minute', l.block_time) = p_2.minute +LEFT JOIN {{ source('prices', 'usd_forward_fill') }} p_2 ON p_2.blockchain = 'solana' + AND date_trunc('minute', l.block_time) = p_2.minute AND l.output_mint = toBase58(p_2.contract_address) {% if is_incremental() %} AND {{ incremental_predicate('p_2.minute') }} diff --git a/models/jupiter/solana/jupiter_solana_perp_events.sql b/solana/models/_sector/dex/jupiter/jupiter_solana_perp_events.sql similarity index 100% rename from models/jupiter/solana/jupiter_solana_perp_events.sql rename to solana/models/_sector/dex/jupiter/jupiter_solana_perp_events.sql diff --git a/models/jupiter/solana/jupiter_solana_schema.yml b/solana/models/_sector/dex/jupiter/jupiter_solana_schema.yml similarity index 100% rename from models/jupiter/solana/jupiter_solana_schema.yml rename to solana/models/_sector/dex/jupiter/jupiter_solana_schema.yml diff --git a/models/lifinity/lifinity_v1_trades.sql b/solana/models/_sector/dex/lifinity/lifinity_v1_trades.sql similarity index 100% rename from models/lifinity/lifinity_v1_trades.sql rename to solana/models/_sector/dex/lifinity/lifinity_v1_trades.sql diff --git a/models/lifinity/lifinity_v2_trades.sql b/solana/models/_sector/dex/lifinity/lifinity_v2_trades.sql similarity index 100% rename from models/lifinity/lifinity_v2_trades.sql rename to solana/models/_sector/dex/lifinity/lifinity_v2_trades.sql diff --git a/models/lifinity/schema.yml b/solana/models/_sector/dex/lifinity/schema.yml similarity index 100% rename from models/lifinity/schema.yml rename to solana/models/_sector/dex/lifinity/schema.yml diff --git a/models/lifinity/sources.yml b/solana/models/_sector/dex/lifinity/sources.yml similarity index 100% rename from models/lifinity/sources.yml rename to solana/models/_sector/dex/lifinity/sources.yml diff --git a/models/meteora/solana/_schema.yml b/solana/models/_sector/dex/meteora/_schema.yml similarity index 100% rename from models/meteora/solana/_schema.yml rename to solana/models/_sector/dex/meteora/_schema.yml diff --git a/models/meteora/solana/meteora_v1_solana_trades.sql b/solana/models/_sector/dex/meteora/meteora_v1_solana_trades.sql similarity index 100% rename from models/meteora/solana/meteora_v1_solana_trades.sql rename to solana/models/_sector/dex/meteora/meteora_v1_solana_trades.sql diff --git a/models/meteora/solana/meteora_v2_solana_trades.sql b/solana/models/_sector/dex/meteora/meteora_v2_solana_trades.sql similarity index 100% rename from models/meteora/solana/meteora_v2_solana_trades.sql rename to solana/models/_sector/dex/meteora/meteora_v2_solana_trades.sql diff --git a/models/orca_whirlpool/orca_whirlpool_schema.yml b/solana/models/_sector/dex/orca_whirlpool/orca_whirlpool_schema.yml similarity index 100% rename from models/orca_whirlpool/orca_whirlpool_schema.yml rename to solana/models/_sector/dex/orca_whirlpool/orca_whirlpool_schema.yml diff --git a/models/orca_whirlpool/orca_whirlpool_token_prices.sql b/solana/models/_sector/dex/orca_whirlpool/orca_whirlpool_token_prices.sql similarity index 100% rename from models/orca_whirlpool/orca_whirlpool_token_prices.sql rename to solana/models/_sector/dex/orca_whirlpool/orca_whirlpool_token_prices.sql diff --git a/models/orca_whirlpool/orca_whirlpool_token_prices_hour.sql b/solana/models/_sector/dex/orca_whirlpool/orca_whirlpool_token_prices_hour.sql similarity index 100% rename from models/orca_whirlpool/orca_whirlpool_token_prices_hour.sql rename to solana/models/_sector/dex/orca_whirlpool/orca_whirlpool_token_prices_hour.sql diff --git a/models/orca_whirlpool/orca_whirlpool_trades.sql b/solana/models/_sector/dex/orca_whirlpool/orca_whirlpool_trades.sql similarity index 100% rename from models/orca_whirlpool/orca_whirlpool_trades.sql rename to solana/models/_sector/dex/orca_whirlpool/orca_whirlpool_trades.sql diff --git a/models/phoenix/phoenix_v1_trades.sql b/solana/models/_sector/dex/phoenix/phoenix_v1_trades.sql similarity index 100% rename from models/phoenix/phoenix_v1_trades.sql rename to solana/models/_sector/dex/phoenix/phoenix_v1_trades.sql diff --git a/models/phoenix/schema.yml b/solana/models/_sector/dex/phoenix/schema.yml similarity index 100% rename from models/phoenix/schema.yml rename to solana/models/_sector/dex/phoenix/schema.yml diff --git a/models/phoenix/sources.yml b/solana/models/_sector/dex/phoenix/sources.yml similarity index 100% rename from models/phoenix/sources.yml rename to solana/models/_sector/dex/phoenix/sources.yml diff --git a/models/pumpdotfun/solana/pumpdotfun_solana_trades.sql b/solana/models/_sector/dex/pumpdotfun/solana/pumpdotfun_solana_trades.sql similarity index 91% rename from models/pumpdotfun/solana/pumpdotfun_solana_trades.sql rename to solana/models/_sector/dex/pumpdotfun/solana/pumpdotfun_solana_trades.sql index fccddea59a0..214e6dc3c07 100644 --- a/models/pumpdotfun/solana/pumpdotfun_solana_trades.sql +++ b/solana/models/_sector/dex/pumpdotfun/solana/pumpdotfun_solana_trades.sql @@ -18,15 +18,15 @@ {% set project_start_date = '2024-01-14' %} --grabbed program deployed at time (account created at) -with +with bonding_curves as ( SELECT account_arguments[1] as token_mint_address - , account_arguments[3] as bonding_curve + , account_arguments[3] as bonding_curve , account_arguments[4] as bonding_curve_vault FROM {{ source('solana','instruction_calls') }} WHERE executing_account = '6EF8rrecthR5Dkzon8Nwu78hRvfCKubJ14M5uBEwF6P' - AND bytearray_substring(data,1,8) = 0x181ec828051c0777 --Create https://solscan.io/tx/2Vfq4gS9nq2jvpmZSxVXJ3uHGeheENXetwUus6KnhBzFu23Brqbt5EoNiTLds6jr72yZYGJ9YbMDG1BYKMRe3hSQ + AND bytearray_substring(data,1,8) = 0x181ec828051c0777 --Create https://solscan.io/tx/2Vfq4gS9nq2jvpmZSxVXJ3uHGeheENXetwUus6KnhBzFu23Brqbt5EoNiTLds6jr72yZYGJ9YbMDG1BYKMRe3hSQ and tx_success = true {% if is_incremental() %} AND {{incremental_predicate('block_time')}} @@ -35,7 +35,7 @@ with -- and block_time >= now() - interval '7' day {% endif %} ) - + , swaps as ( SELECT to_base58(bytearray_substring(data,1+16,32)) as token_mint_address @@ -56,7 +56,7 @@ with , outer_executing_account FROM {{ source('solana','instruction_calls') }} WHERE executing_account = '6EF8rrecthR5Dkzon8Nwu78hRvfCKubJ14M5uBEwF6P' - AND bytearray_substring(data,1,16) = 0xe445a52e51cb9a1dbddb7fd34ee661ee --SwapEvent + AND bytearray_substring(data,1,16) = 0xe445a52e51cb9a1dbddb7fd34ee661ee --SwapEvent and tx_success = true {% if is_incremental() %} AND {{incremental_predicate('block_time')}} @@ -69,9 +69,9 @@ with --buy https://solscan.io/tx/5782i58ZHCKSANgTi3WXL5vjFAWEVJuPMGAr6gjx2qqcRP6LLNRH2g6pb61tk44H7PC9ohFNCUKQVUAMd5vYeUns --sell https://solscan.io/tx/4thHCu9SX166TP2cnjgwJ7mDSSMn5MBe8xLsTYzRLJmE7jPgwQatXh4ehh3At4xvVcgUefFzzsYVBaVwYyS1bA6v ) - + , trades_base as ( - SELECT + SELECT sp.block_time , 'pumpdotfun' as project , 1 as version @@ -85,8 +85,8 @@ with end as token_pair --bought , case when is_buy = 1 then COALESCE(tk.symbol, sp.token_mint_address) - else COALESCE(tk_sol.symbol, 'So11111111111111111111111111111111111111112') - end as token_bought_symbol + else COALESCE(tk_sol.symbol, 'So11111111111111111111111111111111111111112') + end as token_bought_symbol , case when is_buy = 1 then sp.token_mint_address else 'So11111111111111111111111111111111111111112' end as token_bought_mint_address @@ -94,12 +94,12 @@ with else sol_amount end as token_bought_amount_raw , case when is_buy = 1 then token_amount/pow(10,tk.decimals) - else sol_amount/pow(10,tk_sol.decimals) + else sol_amount/pow(10,tk_sol.decimals) end as token_bought_amount --sold , case when is_buy = 0 then COALESCE(tk.symbol, sp.token_mint_address) - else COALESCE(tk_sol.symbol, 'So11111111111111111111111111111111111111112') - end as token_sold_symbol + else COALESCE(tk_sol.symbol, 'So11111111111111111111111111111111111111112') + end as token_sold_symbol , case when is_buy = 0 then sp.token_mint_address else 'So11111111111111111111111111111111111111112' end as token_sold_mint_address @@ -107,7 +107,7 @@ with else sol_amount end as token_sold_amount_raw , case when is_buy = 0 then token_amount/pow(10,tk.decimals) - else sol_amount/pow(10,tk_sol.decimals) + else sol_amount/pow(10,tk_sol.decimals) end as token_sold_amount , sp.sol_amount*0.01 as sol_fee_raw , sp.sol_amount/pow(10,tk_sol.decimals)*0.01 as sol_fee @@ -134,10 +134,10 @@ with LEFT JOIN {{ ref('tokens_solana_fungible') }} tk_sol ON tk_sol.token_mint_address = 'So11111111111111111111111111111111111111112' LEFT JOIN bonding_curves bc ON bc.token_mint_address = sp.token_mint_address ) - -SELECT + +SELECT tb.blockchain - , tb.project + , tb.project , tb.version , CAST(date_trunc('month', tb.block_time) AS DATE) as block_month , tb.block_time @@ -154,7 +154,7 @@ SELECT , sol_fee_raw , sol_fee , (case when tb.token_bought_mint_address = 'So11111111111111111111111111111111111111112' then p_bought.price - else p_sold.price + else p_sold.price end) * sol_fee as fee_usd --fees are only in sol , tb.token_sold_mint_address , tb.token_bought_mint_address @@ -172,16 +172,16 @@ SELECT , tb.inner_instruction_index , tb.tx_index FROM trades_base tb -LEFT JOIN {{ ref('prices_usd_forward_fill') }} p_bought ON p_bought.blockchain = 'solana' - AND date_trunc('minute', tb.block_time) = p_bought.minute +LEFT JOIN {{ source('prices', 'usd_forward_fill') }} p_bought ON p_bought.blockchain = 'solana' + AND date_trunc('minute', tb.block_time) = p_bought.minute AND token_bought_mint_address = toBase58(p_bought.contract_address) {% if is_incremental() %} AND {{incremental_predicate('p_bought.minute')}} {% else %} AND p_bought.minute >= TIMESTAMP '{{project_start_date}}' {% endif %} -LEFT JOIN {{ ref('prices_usd_forward_fill') }} p_sold ON p_sold.blockchain = 'solana' - AND date_trunc('minute', tb.block_time) = p_sold.minute +LEFT JOIN {{ source('prices', 'usd_forward_fill') }} p_sold ON p_sold.blockchain = 'solana' + AND date_trunc('minute', tb.block_time) = p_sold.minute AND token_sold_mint_address = toBase58(p_sold.contract_address) {% if is_incremental() %} AND {{incremental_predicate('p_sold.minute')}} diff --git a/models/pumpdotfun/solana/schema.yml b/solana/models/_sector/dex/pumpdotfun/solana/schema.yml similarity index 100% rename from models/pumpdotfun/solana/schema.yml rename to solana/models/_sector/dex/pumpdotfun/solana/schema.yml diff --git a/models/raydium/raydium_v3_trades.sql b/solana/models/_sector/dex/raydium/raydium_v3_trades.sql similarity index 100% rename from models/raydium/raydium_v3_trades.sql rename to solana/models/_sector/dex/raydium/raydium_v3_trades.sql diff --git a/models/raydium/raydium_v4_trades.sql b/solana/models/_sector/dex/raydium/raydium_v4_trades.sql similarity index 100% rename from models/raydium/raydium_v4_trades.sql rename to solana/models/_sector/dex/raydium/raydium_v4_trades.sql diff --git a/models/raydium/schema.yml b/solana/models/_sector/dex/raydium/schema.yml similarity index 100% rename from models/raydium/schema.yml rename to solana/models/_sector/dex/raydium/schema.yml diff --git a/models/raydium/sources.yml b/solana/models/_sector/dex/raydium/sources.yml similarity index 100% rename from models/raydium/sources.yml rename to solana/models/_sector/dex/raydium/sources.yml diff --git a/models/hivemapper/solana/hivemapper_solana_rewards.sql b/solana/models/hivemapper/hivemapper_solana_rewards.sql similarity index 100% rename from models/hivemapper/solana/hivemapper_solana_rewards.sql rename to solana/models/hivemapper/hivemapper_solana_rewards.sql diff --git a/models/hivemapper/solana/hivemapper_solana_schema.yml b/solana/models/hivemapper/hivemapper_solana_schema.yml similarity index 100% rename from models/hivemapper/solana/hivemapper_solana_schema.yml rename to solana/models/hivemapper/hivemapper_solana_schema.yml diff --git a/models/solana_utils/solana_utils_daily_balances.sql b/solana/models/solana_utils/solana_utils_daily_balances.sql similarity index 100% rename from models/solana_utils/solana_utils_daily_balances.sql rename to solana/models/solana_utils/solana_utils_daily_balances.sql diff --git a/models/solana_utils/solana_utils_latest_balances.sql b/solana/models/solana_utils/solana_utils_latest_balances.sql similarity index 100% rename from models/solana_utils/solana_utils_latest_balances.sql rename to solana/models/solana_utils/solana_utils_latest_balances.sql diff --git a/models/solana_utils/solana_utils_schema.yml b/solana/models/solana_utils/solana_utils_schema.yml similarity index 100% rename from models/solana_utils/solana_utils_schema.yml rename to solana/models/solana_utils/solana_utils_schema.yml diff --git a/models/solana_utils/solana_utils_sns_domains.sql b/solana/models/solana_utils/solana_utils_sns_domains.sql similarity index 100% rename from models/solana_utils/solana_utils_sns_domains.sql rename to solana/models/solana_utils/solana_utils_sns_domains.sql diff --git a/models/solana_utils/solana_utils_token_accounts.sql b/solana/models/solana_utils/solana_utils_token_accounts.sql similarity index 100% rename from models/solana_utils/solana_utils_token_accounts.sql rename to solana/models/solana_utils/solana_utils_token_accounts.sql diff --git a/models/solana_utils/solana_utils_total_rewards.sql b/solana/models/solana_utils/solana_utils_total_rewards.sql similarity index 100% rename from models/solana_utils/solana_utils_total_rewards.sql rename to solana/models/solana_utils/solana_utils_total_rewards.sql diff --git a/models/tokens/solana/tokens_solana.sources.yml b/solana/models/tokens/solana/tokens_solana.sources.yml similarity index 100% rename from models/tokens/solana/tokens_solana.sources.yml rename to solana/models/tokens/solana/tokens_solana.sources.yml diff --git a/models/tokens/solana/tokens_solana_fees_history.sql b/solana/models/tokens/solana/tokens_solana_fees_history.sql similarity index 100% rename from models/tokens/solana/tokens_solana_fees_history.sql rename to solana/models/tokens/solana/tokens_solana_fees_history.sql diff --git a/models/tokens/solana/tokens_solana_fungible.sql b/solana/models/tokens/solana/tokens_solana_fungible.sql similarity index 100% rename from models/tokens/solana/tokens_solana_fungible.sql rename to solana/models/tokens/solana/tokens_solana_fungible.sql diff --git a/models/tokens/solana/tokens_solana_nft.sql b/solana/models/tokens/solana/tokens_solana_nft.sql similarity index 100% rename from models/tokens/solana/tokens_solana_nft.sql rename to solana/models/tokens/solana/tokens_solana_nft.sql diff --git a/models/tokens/solana/tokens_solana_schema.yml b/solana/models/tokens/solana/tokens_solana_schema.yml similarity index 100% rename from models/tokens/solana/tokens_solana_schema.yml rename to solana/models/tokens/solana/tokens_solana_schema.yml diff --git a/models/tokens/solana/tokens_solana_spl_transfers.sql b/solana/models/tokens/solana/tokens_solana_spl_transfers.sql similarity index 100% rename from models/tokens/solana/tokens_solana_spl_transfers.sql rename to solana/models/tokens/solana/tokens_solana_spl_transfers.sql diff --git a/models/tokens/solana/tokens_solana_spl_transfers_call_transfer.sql b/solana/models/tokens/solana/tokens_solana_spl_transfers_call_transfer.sql similarity index 100% rename from models/tokens/solana/tokens_solana_spl_transfers_call_transfer.sql rename to solana/models/tokens/solana/tokens_solana_spl_transfers_call_transfer.sql diff --git a/models/tokens/solana/tokens_solana_token22_spl_transfers.sql b/solana/models/tokens/solana/tokens_solana_token22_spl_transfers.sql similarity index 100% rename from models/tokens/solana/tokens_solana_token22_spl_transfers.sql rename to solana/models/tokens/solana/tokens_solana_token22_spl_transfers.sql diff --git a/models/tokens/solana/tokens_solana_transfers.sql b/solana/models/tokens/solana/tokens_solana_transfers.sql similarity index 100% rename from models/tokens/solana/tokens_solana_transfers.sql rename to solana/models/tokens/solana/tokens_solana_transfers.sql diff --git a/solana/package-lock.yml b/solana/package-lock.yml new file mode 100644 index 00000000000..8f9a42c7b94 --- /dev/null +++ b/solana/package-lock.yml @@ -0,0 +1,4 @@ +packages: +- package: dbt-labs/dbt_utils + version: 1.2.0 +sha1_hash: d4f259856543b0ef301e0b3b0bbc94ccb6b12a54 diff --git a/solana/packages.yml b/solana/packages.yml new file mode 100644 index 00000000000..d4f9f38e076 --- /dev/null +++ b/solana/packages.yml @@ -0,0 +1,3 @@ +packages: + - package: dbt-labs/dbt_utils + version: 1.2.0 \ No newline at end of file diff --git a/solana/profiles.yml b/solana/profiles.yml new file mode 100644 index 00000000000..10d52bd460f --- /dev/null +++ b/solana/profiles.yml @@ -0,0 +1,12 @@ +spellbook-local: + target: dev + outputs: + dev: + type: trino + user: trino + password: trino + host: trino + port: 1234 + database: hive + schema: wizard + threads: 1 \ No newline at end of file diff --git a/seeds/banana_gun/banana_gun_solana_trades_seed.csv b/solana/seeds/banana_gun/banana_gun_solana_trades_seed.csv similarity index 100% rename from seeds/banana_gun/banana_gun_solana_trades_seed.csv rename to solana/seeds/banana_gun/banana_gun_solana_trades_seed.csv diff --git a/seeds/readyswap/readyswap_solana_trades_seed.csv b/solana/seeds/banana_gun/readyswap/readyswap_solana_trades_seed.csv similarity index 100% rename from seeds/readyswap/readyswap_solana_trades_seed.csv rename to solana/seeds/banana_gun/readyswap/readyswap_solana_trades_seed.csv diff --git a/seeds/bonkbot/bonkbot_solana_trades_seed.csv b/solana/seeds/bonkbot/bonkbot_solana_trades_seed.csv similarity index 100% rename from seeds/bonkbot/bonkbot_solana_trades_seed.csv rename to solana/seeds/bonkbot/bonkbot_solana_trades_seed.csv diff --git a/seeds/consortium_key/consortium_key_solana_trades_seed.csv b/solana/seeds/consortium_key/consortium_key_solana_trades_seed.csv similarity index 100% rename from seeds/consortium_key/consortium_key_solana_trades_seed.csv rename to solana/seeds/consortium_key/consortium_key_solana_trades_seed.csv diff --git a/seeds/looter/looter_solana_trades_seed.csv b/solana/seeds/looter/looter_solana_trades_seed.csv similarity index 100% rename from seeds/looter/looter_solana_trades_seed.csv rename to solana/seeds/looter/looter_solana_trades_seed.csv diff --git a/seeds/maestro/maestro_solana_trades_seed.csv b/solana/seeds/maestro/maestro_solana_trades_seed.csv similarity index 100% rename from seeds/maestro/maestro_solana_trades_seed.csv rename to solana/seeds/maestro/maestro_solana_trades_seed.csv diff --git a/seeds/magnum/magnum_solana_trades_seed.csv b/solana/seeds/magnum/magnum_solana_trades_seed.csv similarity index 100% rename from seeds/magnum/magnum_solana_trades_seed.csv rename to solana/seeds/magnum/magnum_solana_trades_seed.csv diff --git a/seeds/pepe_boost/pepe_boost_solana_trades_seed.csv b/solana/seeds/pepe_boost/pepe_boost_solana_trades_seed.csv similarity index 100% rename from seeds/pepe_boost/pepe_boost_solana_trades_seed.csv rename to solana/seeds/pepe_boost/pepe_boost_solana_trades_seed.csv diff --git a/seeds/shuriken/shuriken_solana_trades_seed.csv b/solana/seeds/shuriken/shuriken_solana_trades_seed.csv similarity index 100% rename from seeds/shuriken/shuriken_solana_trades_seed.csv rename to solana/seeds/shuriken/shuriken_solana_trades_seed.csv diff --git a/seeds/sol_gun/sol_gun_solana_trades_seed.csv b/solana/seeds/sol_gun/sol_gun_solana_trades_seed.csv similarity index 100% rename from seeds/sol_gun/sol_gun_solana_trades_seed.csv rename to solana/seeds/sol_gun/sol_gun_solana_trades_seed.csv diff --git a/seeds/sol_trading_bot/sol_trading_bot_solana_trades_seed.csv b/solana/seeds/sol_trading_bot/sol_trading_bot_solana_trades_seed.csv similarity index 100% rename from seeds/sol_trading_bot/sol_trading_bot_solana_trades_seed.csv rename to solana/seeds/sol_trading_bot/sol_trading_bot_solana_trades_seed.csv diff --git a/seeds/soulbot/soul_sniper_solana_trades_seed.csv b/solana/seeds/soulbot/soul_sniper_solana_trades_seed.csv similarity index 100% rename from seeds/soulbot/soul_sniper_solana_trades_seed.csv rename to solana/seeds/soulbot/soul_sniper_solana_trades_seed.csv diff --git a/seeds/trojan/trojan_solana_trades_seed.csv b/solana/seeds/trojan/trojan_solana_trades_seed.csv similarity index 100% rename from seeds/trojan/trojan_solana_trades_seed.csv rename to solana/seeds/trojan/trojan_solana_trades_seed.csv diff --git a/seeds/wifbot/wifbot_solana_trades_seed.csv b/solana/seeds/wifbot/wifbot_solana_trades_seed.csv similarity index 100% rename from seeds/wifbot/wifbot_solana_trades_seed.csv rename to solana/seeds/wifbot/wifbot_solana_trades_seed.csv diff --git a/tests/generic/check_bot_trades_seed.sql b/solana/tests/generic/check_bot_trades_seed.sql similarity index 100% rename from tests/generic/check_bot_trades_seed.sql rename to solana/tests/generic/check_bot_trades_seed.sql