diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml index 65bfb95..62c6e05 100644 --- a/.github/workflows/dbt_run_adhoc.yml +++ b/.github/workflows/dbt_run_adhoc.yml @@ -60,7 +60,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_core.yml b/.github/workflows/dbt_run_core.yml index 76a513f..2984b65 100644 --- a/.github/workflows/dbt_run_core.yml +++ b/.github/workflows/dbt_run_core.yml @@ -39,7 +39,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core + pip install -r requirements.txt workstream-dbt-core dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_daily.yml b/.github/workflows/dbt_run_daily.yml index 516d443..a7014b0 100644 --- a/.github/workflows/dbt_run_daily.yml +++ b/.github/workflows/dbt_run_daily.yml @@ -39,7 +39,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core + pip install -r requirements.txt workstream-dbt-core dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml index 1a65135..7ed753f 100644 --- a/.github/workflows/dbt_run_dev_refresh.yml +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -38,7 +38,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core + pip install -r requirements.txt workstream-dbt-core dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_non_core.yml b/.github/workflows/dbt_run_non_core.yml index c52d377..1dc6db2 100644 --- a/.github/workflows/dbt_run_non_core.yml +++ b/.github/workflows/dbt_run_non_core.yml @@ -39,7 +39,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core + pip install -r requirements.txt workstream-dbt-core dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_observability.yml b/.github/workflows/dbt_run_observability.yml index 86910fe..63c8b5a 100644 --- a/.github/workflows/dbt_run_observability.yml +++ b/.github/workflows/dbt_run_observability.yml @@ -39,7 +39,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core + pip install -r requirements.txt workstream-dbt-core dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_observability_monthly.yml b/.github/workflows/dbt_run_observability_monthly.yml index a258221..1b34e51 100644 --- a/.github/workflows/dbt_run_observability_monthly.yml +++ b/.github/workflows/dbt_run_observability_monthly.yml @@ -38,7 +38,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core + pip install -r requirements.txt workstream-dbt-core dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_streamline_blocks_txcount_realtime.yml b/.github/workflows/dbt_run_streamline_blocks_txcount_realtime.yml index 13b4eca..f15c86c 100644 --- a/.github/workflows/dbt_run_streamline_blocks_txcount_realtime.yml +++ b/.github/workflows/dbt_run_streamline_blocks_txcount_realtime.yml @@ -38,7 +38,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core + pip install -r requirements.txt workstream-dbt-core dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_streamline_transactions_realtime.yml b/.github/workflows/dbt_run_streamline_transactions_realtime.yml index e9e0e92..b23ae09 100644 --- a/.github/workflows/dbt_run_streamline_transactions_realtime.yml +++ b/.github/workflows/dbt_run_streamline_transactions_realtime.yml @@ -38,7 +38,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core + pip install -r requirements.txt workstream-dbt-core dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_test_daily.yml b/.github/workflows/dbt_test_daily.yml index d01e7a9..2c0ed6a 100644 --- a/.github/workflows/dbt_test_daily.yml +++ b/.github/workflows/dbt_test_daily.yml @@ -39,7 +39,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core + pip install -r requirements.txt workstream-dbt-core dbt deps - name: Run DBT Jobs run: | diff --git a/models/bronze/bronze_api/bronze_api__get_contract_contract_info.sql b/models/bronze/bronze_api/bronze_api__get_contract_contract_info.sql index 6e35c2a..e8e2335 100644 --- a/models/bronze/bronze_api/bronze_api__get_contract_contract_info.sql +++ b/models/bronze/bronze_api/bronze_api__get_contract_contract_info.sql @@ -6,13 +6,14 @@ WITH contracts AS ( SELECT - top 20 A.contract_address + top 20 A.nft_address AS contract_address FROM - {{ ref('silver__contracts') }} A + {# {{ ref('silver__contracts') }} A #} + sei_dev.silver.nft_sales_pallet A {% if is_incremental() %} LEFT JOIN silver.contract_contract_info b -ON A.contract_address = b.contract_address +ON A.nft_address = b.contract_address WHERE ( b.contract_address IS NULL diff --git a/models/silver/nft/silver__nft_sales_pallet.sql b/models/silver/nft/silver__nft_sales_pallet.sql new file mode 100644 index 0000000..a8dfdbc --- /dev/null +++ b/models/silver/nft/silver__nft_sales_pallet.sql @@ -0,0 +1,168 @@ +{{ config( + materialized = 'incremental', + unique_key = ['tx_id','msg_index'], + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['_inserted_timestamp::DATE', 'block_timestamp::DATE' ], + tags = ['noncore'] +) }} + +WITH msg_atts_base AS ( + + SELECT + block_id, + block_timestamp, + tx_succeeded, + tx_id, + msg_group, + msg_sub_group, + msg_index, + msg_type, + _inserted_timestamp, + attribute_key, + attribute_value + FROM + {{ ref('silver__msg_attributes') }} + WHERE + tx_succeeded + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) + FROM + {{ this }} +) +{% endif %} +), +nft_sales_tx AS ( + SELECT + tx_id + FROM + msg_atts_base + WHERE + -- BUYS on Pallet Exchange contract: sei152u2u0lqc27428cuf8dx48k8saua74m6nql5kgvsu4rfeqm547rsnhy4y9 + msg_type IN ( + 'wasm-buy_now', + 'wasm-accept_bid' + ) + AND attribute_key = '_contract_address' + AND attribute_value = 'sei152u2u0lqc27428cuf8dx48k8saua74m6nql5kgvsu4rfeqm547rsnhy4y9' qualify(ROW_NUMBER() over (PARTITION BY tx_id + ORDER BY + msg_index)) = 1 +), +nft_sales_buydata AS ( + SELECT + A.block_id, + A.block_timestamp, + A.tx_succeeded, + A.tx_id, + A.msg_group, + A.msg_sub_group, + A.msg_index, + A.msg_type, + A._inserted_timestamp, + OBJECT_AGG( + attribute_key :: STRING, + attribute_value :: variant + ) AS j, + j :nft_address :: STRING AS nft_address, + j :nft_token_id :: STRING AS token_id, + j :nft_seller :: STRING AS nft_seller, + j :_contract_address :: STRING AS marketplace_contract, + j :sale_price :: STRING AS amount_raw, + SPLIT_PART( + TRIM( + REGEXP_REPLACE( + amount_raw, + '[^[:digit:]]', + ' ' + ) + ), + ' ', + 0 + ) AS amount, + RIGHT(amount_raw, LENGTH(amount_raw) - LENGTH(SPLIT_PART(TRIM(REGEXP_REPLACE(amount_raw, '[^[:digit:]]', ' ')), ' ', 0))) AS currency + FROM + msg_atts_base A + INNER JOIN nft_sales_tx s USING (tx_id) + WHERE + A.msg_type IN ( + 'wasm-buy_now', + 'wasm-accept_bid' + ) + GROUP BY + A.block_id, + A.block_timestamp, + A.tx_succeeded, + A.tx_id, + A.msg_group, + A.msg_sub_group, + A.msg_index, + A.msg_type, + A._inserted_timestamp +), +nft_sales_transfer_data AS ( + SELECT + A.block_id, + A.block_timestamp, + A.tx_succeeded, + A.tx_id, + A.msg_group, + A.msg_sub_group, + A.msg_index, + A._inserted_timestamp, + OBJECT_AGG( + attribute_key :: STRING, + attribute_value :: variant + ) AS j, + j :sender :: STRING AS marketplace_contract, + j :token_id :: STRING AS token_id, + j :recipient :: STRING AS nft_buyer, + j :_contract_address :: STRING AS nft_address + FROM + msg_atts_base A + INNER JOIN nft_sales_tx s USING (tx_id) + WHERE + msg_type = 'wasm' + GROUP BY + A.block_id, + A.block_timestamp, + A.tx_succeeded, + A.tx_id, + A.msg_group, + A.msg_sub_group, + A.msg_index, + A._inserted_timestamp +) +SELECT + A.block_id, + A.block_timestamp, + A.tx_succeeded, + A.tx_id, + A.msg_group, + A.msg_sub_group, + A.msg_index, + A.msg_type, + A.nft_address, + A.token_id, + b.nft_buyer AS buyer_address, + A.nft_seller AS seller_address, + A.amount_raw, + A.amount, + A.currency, + A.marketplace_contract, + {{ dbt_utils.generate_surrogate_key( + ['a.tx_id','a.msg_index'] + ) }} AS nft_sales_pallet_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + nft_sales_buydata A + LEFT JOIN nft_sales_transfer_data b + ON A.tx_id = b.tx_id + AND A.nft_address = b.nft_address + AND A.token_id = b.token_id + AND A.marketplace_contract = b.marketplace_contract diff --git a/models/silver/nft/silver__nft_sales_pallet.yml b/models/silver/nft/silver__nft_sales_pallet.yml new file mode 100644 index 0000000..fb2b46f --- /dev/null +++ b/models/silver/nft/silver__nft_sales_pallet.yml @@ -0,0 +1,61 @@ +version: 2 +models: + - name: silver__nft_sales_pallet + description: Records of nft sales on pallet + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - MSG_INDEX + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: TX_SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + tests: + - not_null + - name: MSG_GROUP + description: "{{ doc('msg_group') }}" + tests: + - not_null + - name: MSG_SUB_GROUP + description: "{{ doc('msg_sub_group') }}" + tests: + - not_null + - name: MSG_INDEX + description: "{{ doc('msg_index') }}" + tests: + - not_null + - name: NFT_ADDRESS + # description: "{{ doc('nft_address') }}" + tests: + - not_null + - name: BUYER_ADDRESS + # description: "{{ doc('buyer_address') }}" + tests: + - not_null + - name: SELLER_ADDRESS + # description: "{{ doc('seller_address') }}" + tests: + - not_null + - name: AMOUNT + description: "{{ doc('amount') }}" + tests: + - not_null + - name: CURRENCY + description: "{{ doc('currency') }}" + tests: + - not_null + - name: MARKETPLACE_CONTRACT + # description: "{{ doc('marketplace_contract') }}" + diff --git a/requirements.txt b/requirements.txt index 5921743..39b82bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -dbt-snowflake>=1.7,<1.8 \ No newline at end of file +dbt-snowflake>=1.7,<1.8 +protobuf==4.25.3 \ No newline at end of file