workflows to reqs

.github/workflows/dbt_run_adhoc.yml

@@ -60,7 +60,7 @@ jobs:
 
       - name: install dependencies
         run: |
-          pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click
+          pip install -r requirements.txt
           dbt deps
       - name: Run DBT Jobs
         run: |

.github/workflows/dbt_run_core.yml

@@ -39,7 +39,7 @@ jobs:
 
       - name: install dependencies
         run: |
-          pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core
+          pip install -r requirements.txt workstream-dbt-core
           dbt deps
       - name: Run DBT Jobs
         run: |

.github/workflows/dbt_run_daily.yml

@@ -39,7 +39,7 @@ jobs:
 
       - name: install dependencies
         run: |
-          pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core
+          pip install -r requirements.txt workstream-dbt-core
           dbt deps
       - name: Run DBT Jobs
         run: |

.github/workflows/dbt_run_dev_refresh.yml

@@ -38,7 +38,7 @@ jobs:
 
       - name: install dependencies
         run: |
-          pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core
+          pip install -r requirements.txt workstream-dbt-core
           dbt deps
       - name: Run DBT Jobs
         run: |

.github/workflows/dbt_run_non_core.yml

@@ -39,7 +39,7 @@ jobs:
 
       - name: install dependencies
         run: |
-          pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core
+          pip install -r requirements.txt workstream-dbt-core
           dbt deps
       - name: Run DBT Jobs
         run: |

.github/workflows/dbt_run_observability.yml

@@ -39,7 +39,7 @@ jobs:
 
       - name: install dependencies
         run: |
-          pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core
+          pip install -r requirements.txt workstream-dbt-core
           dbt deps
       - name: Run DBT Jobs
         run: |

.github/workflows/dbt_run_observability_monthly.yml

@@ -38,7 +38,7 @@ jobs:
 
       - name: install dependencies
         run: |
-          pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core
+          pip install -r requirements.txt workstream-dbt-core
           dbt deps
       - name: Run DBT Jobs
         run: |

.github/workflows/dbt_run_streamline_blocks_txcount_realtime.yml

@@ -38,7 +38,7 @@ jobs:
 
       - name: install dependencies
         run: |
-          pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core
+          pip install -r requirements.txt workstream-dbt-core
           dbt deps
       - name: Run DBT Jobs
         run: |

.github/workflows/dbt_run_streamline_transactions_realtime.yml

@@ -38,7 +38,7 @@ jobs:
 
       - name: install dependencies
         run: |
-          pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core
+          pip install -r requirements.txt workstream-dbt-core
           dbt deps
       - name: Run DBT Jobs
         run: |

.github/workflows/dbt_test_daily.yml

@@ -39,7 +39,7 @@ jobs:
 
       - name: install dependencies
         run: |
-          pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core
+          pip install -r requirements.txt workstream-dbt-core
           dbt deps
       - name: Run DBT Jobs
         run: |

models/bronze/bronze_api/bronze_api__get_contract_contract_info.sql

@@ -6,13 +6,14 @@
 WITH contracts AS (
 
     SELECT
-        top 20 A.contract_address
+        top 20 A.nft_address AS contract_address
     FROM
-        {{ ref('silver__contracts') }} A
+        {# {{ ref('silver__contracts') }} A #}
+        sei_dev.silver.nft_sales_pallet A
 
 {% if is_incremental() %}
 LEFT JOIN silver.contract_contract_info b
-ON A.contract_address = b.contract_address
+ON A.nft_address = b.contract_address
 WHERE
     (
         b.contract_address IS NULL

models/silver/nft/silver__nft_sales_pallet.sql

@@ -0,0 +1,168 @@
+{{ config(
+    materialized = 'incremental',
+    unique_key = ['tx_id','msg_index'],
+    incremental_strategy = 'merge',
+    merge_exclude_columns = ["inserted_timestamp"],
+    cluster_by = ['_inserted_timestamp::DATE', 'block_timestamp::DATE' ],
+    tags = ['noncore']
+) }}
+
+WITH msg_atts_base AS (
+
+    SELECT
+        block_id,
+        block_timestamp,
+        tx_succeeded,
+        tx_id,
+        msg_group,
+        msg_sub_group,
+        msg_index,
+        msg_type,
+        _inserted_timestamp,
+        attribute_key,
+        attribute_value
+    FROM
+        {{ ref('silver__msg_attributes') }}
+    WHERE
+        tx_succeeded
+
+{% if is_incremental() %}
+AND _inserted_timestamp >= (
+    SELECT
+        MAX(_inserted_timestamp)
+    FROM
+        {{ this }}
+)
+{% endif %}
+),
+nft_sales_tx AS (
+    SELECT
+        tx_id
+    FROM
+        msg_atts_base
+    WHERE
+        -- BUYS on Pallet Exchange contract: sei152u2u0lqc27428cuf8dx48k8saua74m6nql5kgvsu4rfeqm547rsnhy4y9
+        msg_type IN (
+            'wasm-buy_now',
+            'wasm-accept_bid'
+        )
+        AND attribute_key = '_contract_address'
+        AND attribute_value = 'sei152u2u0lqc27428cuf8dx48k8saua74m6nql5kgvsu4rfeqm547rsnhy4y9' qualify(ROW_NUMBER() over (PARTITION BY tx_id
+    ORDER BY
+        msg_index)) = 1
+),
+nft_sales_buydata AS (
+    SELECT
+        A.block_id,
+        A.block_timestamp,
+        A.tx_succeeded,
+        A.tx_id,
+        A.msg_group,
+        A.msg_sub_group,
+        A.msg_index,
+        A.msg_type,
+        A._inserted_timestamp,
+        OBJECT_AGG(
+            attribute_key :: STRING,
+            attribute_value :: variant
+        ) AS j,
+        j :nft_address :: STRING AS nft_address,
+        j :nft_token_id :: STRING AS token_id,
+        j :nft_seller :: STRING AS nft_seller,
+        j :_contract_address :: STRING AS marketplace_contract,
+        j :sale_price :: STRING AS amount_raw,
+        SPLIT_PART(
+            TRIM(
+                REGEXP_REPLACE(
+                    amount_raw,
+                    '[^[:digit:]]',
+                    ' '
+                )
+            ),
+            ' ',
+            0
+        ) AS amount,
+        RIGHT(amount_raw, LENGTH(amount_raw) - LENGTH(SPLIT_PART(TRIM(REGEXP_REPLACE(amount_raw, '[^[:digit:]]', ' ')), ' ', 0))) AS currency
+    FROM
+        msg_atts_base A
+        INNER JOIN nft_sales_tx s USING (tx_id)
+    WHERE
+        A.msg_type IN (
+            'wasm-buy_now',
+            'wasm-accept_bid'
+        )
+    GROUP BY
+        A.block_id,
+        A.block_timestamp,
+        A.tx_succeeded,
+        A.tx_id,
+        A.msg_group,
+        A.msg_sub_group,
+        A.msg_index,
+        A.msg_type,
+        A._inserted_timestamp
+),
+nft_sales_transfer_data AS (
+    SELECT
+        A.block_id,
+        A.block_timestamp,
+        A.tx_succeeded,
+        A.tx_id,
+        A.msg_group,
+        A.msg_sub_group,
+        A.msg_index,
+        A._inserted_timestamp,
+        OBJECT_AGG(
+            attribute_key :: STRING,
+            attribute_value :: variant
+        ) AS j,
+        j :sender :: STRING AS marketplace_contract,
+        j :token_id :: STRING AS token_id,
+        j :recipient :: STRING AS nft_buyer,
+        j :_contract_address :: STRING AS nft_address
+    FROM
+        msg_atts_base A
+        INNER JOIN nft_sales_tx s USING (tx_id)
+    WHERE
+        msg_type = 'wasm'
+    GROUP BY
+        A.block_id,
+        A.block_timestamp,
+        A.tx_succeeded,
+        A.tx_id,
+        A.msg_group,
+        A.msg_sub_group,
+        A.msg_index,
+        A._inserted_timestamp
+)
+SELECT
+    A.block_id,
+    A.block_timestamp,
+    A.tx_succeeded,
+    A.tx_id,
+    A.msg_group,
+    A.msg_sub_group,
+    A.msg_index,
+    A.msg_type,
+    A.nft_address,
+    A.token_id,
+    b.nft_buyer AS buyer_address,
+    A.nft_seller AS seller_address,
+    A.amount_raw,
+    A.amount,
+    A.currency,
+    A.marketplace_contract,
+    {{ dbt_utils.generate_surrogate_key(
+        ['a.tx_id','a.msg_index']
+    ) }} AS nft_sales_pallet_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    A._inserted_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
+FROM
+    nft_sales_buydata A
+    LEFT JOIN nft_sales_transfer_data b
+    ON A.tx_id = b.tx_id
+    AND A.nft_address = b.nft_address
+    AND A.token_id = b.token_id
+    AND A.marketplace_contract = b.marketplace_contract

models/silver/nft/silver__nft_sales_pallet.yml

@@ -0,0 +1,61 @@
+version: 2
+models:
+  - name: silver__nft_sales_pallet
+    description: Records of nft sales on pallet
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - TX_ID
+            - MSG_INDEX
+    columns:
+      - name: BLOCK_ID
+        description: "{{ doc('block_id') }}"
+      - name: BLOCK_TIMESTAMP
+        description: "{{ doc('block_timestamp') }}"
+        tests:
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 1
+      - name: TX_ID
+        description: "{{ doc('tx_id') }}"
+        tests: 
+          - not_null
+      - name: TX_SUCCEEDED
+        description: "{{ doc('tx_succeeded') }}"
+        tests: 
+          - not_null
+      - name: MSG_GROUP
+        description: "{{ doc('msg_group') }}"
+        tests: 
+          - not_null
+      - name: MSG_SUB_GROUP
+        description: "{{ doc('msg_sub_group') }}"
+        tests: 
+          - not_null
+      - name: MSG_INDEX
+        description: "{{ doc('msg_index') }}"
+        tests: 
+          - not_null
+      - name: NFT_ADDRESS
+        # description: "{{ doc('nft_address') }}"
+        tests: 
+          - not_null
+      - name: BUYER_ADDRESS
+        # description: "{{ doc('buyer_address') }}"
+        tests: 
+          - not_null
+      - name: SELLER_ADDRESS
+        # description: "{{ doc('seller_address') }}"
+        tests: 
+          - not_null
+      - name: AMOUNT
+        description: "{{ doc('amount') }}"
+        tests: 
+          - not_null
+      - name: CURRENCY
+        description: "{{ doc('currency') }}"
+        tests: 
+          - not_null
+      - name: MARKETPLACE_CONTRACT
+        # description: "{{ doc('marketplace_contract') }}"
+  

requirements.txt

@@ -1 +1,2 @@
-dbt-snowflake>=1.7,<1.8
+dbt-snowflake>=1.7,<1.8
+protobuf==4.25.3