diff --git a/.github/workflows/dbt_run_core.yml b/.github/workflows/dbt_run_core.yml index 2984b65..57af016 100644 --- a/.github/workflows/dbt_run_core.yml +++ b/.github/workflows/dbt_run_core.yml @@ -39,12 +39,8 @@ jobs: - name: install dependencies run: | - pip install -r requirements.txt workstream-dbt-core + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | - dbt run -m models/bronze/core "sei_models,tag:core" || true - workstream report --exit-nonzero - env: - WORKSTREAM_DBT_CLIENT_ID: ${{ secrets.WORKSTREAM_DBT_CLIENT_ID }} - WORKSTREAM_DBT_CLIENT_SECRET: ${{ secrets.WORKSTREAM_DBT_CLIENT_SECRET }} + dbt run -m models/bronze/core "sei_models,tag:core" diff --git a/.github/workflows/dbt_run_daily.yml b/.github/workflows/dbt_run_daily.yml index a7014b0..8edcaea 100644 --- a/.github/workflows/dbt_run_daily.yml +++ b/.github/workflows/dbt_run_daily.yml @@ -39,12 +39,8 @@ jobs: - name: install dependencies run: | - pip install -r requirements.txt workstream-dbt-core + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | - dbt run -m "sei_models,tag:daily" --exclude models/bronze/bronze_api/bronze_api__get_seaswap_pools.sql || true - workstream report --exit-nonzero - env: - WORKSTREAM_DBT_CLIENT_ID: ${{ secrets.WORKSTREAM_DBT_CLIENT_ID }} - WORKSTREAM_DBT_CLIENT_SECRET: ${{ secrets.WORKSTREAM_DBT_CLIENT_SECRET }} \ No newline at end of file + dbt run -m "sei_models,tag:daily" --exclude models/bronze/bronze_api/bronze_api__get_seaswap_pools.sql \ No newline at end of file diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml index 7ed753f..c8a9af7 100644 --- a/.github/workflows/dbt_run_dev_refresh.yml +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -38,12 +38,8 @@ jobs: - name: install dependencies run: | - pip install -r requirements.txt workstream-dbt-core + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | - dbt run-operation run_sp_create_prod_clone || true - workstream report --exit-nonzero - env: - WORKSTREAM_DBT_CLIENT_ID: ${{ secrets.WORKSTREAM_DBT_CLIENT_ID }} - WORKSTREAM_DBT_CLIENT_SECRET: ${{ secrets.WORKSTREAM_DBT_CLIENT_SECRET }} \ No newline at end of file + dbt run-operation run_sp_create_prod_clone \ No newline at end of file diff --git a/.github/workflows/dbt_run_non_core.yml b/.github/workflows/dbt_run_non_core.yml index 1dc6db2..12f8134 100644 --- a/.github/workflows/dbt_run_non_core.yml +++ b/.github/workflows/dbt_run_non_core.yml @@ -39,12 +39,8 @@ jobs: - name: install dependencies run: | - pip install -r requirements.txt workstream-dbt-core + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | - dbt run -m "sei_models,tag:noncore" models/bronze/bronze_api --exclude models/bronze/bronze_api/bronze_api__get_seaswap_pools.sql || true - workstream report --exit-nonzero - env: - WORKSTREAM_DBT_CLIENT_ID: ${{ secrets.WORKSTREAM_DBT_CLIENT_ID }} - WORKSTREAM_DBT_CLIENT_SECRET: ${{ secrets.WORKSTREAM_DBT_CLIENT_SECRET }} \ No newline at end of file + dbt run -m "sei_models,tag:noncore" models/bronze/bronze_api --exclude models/bronze/bronze_api/bronze_api__get_seaswap_pools.sql \ No newline at end of file diff --git a/.github/workflows/dbt_run_observability.yml b/.github/workflows/dbt_run_observability.yml index 63c8b5a..f5663f8 100644 --- a/.github/workflows/dbt_run_observability.yml +++ b/.github/workflows/dbt_run_observability.yml @@ -39,12 +39,8 @@ jobs: - name: install dependencies run: | - pip install -r requirements.txt workstream-dbt-core + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | - dbt run -m models/silver/_observability/silver_observability* || true - workstream report --exit-nonzero - env: - WORKSTREAM_DBT_CLIENT_ID: ${{ secrets.WORKSTREAM_DBT_CLIENT_ID }} - WORKSTREAM_DBT_CLIENT_SECRET: ${{ secrets.WORKSTREAM_DBT_CLIENT_SECRET }} \ No newline at end of file + dbt run -m models/silver/_observability/silver_observability* \ No newline at end of file diff --git a/.github/workflows/dbt_run_observability_monthly.yml b/.github/workflows/dbt_run_observability_monthly.yml index 1b34e51..19c6d8f 100644 --- a/.github/workflows/dbt_run_observability_monthly.yml +++ b/.github/workflows/dbt_run_observability_monthly.yml @@ -38,12 +38,8 @@ jobs: - name: install dependencies run: | - pip install -r requirements.txt workstream-dbt-core + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | - dbt run -m models/silver/_observability/silver_observability* --vars "OBSERV_FULL_TEST: true" || true - workstream report --exit-nonzero - env: - WORKSTREAM_DBT_CLIENT_ID: ${{ secrets.WORKSTREAM_DBT_CLIENT_ID }} - WORKSTREAM_DBT_CLIENT_SECRET: ${{ secrets.WORKSTREAM_DBT_CLIENT_SECRET }} \ No newline at end of file + dbt run -m models/silver/_observability/silver_observability* --vars "OBSERV_FULL_TEST: true" \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_blocks_txcount_realtime.yml b/.github/workflows/dbt_run_streamline_blocks_txcount_realtime.yml index 4f3a004..7db161a 100644 --- a/.github/workflows/dbt_run_streamline_blocks_txcount_realtime.yml +++ b/.github/workflows/dbt_run_streamline_blocks_txcount_realtime.yml @@ -38,12 +38,8 @@ jobs: - name: install dependencies run: | - pip install -r requirements.txt workstream-dbt-core + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/streamline/core/realtime/streamline__tx_counts_realtime.sql 2+models/streamline/core/realtime/streamline__blocks_realtime.sql || true - workstream report --exit-nonzero - env: - WORKSTREAM_DBT_CLIENT_ID: ${{ secrets.WORKSTREAM_DBT_CLIENT_ID }} - WORKSTREAM_DBT_CLIENT_SECRET: ${{ secrets.WORKSTREAM_DBT_CLIENT_SECRET }} \ No newline at end of file + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/streamline/core/realtime/streamline__tx_counts_realtime.sql 2+models/streamline/core/realtime/streamline__blocks_realtime.sql \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_transactions_realtime.yml b/.github/workflows/dbt_run_streamline_transactions_realtime.yml index 0a3a53c..dbdda0e 100644 --- a/.github/workflows/dbt_run_streamline_transactions_realtime.yml +++ b/.github/workflows/dbt_run_streamline_transactions_realtime.yml @@ -38,12 +38,8 @@ jobs: - name: install dependencies run: | - pip install -r requirements.txt workstream-dbt-core + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/streamline/core/realtime/streamline__transactions_realtime.sql || true - workstream report --exit-nonzero - env: - WORKSTREAM_DBT_CLIENT_ID: ${{ secrets.WORKSTREAM_DBT_CLIENT_ID }} - WORKSTREAM_DBT_CLIENT_SECRET: ${{ secrets.WORKSTREAM_DBT_CLIENT_SECRET }} \ No newline at end of file + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/streamline/core/realtime/streamline__transactions_realtime.sql \ No newline at end of file diff --git a/.github/workflows/dbt_test_monthly.yml b/.github/workflows/dbt_test_monthly.yml index 84357cc..8ec4edf 100644 --- a/.github/workflows/dbt_test_monthly.yml +++ b/.github/workflows/dbt_test_monthly.yml @@ -38,12 +38,8 @@ jobs: - name: install dependencies run: | - pip install -r requirements.txt workstream-dbt-core + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | - dbt test -m models/bronze/core "sei_models,tag:full_test" || true - workstream report --exit-nonzero - env: - WORKSTREAM_DBT_CLIENT_ID: ${{ secrets.WORKSTREAM_DBT_CLIENT_ID }} - WORKSTREAM_DBT_CLIENT_SECRET: ${{ secrets.WORKSTREAM_DBT_CLIENT_SECRET }} \ No newline at end of file + dbt test -m models/bronze/core "sei_models,tag:full_test" \ No newline at end of file diff --git a/.github/workflows/dbt_test_recent.yml b/.github/workflows/dbt_test_recent.yml index bfd833d..55c3895 100644 --- a/.github/workflows/dbt_test_recent.yml +++ b/.github/workflows/dbt_test_recent.yml @@ -39,12 +39,8 @@ jobs: - name: install dependencies run: | - pip install -r requirements.txt workstream-dbt-core + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | - dbt test -m models/bronze/core "sei_models,tag:recent_test" || true - workstream report --exit-nonzero - env: - WORKSTREAM_DBT_CLIENT_ID: ${{ secrets.WORKSTREAM_DBT_CLIENT_ID }} - WORKSTREAM_DBT_CLIENT_SECRET: ${{ secrets.WORKSTREAM_DBT_CLIENT_SECRET }} \ No newline at end of file + dbt test -m models/bronze/core "sei_models,tag:recent_test" \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 04691ad..56e428e 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -70,6 +70,19 @@ vars: - AWS_LAMBDA_AXELAR_API - DBT_CLOUD_AXELAR - INTERNAL_DEV + prod-2xl: + API_INTEGRATION: aws_sei_api + EXTERNAL_FUNCTION_URI: dbtc9lfp0k.execute-api.us-east-1.amazonaws.com/prod/ + ROLES: + - AWS_LAMBDA_AXELAR_API + - DBT_CLOUD_AXELAR + - INTERNAL_DEV + dev-2xl: + API_INTEGRATION: aws_sei_api_dev + EXTERNAL_FUNCTION_URI: ibj933oi6f.execute-api.us-east-1.amazonaws.com/stg/ + ROLES: + - AWS_LAMBDA_SEI_API + - INTERNAL_DEV #### STREAMLINE 2.0 END #### diff --git a/models/silver/core/silver__transactions_final.sql b/models/silver/core/silver__transactions_final.sql index f672c6a..5607edc 100644 --- a/models/silver/core/silver__transactions_final.sql +++ b/models/silver/core/silver__transactions_final.sql @@ -18,8 +18,15 @@ WITH atts AS ( FROM {{ ref('silver__msg_attributes') }} WHERE - msg_type = 'tx' - AND attribute_key IN ('fee', 'acc_seq') + msg_type IN ( + 'tx', + 'signer' + ) + AND attribute_key IN ( + 'fee', + 'acc_seq', + 'sei_addr' + ) {% if is_incremental() %} AND _inserted_timestamp >= ( @@ -54,7 +61,7 @@ spender AS ( FROM atts WHERE - attribute_key = 'acc_seq' qualify(ROW_NUMBER() over(PARTITION BY tx_id + attribute_key IN ('acc_seq', 'sei_addr') qualify(ROW_NUMBER() over(PARTITION BY tx_id ORDER BY msg_index)) = 1 )