From dcd4428256049cab0963985c5f409305a896cdda Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Wed, 18 Oct 2023 13:58:49 -0600 Subject: [PATCH 01/46] upd docs --- models/descriptions/__overview__.md | 8 ++++---- models/descriptions/blocks_table_doc.md | 4 +++- models/descriptions/tx_table_doc.md | 4 +++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/models/descriptions/__overview__.md b/models/descriptions/__overview__.md index 5b52006..95818da 100644 --- a/models/descriptions/__overview__.md +++ b/models/descriptions/__overview__.md @@ -23,10 +23,10 @@ There is more information on how to use dbt docs in the last section of this doc **Fact Tables:** -- [fact_blocks](#!/model/model.aurora.core__fact_blocks) -- [fact_transactions](#!/model/model.aurora.core__fact_transactions) -- [fact_logs](#!/model/model.aurora.core__fact_logs) -- [fact_receipts](#!/model/model.aurora.core__fact_receipts) +- [fact_blocks](#!/model/model.aurora_models.core__fact_blocks) +- [fact_transactions](#!/model/model.aurora_models.core__fact_transactions) +- [fact_logs](#!/model/model.aurora_models.core__fact_logs) +- [fact_receipts](#!/model/model.aurora_models.core__fact_receipts) ## **⚠️ Aurora Data Notes and Known Issues** diff --git a/models/descriptions/blocks_table_doc.md b/models/descriptions/blocks_table_doc.md index d2388b7..7c96773 100644 --- a/models/descriptions/blocks_table_doc.md +++ b/models/descriptions/blocks_table_doc.md @@ -1,5 +1,7 @@ {% docs blocks_table_doc %} -This table contains block level data for the Aurora Blockchain. This table can be used to analyze trends at a block level, for example gas fees vs. total transactions over time. For more information, please see Aurora scan Resources. +This table contains block level data for the Aurora Blockchain. This table can be used to analyze trends at a block level, for example gas fees vs. total transactions over time. For more information, please see Aurora scan Resources. + +Please note, Aurora contains pre-history and only launched as a public blockchain with block 37,157,757. Early blocks contain incomplete data, such as 0x0 as the block timestamp and no transactions. {% enddocs %} diff --git a/models/descriptions/tx_table_doc.md b/models/descriptions/tx_table_doc.md index c973254..2bcf584 100644 --- a/models/descriptions/tx_table_doc.md +++ b/models/descriptions/tx_table_doc.md @@ -1,5 +1,7 @@ {% docs tx_table_doc %} -This table contains transaction level data for the Aurora. +This table contains transaction level data for the Aurora. + +Please note, Aurora contains pre-history and only launched as a public blockchain with block 37,157,757. Early blocks contain incomplete data, such as 0x0 as the block timestamp and no transactions. {% enddocs %} From c73ca3dcb7ed81955978f4cb33a4c7939a5e0a8d Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Wed, 18 Oct 2023 14:06:25 -0600 Subject: [PATCH 02/46] add resolution notice --- models/descriptions/__overview__.md | 53 +++++++++++++++-------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/models/descriptions/__overview__.md b/models/descriptions/__overview__.md index 95818da..a895481 100644 --- a/models/descriptions/__overview__.md +++ b/models/descriptions/__overview__.md @@ -28,31 +28,6 @@ There is more information on how to use dbt docs in the last section of this doc - [fact_logs](#!/model/model.aurora_models.core__fact_logs) - [fact_receipts](#!/model/model.aurora_models.core__fact_receipts) - -## **⚠️ Aurora Data Notes and Known Issues** -*Update: July 27, 2023:* -We have identified 5 primary issues with the data coming from the node, and a path forward to resolution after discussions with the Aurora Dev team: -1. Duplicated Transactions - - Aurora Node issue, fix in progress by Aurora devs. Resolution timeline: 1-2 weeks -1. Reverted Transactions: Transactions or events that have been previously canceled or reverted are still being received from the chain. - - Aurora Node issue, fix in progress by Aurora devs. Resolution timeline: 1-2 weeks -1. Inaccurate Data: We've detected some inaccuracies in the data, namely incorrect timestamps (10M first block set to 1970) and transactions appearing in the wrong blocks. - - This is not a data integrity issue. Aurora contains pre-history and only launched as a public blockchain with **block 37,157,757**. Early blocks contain incomplete data, such as `0x0` as the block timestamp. -1. Incomplete Data: We've noticed certain transaction data missing, particularly with regards to received transactions. Incomplete data includes some blocks get read with wrong txs count and txs info there, we believed that is from the out of sync status of the node which needs a full backfill again when the node is back sync - - This is likely due to our current node provider using an outdated version of the Aurora RPC package. Resolution: change node provider. -1. Block Confirmation Discrepancies: Transactions were confirmed on different blocks than those indicated in Explorer. - - This is likely due to our current node provider using an outdated version of the Aurora RPC package. Resolution: change node provider. - -Our plan of action is (likely) to move to a dedicated node provided by Aurora which will solve the 4 major issues with a single decision. This timeline is dependent on the patch by Aurora, and our timeline will be updated as we learn more. - - -*Update: July 20, 2023:* -In onboarding Aurora data, our team has encountered several issues with data returned from the node. These are primarily associated with transactions that are either reverted or cancelled. At present, the node returns these transactions across multiple blocks and in different positions within the block at each time. This is uncommon, as the position should be constant. We may see pending transactions within a block on other EVMs, but on re-request the transaction would be finalized. These seem to be persistent across multiple blocks, even in subsequent requests. - -At present, these transactions are included in our data. They will have `null` fields like status, fee, and others that are typically derived from receipts. These transactions do not have receipts, so we can identify them through their lack of receipt data. - -Flipside is working closely with Near and Aurora to determine how this data should best be presented. - ## **Data Model Overview** The Aurora models are built a few different ways, but the core fact tables are built using three layers of sql models: **bronze, silver, and gold (or core).** @@ -93,4 +68,32 @@ Note that you can also right-click on models to interactively filter and explore - [Github](https://github.com/FlipsideCrypto/aurora-models) - [What is dbt?](https://docs.getdbt.com/docs/introduction) +## Archived Notices +### **⚠️ Aurora Data Notes and Known Issues** +*Final Update* +We have switched to a node hosted by the Aurora team, and they pushed relevant fixes. The below issues are since resolved. + +*Update: July 27, 2023:* +We have identified 5 primary issues with the data coming from the node, and a path forward to resolution after discussions with the Aurora Dev team: +1. Duplicated Transactions + - Aurora Node issue, fix in progress by Aurora devs. Resolution timeline: 1-2 weeks +1. Reverted Transactions: Transactions or events that have been previously canceled or reverted are still being received from the chain. + - Aurora Node issue, fix in progress by Aurora devs. Resolution timeline: 1-2 weeks +1. Inaccurate Data: We've detected some inaccuracies in the data, namely incorrect timestamps (10M first block set to 1970) and transactions appearing in the wrong blocks. + - This is not a data integrity issue. Aurora contains pre-history and only launched as a public blockchain with **block 37,157,757**. Early blocks contain incomplete data, such as `0x0` as the block timestamp. +1. Incomplete Data: We've noticed certain transaction data missing, particularly with regards to received transactions. Incomplete data includes some blocks get read with wrong txs count and txs info there, we believed that is from the out of sync status of the node which needs a full backfill again when the node is back sync + - This is likely due to our current node provider using an outdated version of the Aurora RPC package. Resolution: change node provider. +1. Block Confirmation Discrepancies: Transactions were confirmed on different blocks than those indicated in Explorer. + - This is likely due to our current node provider using an outdated version of the Aurora RPC package. Resolution: change node provider. + +Our plan of action is (likely) to move to a dedicated node provided by Aurora which will solve the 4 major issues with a single decision. This timeline is dependent on the patch by Aurora, and our timeline will be updated as we learn more. + + +*Update: July 20, 2023:* +In onboarding Aurora data, our team has encountered several issues with data returned from the node. These are primarily associated with transactions that are either reverted or cancelled. At present, the node returns these transactions across multiple blocks and in different positions within the block at each time. This is uncommon, as the position should be constant. We may see pending transactions within a block on other EVMs, but on re-request the transaction would be finalized. These seem to be persistent across multiple blocks, even in subsequent requests. + +At present, these transactions are included in our data. They will have `null` fields like status, fee, and others that are typically derived from receipts. These transactions do not have receipts, so we can identify them through their lack of receipt data. + +Flipside is working closely with Near and Aurora to determine how this data should best be presented. + {% enddocs %} From 3e6ac54f75f631fe7fdec20bed671c151b991c3b Mon Sep 17 00:00:00 2001 From: WHYTEWYLL Date: Wed, 1 Nov 2023 17:53:12 -0300 Subject: [PATCH 03/46] 20mins update --- .github/workflows/dbt_run_scheduled.yml | 4 ++-- .github/workflows/dbt_run_streamline_history.yml | 4 ++-- .github/workflows/dbt_run_streamline_realtime.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/dbt_run_scheduled.yml b/.github/workflows/dbt_run_scheduled.yml index c603d73..d719325 100644 --- a/.github/workflows/dbt_run_scheduled.yml +++ b/.github/workflows/dbt_run_scheduled.yml @@ -4,8 +4,8 @@ run-name: dbt_run_scheduled on: workflow_dispatch: schedule: - # Runs "every hour at minute 10" (see https://crontab.guru) - - cron: "10 */1 * * *" + # Runs "every 20 minutes" + - cron: "*/20 * * * *" env: USE_VARS: "${{ vars.USE_VARS }}" diff --git a/.github/workflows/dbt_run_streamline_history.yml b/.github/workflows/dbt_run_streamline_history.yml index 377d97f..31d5390 100644 --- a/.github/workflows/dbt_run_streamline_history.yml +++ b/.github/workflows/dbt_run_streamline_history.yml @@ -4,8 +4,8 @@ run-name: dbt_run_streamline_history on: workflow_dispatch: schedule: - # Runs "every 4 hours" (see https://crontab.guru) - - cron: "0 */4 * * *" + # Runs "every 20 minutes" + - cron: "*/20 * * * *" env: USE_VARS: "${{ vars.USE_VARS }}" diff --git a/.github/workflows/dbt_run_streamline_realtime.yml b/.github/workflows/dbt_run_streamline_realtime.yml index ccacd97..59e7e20 100644 --- a/.github/workflows/dbt_run_streamline_realtime.yml +++ b/.github/workflows/dbt_run_streamline_realtime.yml @@ -4,8 +4,8 @@ run-name: dbt_run_streamline_realtime on: workflow_dispatch: schedule: - # Runs "every 1 hours" (see https://crontab.guru) - - cron: "0 */1 * * *" + # Runs "every 20 minutes" + - cron: "*/20 * * * *" env: USE_VARS: "${{ vars.USE_VARS }}" From ad381027cadf904b7cbe8063abf3090f8ba8f363 Mon Sep 17 00:00:00 2001 From: forgash_ Date: Thu, 2 Nov 2023 14:31:03 -0600 Subject: [PATCH 04/46] upd schedule --- .github/workflows/dbt_run_scheduled.yml | 4 ++-- .github/workflows/dbt_run_streamline_history.yml | 3 --- .github/workflows/dbt_run_streamline_realtime.yml | 4 ++-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dbt_run_scheduled.yml b/.github/workflows/dbt_run_scheduled.yml index d719325..49cb3ca 100644 --- a/.github/workflows/dbt_run_scheduled.yml +++ b/.github/workflows/dbt_run_scheduled.yml @@ -4,8 +4,8 @@ run-name: dbt_run_scheduled on: workflow_dispatch: schedule: - # Runs "every 20 minutes" - - cron: "*/20 * * * *" + # Runs "every 15 minutes" + - cron: "*/15 * * * *" env: USE_VARS: "${{ vars.USE_VARS }}" diff --git a/.github/workflows/dbt_run_streamline_history.yml b/.github/workflows/dbt_run_streamline_history.yml index 31d5390..07ac885 100644 --- a/.github/workflows/dbt_run_streamline_history.yml +++ b/.github/workflows/dbt_run_streamline_history.yml @@ -3,9 +3,6 @@ run-name: dbt_run_streamline_history on: workflow_dispatch: - schedule: - # Runs "every 20 minutes" - - cron: "*/20 * * * *" env: USE_VARS: "${{ vars.USE_VARS }}" diff --git a/.github/workflows/dbt_run_streamline_realtime.yml b/.github/workflows/dbt_run_streamline_realtime.yml index 59e7e20..d4af813 100644 --- a/.github/workflows/dbt_run_streamline_realtime.yml +++ b/.github/workflows/dbt_run_streamline_realtime.yml @@ -4,8 +4,8 @@ run-name: dbt_run_streamline_realtime on: workflow_dispatch: schedule: - # Runs "every 20 minutes" - - cron: "*/20 * * * *" + # Runs "every 10 minutes" + - cron: "*/10 * * * *" env: USE_VARS: "${{ vars.USE_VARS }}" From 4433dbe8dd3eb73118fbca6a25b6297a94918c0e Mon Sep 17 00:00:00 2001 From: forgash_ Date: Fri, 3 Nov 2023 12:30:46 -0600 Subject: [PATCH 05/46] newline --- .github/workflows/dbt_run_scheduled.yml | 4 ++-- .github/workflows/dbt_run_streamline_realtime.yml | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dbt_run_scheduled.yml b/.github/workflows/dbt_run_scheduled.yml index 49cb3ca..d793284 100644 --- a/.github/workflows/dbt_run_scheduled.yml +++ b/.github/workflows/dbt_run_scheduled.yml @@ -4,8 +4,8 @@ run-name: dbt_run_scheduled on: workflow_dispatch: schedule: - # Runs "every 15 minutes" - - cron: "*/15 * * * *" + # Runs "every 10 minutes, 5 mins after the realtime job + - cron: "5,15,25,35,45,55 * * * *" env: USE_VARS: "${{ vars.USE_VARS }}" diff --git a/.github/workflows/dbt_run_streamline_realtime.yml b/.github/workflows/dbt_run_streamline_realtime.yml index d4af813..9916104 100644 --- a/.github/workflows/dbt_run_streamline_realtime.yml +++ b/.github/workflows/dbt_run_streamline_realtime.yml @@ -4,9 +4,10 @@ run-name: dbt_run_streamline_realtime on: workflow_dispatch: schedule: - # Runs "every 10 minutes" - - cron: "*/10 * * * *" + # Runs every 10 minutes + - cron: "0,10,20,30,40,50 * * * *" + env: USE_VARS: "${{ vars.USE_VARS }}" DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" From e2b72cefe3bb03d7b8894b3cede7936578b7bb7f Mon Sep 17 00:00:00 2001 From: forgash_ Date: Fri, 3 Nov 2023 12:31:17 -0600 Subject: [PATCH 06/46] del extra line --- .github/workflows/dbt_run_streamline_realtime.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/dbt_run_streamline_realtime.yml b/.github/workflows/dbt_run_streamline_realtime.yml index 9916104..743761e 100644 --- a/.github/workflows/dbt_run_streamline_realtime.yml +++ b/.github/workflows/dbt_run_streamline_realtime.yml @@ -7,7 +7,6 @@ on: # Runs every 10 minutes - cron: "0,10,20,30,40,50 * * * *" - env: USE_VARS: "${{ vars.USE_VARS }}" DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" From 1ec10f03f2a85a2b9243ce9956dd08c99b9216a0 Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Fri, 10 Nov 2023 11:43:31 -0500 Subject: [PATCH 07/46] gha tasks --- .github/workflows/dbt_alter_gha_task.yml | 46 +++++++++++++++++++ .github/workflows/dbt_run_scheduled.yml | 6 +-- .../workflows/dbt_run_streamline_realtime.yml | 6 +-- .github/workflows/dbt_test_tasks.yml | 27 +++++++++++ data/github_actions__workflows.csv | 4 ++ dbt_project.yml | 1 + .../github_actions__current_task_status.sql | 6 +++ .../github_actions__current_task_status.yml | 16 +++++++ .../github_actions__task_history.sql | 5 ++ .../github_actions__task_performance.sql | 5 ++ .../github_actions__task_schedule.sql | 5 ++ .../github_actions/github_actions__tasks.sql | 5 ++ models/sources.yml | 6 +++ packages.yml | 2 +- requirements.txt | 1 + 15 files changed, 134 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/dbt_alter_gha_task.yml create mode 100644 .github/workflows/dbt_test_tasks.yml create mode 100644 data/github_actions__workflows.csv create mode 100644 models/github_actions/github_actions__current_task_status.sql create mode 100644 models/github_actions/github_actions__current_task_status.yml create mode 100644 models/github_actions/github_actions__task_history.sql create mode 100644 models/github_actions/github_actions__task_performance.sql create mode 100644 models/github_actions/github_actions__task_schedule.sql create mode 100644 models/github_actions/github_actions__tasks.sql create mode 100644 requirements.txt diff --git a/.github/workflows/dbt_alter_gha_task.yml b/.github/workflows/dbt_alter_gha_task.yml new file mode 100644 index 0000000..da996d8 --- /dev/null +++ b/.github/workflows/dbt_alter_gha_task.yml @@ -0,0 +1,46 @@ +name: dbt_alter_gha_task +run-name: dbt_alter_gha_task + +on: + workflow_dispatch: + branches: + - "main" + inputs: + workflow_name: + type: string + description: Name of the workflow to perform the action on, no .yml extension + required: true + task_action: + type: choice + description: Action to perform + required: true + options: + - SUSPEND + - RESUME + default: SUSPEND + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@main + with: + workflow_name: | + ${{ inputs.workflow_name }} + task_action: | + ${{ inputs.task_action }} + environment: workflow_prod + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled.yml b/.github/workflows/dbt_run_scheduled.yml index d793284..6af6314 100644 --- a/.github/workflows/dbt_run_scheduled.yml +++ b/.github/workflows/dbt_run_scheduled.yml @@ -3,9 +3,9 @@ run-name: dbt_run_scheduled on: workflow_dispatch: - schedule: - # Runs "every 10 minutes, 5 mins after the realtime job - - cron: "5,15,25,35,45,55 * * * *" + # schedule: + # # Runs "every 10 minutes, 5 mins after the realtime job + # - cron: "5,15,25,35,45,55 * * * *" env: USE_VARS: "${{ vars.USE_VARS }}" diff --git a/.github/workflows/dbt_run_streamline_realtime.yml b/.github/workflows/dbt_run_streamline_realtime.yml index 743761e..f0553b7 100644 --- a/.github/workflows/dbt_run_streamline_realtime.yml +++ b/.github/workflows/dbt_run_streamline_realtime.yml @@ -3,9 +3,9 @@ run-name: dbt_run_streamline_realtime on: workflow_dispatch: - schedule: - # Runs every 10 minutes - - cron: "0,10,20,30,40,50 * * * *" + # schedule: + # # Runs every 10 minutes + # - cron: "0,10,20,30,40,50 * * * *" env: USE_VARS: "${{ vars.USE_VARS }}" diff --git a/.github/workflows/dbt_test_tasks.yml b/.github/workflows/dbt_test_tasks.yml new file mode 100644 index 0000000..976440b --- /dev/null +++ b/.github/workflows/dbt_test_tasks.yml @@ -0,0 +1,27 @@ +name: dbt_test_tasks +run-name: dbt_test_tasks + +on: + workflow_dispatch: + branches: + - "main" + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@main + secrets: inherit \ No newline at end of file diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv new file mode 100644 index 0000000..a7dc842 --- /dev/null +++ b/data/github_actions__workflows.csv @@ -0,0 +1,4 @@ +workflow_name,workflow_schedule +dbt_run_scheduled,"5,15,25,35,45,55 * * * *" +dbt_run_streamline_realtime,"0,10,20,30,40,50 * * * *" +dbt_test_tasks,"0,30 * * * *" diff --git a/dbt_project.yml b/dbt_project.yml index 2e73399..109f150 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -48,6 +48,7 @@ vars: UPDATE_SNOWFLAKE_TAGS: True WAIT: 0 OBSERV_FULL_TEST: False + START_GHA_TASKS: False tests: +store_failures: true # all tests diff --git a/models/github_actions/github_actions__current_task_status.sql b/models/github_actions/github_actions__current_task_status.sql new file mode 100644 index 0000000..577a226 --- /dev/null +++ b/models/github_actions/github_actions__current_task_status.sql @@ -0,0 +1,6 @@ +{{ config( + materialized = 'view', + tags = ['gha_tasks'] +) }} + +{{ fsc_utils.gha_task_current_status_view() }} \ No newline at end of file diff --git a/models/github_actions/github_actions__current_task_status.yml b/models/github_actions/github_actions__current_task_status.yml new file mode 100644 index 0000000..7923144 --- /dev/null +++ b/models/github_actions/github_actions__current_task_status.yml @@ -0,0 +1,16 @@ +version: 2 +models: + - name: github_actions__current_task_status + columns: + - name: PIPELINE_ACTIVE + tests: + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: + - TRUE + - name: SUCCESSES + tests: + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: + - 2 + config: + severity: warn \ No newline at end of file diff --git a/models/github_actions/github_actions__task_history.sql b/models/github_actions/github_actions__task_history.sql new file mode 100644 index 0000000..9c35ce7 --- /dev/null +++ b/models/github_actions/github_actions__task_history.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_task_history_view() }} \ No newline at end of file diff --git a/models/github_actions/github_actions__task_performance.sql b/models/github_actions/github_actions__task_performance.sql new file mode 100644 index 0000000..117ded5 --- /dev/null +++ b/models/github_actions/github_actions__task_performance.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_task_performance_view() }} \ No newline at end of file diff --git a/models/github_actions/github_actions__task_schedule.sql b/models/github_actions/github_actions__task_schedule.sql new file mode 100644 index 0000000..ff95a44 --- /dev/null +++ b/models/github_actions/github_actions__task_schedule.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_task_schedule_view() }} \ No newline at end of file diff --git a/models/github_actions/github_actions__tasks.sql b/models/github_actions/github_actions__tasks.sql new file mode 100644 index 0000000..feab82a --- /dev/null +++ b/models/github_actions/github_actions__tasks.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_tasks_view() }} \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index 0959d22..83ab44d 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -24,3 +24,9 @@ sources: schema: silver tables: - name: number_sequence + + - name: github_actions + database: aurora + schema: github_actions + tables: + - name: workflows diff --git a/packages.yml b/packages.yml index ff78cc8..febe9d6 100644 --- a/packages.yml +++ b/packages.yml @@ -2,6 +2,6 @@ packages: - package: calogica/dbt_expectations version: [">=0.4.0", "<0.9.0"] - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: v1.3.0 + revision: v1.11.0 - package: get-select/dbt_snowflake_query_tags version: [">=2.0.0", "<3.0.0"] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ec44b06 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +dbt-snowflake>=1.4,<1.5 \ No newline at end of file From a75eafad01910e12bf308f8907ce0b1cc1786d09 Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Fri, 10 Nov 2023 17:19:04 -0500 Subject: [PATCH 08/46] schedules --- data/github_actions__workflows.csv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv index a7dc842..67220ce 100644 --- a/data/github_actions__workflows.csv +++ b/data/github_actions__workflows.csv @@ -1,4 +1,4 @@ workflow_name,workflow_schedule -dbt_run_scheduled,"5,15,25,35,45,55 * * * *" -dbt_run_streamline_realtime,"0,10,20,30,40,50 * * * *" -dbt_test_tasks,"0,30 * * * *" +dbt_run_streamline_realtime,"12,42 * * * *" +dbt_run_scheduled,"22,52 * * * *" +dbt_test_tasks,"25,55 * * * *" From 15430a39e6cc7dd17254ea2da61066d8acaa3712 Mon Sep 17 00:00:00 2001 From: forgash_ Date: Mon, 13 Nov 2023 14:17:55 -0700 Subject: [PATCH 09/46] atlas maas --- models/silver/atlas/silver__atlas_maa.sql | 73 +++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 models/silver/atlas/silver__atlas_maa.sql diff --git a/models/silver/atlas/silver__atlas_maa.sql b/models/silver/atlas/silver__atlas_maa.sql new file mode 100644 index 0000000..4bab596 --- /dev/null +++ b/models/silver/atlas/silver__atlas_maa.sql @@ -0,0 +1,73 @@ +{{ config( + materialized = 'incremental', + unique_key = 'day' +) }} + +WITH dates AS ( + + SELECT + date_day AS DAY + FROM + {{ source( + 'crosschain', + 'dim_dates' + ) }} + +{% if is_incremental() %} +WHERE + date_day > ( + SELECT + MAX(DAY) + FROM + {{ this }} + ) + AND date_day < SYSDATE() :: DATE +{% else %} +WHERE + date_day BETWEEN '2020-07-22' + AND SYSDATE() :: DATE +{% endif %} +), +txns AS ( + SELECT + block_timestamp :: DATE AS active_day, + from_address + FROM + {{ ref('silver__transactions') }} + +{% if is_incremental() %} +WHERE + block_timestamp :: DATE >= ( + SELECT + MAX(DAY) + FROM + {{ this }} + ) - INTERVAL '30 days' +{% endif %} +), +FINAL AS ( + SELECT + DAY, + COUNT( + DISTINCT from_address + ) AS maa + FROM + dates d + LEFT JOIN txns t + ON t.active_day < d.day + AND t.active_day >= d.day - INTERVAL '30 day' + WHERE + DAY != CURRENT_DATE() + GROUP BY + 1 + ORDER BY + 1 DESC +) +SELECT + DAY, + maa, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS invocation_id +FROM + FINAL From 29a2a32054da4a01e61e0c35b59eedc19e88dc63 Mon Sep 17 00:00:00 2001 From: forgash_ Date: Mon, 13 Nov 2023 15:25:49 -0700 Subject: [PATCH 10/46] ymls and gold view --- models/descriptions/active_day.md | 5 +++ models/descriptions/id.md | 5 +++ models/descriptions/inserted_timestamp.md | 5 +++ models/descriptions/invocation_id.md | 5 +++ models/descriptions/maa.md | 5 +++ models/descriptions/modified_timestamp.md | 5 +++ models/gold/atlas/atlas__fact_maas.sql | 13 +++++++ models/gold/atlas/atlas__fact_maas.yml | 34 ++++++++++++++++ models/gold/{ => core}/core__fact_blocks.sql | 0 models/gold/{ => core}/core__fact_blocks.yml | 0 models/gold/{ => core}/core__fact_logs.sql | 0 models/gold/{ => core}/core__fact_logs.yml | 0 .../gold/{ => core}/core__fact_receipts.sql | 0 .../gold/{ => core}/core__fact_receipts.yml | 0 .../{ => core}/core__fact_transactions.sql | 0 .../{ => core}/core__fact_transactions.yml | 0 models/silver/atlas/silver__atlas_maa.sql | 8 +++- models/silver/atlas/silver__atlas_maa.yml | 39 +++++++++++++++++++ 18 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 models/descriptions/active_day.md create mode 100644 models/descriptions/id.md create mode 100644 models/descriptions/inserted_timestamp.md create mode 100644 models/descriptions/invocation_id.md create mode 100644 models/descriptions/maa.md create mode 100644 models/descriptions/modified_timestamp.md create mode 100644 models/gold/atlas/atlas__fact_maas.sql create mode 100644 models/gold/atlas/atlas__fact_maas.yml rename models/gold/{ => core}/core__fact_blocks.sql (100%) rename models/gold/{ => core}/core__fact_blocks.yml (100%) rename models/gold/{ => core}/core__fact_logs.sql (100%) rename models/gold/{ => core}/core__fact_logs.yml (100%) rename models/gold/{ => core}/core__fact_receipts.sql (100%) rename models/gold/{ => core}/core__fact_receipts.yml (100%) rename models/gold/{ => core}/core__fact_transactions.sql (100%) rename models/gold/{ => core}/core__fact_transactions.yml (100%) create mode 100644 models/silver/atlas/silver__atlas_maa.yml diff --git a/models/descriptions/active_day.md b/models/descriptions/active_day.md new file mode 100644 index 0000000..959ab0a --- /dev/null +++ b/models/descriptions/active_day.md @@ -0,0 +1,5 @@ +{% docs active_day %} + +Date of activity. + +{% enddocs %} diff --git a/models/descriptions/id.md b/models/descriptions/id.md new file mode 100644 index 0000000..38c401a --- /dev/null +++ b/models/descriptions/id.md @@ -0,0 +1,5 @@ +{% docs id %} + +A unique identifier for the record. + +{% enddocs %} diff --git a/models/descriptions/inserted_timestamp.md b/models/descriptions/inserted_timestamp.md new file mode 100644 index 0000000..94447a4 --- /dev/null +++ b/models/descriptions/inserted_timestamp.md @@ -0,0 +1,5 @@ +{% docs inserted_timestamp %} + +The timestamp at which the record was initially created and inserted into this table. + +{% enddocs %} diff --git a/models/descriptions/invocation_id.md b/models/descriptions/invocation_id.md new file mode 100644 index 0000000..64a03b7 --- /dev/null +++ b/models/descriptions/invocation_id.md @@ -0,0 +1,5 @@ +{% docs invocation_id %} + +A job ID to identify the run that last modified a record. + +{% enddocs %} diff --git a/models/descriptions/maa.md b/models/descriptions/maa.md new file mode 100644 index 0000000..965c402 --- /dev/null +++ b/models/descriptions/maa.md @@ -0,0 +1,5 @@ +{% docs maa %} + +Monthly Active Accounts (wallets), as determined by transaction signers. + +{% enddocs %} diff --git a/models/descriptions/modified_timestamp.md b/models/descriptions/modified_timestamp.md new file mode 100644 index 0000000..abda32e --- /dev/null +++ b/models/descriptions/modified_timestamp.md @@ -0,0 +1,5 @@ +{% docs modified_timestamp %} + +The timestamp at which this record was last modified by an internal process. + +{% enddocs %} diff --git a/models/gold/atlas/atlas__fact_maas.sql b/models/gold/atlas/atlas__fact_maas.sql new file mode 100644 index 0000000..cc21a81 --- /dev/null +++ b/models/gold/atlas/atlas__fact_maas.sql @@ -0,0 +1,13 @@ +{{ config( + materialized = 'view', + tags = ['atlas'] +) }} + +SELECT + atlas_maa_id AS fact_maas_id, + day, + maa, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__atlas_maa') }} diff --git a/models/gold/atlas/atlas__fact_maas.yml b/models/gold/atlas/atlas__fact_maas.yml new file mode 100644 index 0000000..abdf8e0 --- /dev/null +++ b/models/gold/atlas/atlas__fact_maas.yml @@ -0,0 +1,34 @@ +version: 2 + +models: + - name: atlas__fact_maas + description: |- + Monthly Active Accounts (wallets) on Aurora, calculated over a rolling 30 day window. An active account, here, is defined as the signing of at least one transaction. + + columns: + - name: fact_maas_id + description: "{{ doc('id') }}" + tests: + - not_null + - unique + + - name: day + description: "{{ doc('active_day') }}" + tests: + - not_null + - unique + + - name: maa + description: "{{ doc('maa')}}" + tests: + - not_null + + - name: inserted_timestamp + description: "{{ doc('inserted_timestamp') }}" + tests: + - not_null + + - name: modified_timestamp + description: "{{ doc('modified_timestamp') }}" + tests: + - not_null diff --git a/models/gold/core__fact_blocks.sql b/models/gold/core/core__fact_blocks.sql similarity index 100% rename from models/gold/core__fact_blocks.sql rename to models/gold/core/core__fact_blocks.sql diff --git a/models/gold/core__fact_blocks.yml b/models/gold/core/core__fact_blocks.yml similarity index 100% rename from models/gold/core__fact_blocks.yml rename to models/gold/core/core__fact_blocks.yml diff --git a/models/gold/core__fact_logs.sql b/models/gold/core/core__fact_logs.sql similarity index 100% rename from models/gold/core__fact_logs.sql rename to models/gold/core/core__fact_logs.sql diff --git a/models/gold/core__fact_logs.yml b/models/gold/core/core__fact_logs.yml similarity index 100% rename from models/gold/core__fact_logs.yml rename to models/gold/core/core__fact_logs.yml diff --git a/models/gold/core__fact_receipts.sql b/models/gold/core/core__fact_receipts.sql similarity index 100% rename from models/gold/core__fact_receipts.sql rename to models/gold/core/core__fact_receipts.sql diff --git a/models/gold/core__fact_receipts.yml b/models/gold/core/core__fact_receipts.yml similarity index 100% rename from models/gold/core__fact_receipts.yml rename to models/gold/core/core__fact_receipts.yml diff --git a/models/gold/core__fact_transactions.sql b/models/gold/core/core__fact_transactions.sql similarity index 100% rename from models/gold/core__fact_transactions.sql rename to models/gold/core/core__fact_transactions.sql diff --git a/models/gold/core__fact_transactions.yml b/models/gold/core/core__fact_transactions.yml similarity index 100% rename from models/gold/core__fact_transactions.yml rename to models/gold/core/core__fact_transactions.yml diff --git a/models/silver/atlas/silver__atlas_maa.sql b/models/silver/atlas/silver__atlas_maa.sql index 4bab596..36d1ae6 100644 --- a/models/silver/atlas/silver__atlas_maa.sql +++ b/models/silver/atlas/silver__atlas_maa.sql @@ -1,6 +1,9 @@ {{ config( materialized = 'incremental', - unique_key = 'day' + incremental_stratege = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + unique_key = 'day', + tags = ['atlas'] ) }} WITH dates AS ( @@ -64,6 +67,9 @@ FINAL AS ( 1 DESC ) SELECT + {{ dbt_utils.generate_surrogate_key( + ['day'] + ) }} AS atlas_maa_id, DAY, maa, SYSDATE() AS inserted_timestamp, diff --git a/models/silver/atlas/silver__atlas_maa.yml b/models/silver/atlas/silver__atlas_maa.yml new file mode 100644 index 0000000..644dc2e --- /dev/null +++ b/models/silver/atlas/silver__atlas_maa.yml @@ -0,0 +1,39 @@ +version: 2 + +models: + - name: silver__atlas_maa + description: |- + Monthly Active Accounts (wallets) on Aurora, calculated over a rolling 30 day window. An active account, here, is defined as the signing of at least one transaction. + + columns: + - name: atlas_maa_id + description: "{{ doc('id') }}" + tests: + - not_null + - unique + + - name: day + description: "{{ doc('active_day') }}" + tests: + - not_null + - unique + + - name: maa + description: "{{ doc('maa')}}" + tests: + - not_null + + - name: inserted_timestamp + description: "{{ doc('inserted_timestamp') }}" + tests: + - not_null + + - name: modified_timestamp + description: "{{ doc('modified_timestamp') }}" + tests: + - not_null + + - name: invocation_id + description: "{{ doc('invocation_id') }}" + tests: + - not_null From 195d687b536bac7a5d040b4d83b5cb9e61a199f2 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Tue, 14 Nov 2023 17:44:01 -0700 Subject: [PATCH 11/46] atlas workflow --- .github/workflows/dbt_run_atlas.yml | 34 +++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/dbt_run_atlas.yml diff --git a/.github/workflows/dbt_run_atlas.yml b/.github/workflows/dbt_run_atlas.yml new file mode 100644 index 0000000..d4949d6 --- /dev/null +++ b/.github/workflows/dbt_run_atlas.yml @@ -0,0 +1,34 @@ +name: dbt_run_atlas +run-name: dbt_run_atlas + +on: + workflow_dispatch: + schedule: + # Runs once per day at 0 UTC + - cron: "0 0 * * *" + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + with: + dbt_command: > + dbt run -s tag:atlas + environment: workflow_prod + warehouse: ${{ vars.WAREHOUSE }} + secrets: inherit From a81815659f610e542270646d8177a5edcf82eae1 Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Fri, 17 Nov 2023 13:50:17 -0500 Subject: [PATCH 12/46] schedule updates and split streamline job --- ...treamline_realtime_blocks_transactions.yml | 34 +++++++++++++++++++ ...t_run_streamline_realtime_tx_receipts.yml} | 6 ++-- data/github_actions__workflows.csv | 7 ++-- 3 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/dbt_run_streamline_realtime_blocks_transactions.yml rename .github/workflows/{dbt_run_streamline_realtime.yml => dbt_run_streamline_realtime_tx_receipts.yml} (83%) diff --git a/.github/workflows/dbt_run_streamline_realtime_blocks_transactions.yml b/.github/workflows/dbt_run_streamline_realtime_blocks_transactions.yml new file mode 100644 index 0000000..a4689ee --- /dev/null +++ b/.github/workflows/dbt_run_streamline_realtime_blocks_transactions.yml @@ -0,0 +1,34 @@ +name: dbt_run_streamline_realtime_blocks_transactions +run-name: dbt_run_streamline_realtime_blocks_transactions + +on: + workflow_dispatch: + # schedule: + # # Runs every 10 minutes + # - cron: "0,10,20,30,40,50 * * * *" + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + with: + dbt_command: > + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/realtime/streamline__blocks_realtime.sql 1+models/silver/streamline/realtime/streamline__transactions_realtime.sql + environment: workflow_prod + warehouse: ${{ vars.WAREHOUSE }} + secrets: inherit diff --git a/.github/workflows/dbt_run_streamline_realtime.yml b/.github/workflows/dbt_run_streamline_realtime_tx_receipts.yml similarity index 83% rename from .github/workflows/dbt_run_streamline_realtime.yml rename to .github/workflows/dbt_run_streamline_realtime_tx_receipts.yml index f0553b7..44b8a26 100644 --- a/.github/workflows/dbt_run_streamline_realtime.yml +++ b/.github/workflows/dbt_run_streamline_realtime_tx_receipts.yml @@ -1,5 +1,5 @@ -name: dbt_run_streamline_realtime -run-name: dbt_run_streamline_realtime +name: dbt_run_streamline_realtime_tx_receipts +run-name: dbt_run_streamline_realtime_tx_receipts on: workflow_dispatch: @@ -28,7 +28,7 @@ jobs: uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main with: dbt_command: > - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/realtime/ + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/realtime/streamline__tx_receipts_realtime.sql environment: workflow_prod warehouse: ${{ vars.WAREHOUSE }} secrets: inherit diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv index 67220ce..f77a3a3 100644 --- a/data/github_actions__workflows.csv +++ b/data/github_actions__workflows.csv @@ -1,4 +1,5 @@ workflow_name,workflow_schedule -dbt_run_streamline_realtime,"12,42 * * * *" -dbt_run_scheduled,"22,52 * * * *" -dbt_test_tasks,"25,55 * * * *" +dbt_run_streamline_realtime_blocks_transactions,"12,42 * * * *" +dbt_run_streamline_realtime_tx_receipts,"17,47 * * * *" +dbt_run_scheduled,"25,55 * * * *" +dbt_test_tasks,"28,58 * * * *" From ec247316468d88ba2f5f077fc30854a93f8ba119 Mon Sep 17 00:00:00 2001 From: xiuy001 Date: Mon, 27 Nov 2023 15:21:15 -0500 Subject: [PATCH 13/46] updated --- .../realtime/streamline__blocks_realtime.sql | 56 +++++++++++-------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/models/silver/streamline/realtime/streamline__blocks_realtime.sql b/models/silver/streamline/realtime/streamline__blocks_realtime.sql index 555807b..cf427d8 100644 --- a/models/silver/streamline/realtime/streamline__blocks_realtime.sql +++ b/models/silver/streamline/realtime/streamline__blocks_realtime.sql @@ -6,22 +6,24 @@ ) ) }} -WITH last_3_days AS ({% if var('STREAMLINE_RUN_HISTORY') %} +WITH last_3_days AS ( - SELECT - 0 AS block_number - {% else %} SELECT MAX(block_number) - 50000 AS block_number --aprox 3 days FROM {{ ref("streamline__blocks") }} - {% endif %}), - tbl AS ( - SELECT - block_number, - block_number_hex - FROM - {{ ref("streamline__blocks") }} +), +tbl AS ( + SELECT + block_number, + block_number_hex + FROM + {{ ref("streamline__blocks") }} + + {% if var('STREAMLINE_RUN_HISTORY') %} + WHERE + block_number IS NOT NULL + {% else %} WHERE ( block_number >= ( @@ -32,16 +34,22 @@ WITH last_3_days AS ({% if var('STREAMLINE_RUN_HISTORY') %} ) ) AND block_number IS NOT NULL - EXCEPT - SELECT - block_number, - REPLACE( - concat_ws('', '0x', to_char(block_number, 'XXXXXXXX')), - ' ', - '' - ) AS block_number_hex - FROM - {{ ref("streamline__complete_blocks") }} + {% endif %} + EXCEPT + SELECT + block_number, + REPLACE( + concat_ws('', '0x', to_char(block_number, 'XXXXXXXX')), + ' ', + '' + ) AS block_number_hex + FROM + {{ ref("streamline__complete_blocks") }} + + {% if var('STREAMLINE_RUN_HISTORY') %} + WHERE + block_number IS NOT NULL + {% else %} WHERE ( block_number >= ( @@ -52,7 +60,8 @@ WITH last_3_days AS ({% if var('STREAMLINE_RUN_HISTORY') %} ) ) AND block_number IS NOT NULL - ) + {% endif %} +) SELECT block_number, 'eth_getBlockByNumber' AS method, @@ -61,4 +70,5 @@ SELECT '_-_', 'false' ) AS params -FROM tbl \ No newline at end of file +FROM + tbl From f53fed101d62e49ffece30547c0f7df14a904b58 Mon Sep 17 00:00:00 2001 From: xiuy001 Date: Mon, 27 Nov 2023 15:39:56 -0500 Subject: [PATCH 14/46] updated the range of the blocks --- .../realtime/streamline__blocks_realtime.sql | 56 ++++++++----------- .../streamline__transactions_realtime.sql | 2 +- .../streamline__tx_receipts_realtime.sql | 2 +- 3 files changed, 25 insertions(+), 35 deletions(-) diff --git a/models/silver/streamline/realtime/streamline__blocks_realtime.sql b/models/silver/streamline/realtime/streamline__blocks_realtime.sql index cf427d8..349a917 100644 --- a/models/silver/streamline/realtime/streamline__blocks_realtime.sql +++ b/models/silver/streamline/realtime/streamline__blocks_realtime.sql @@ -6,24 +6,22 @@ ) ) }} -WITH last_3_days AS ( +WITH last_3_days AS ({% if var('STREAMLINE_RUN_HISTORY') %} SELECT - MAX(block_number) - 50000 AS block_number --aprox 3 days - FROM - {{ ref("streamline__blocks") }} -), -tbl AS ( + 0 AS block_number + {% else %} SELECT - block_number, - block_number_hex + MAX(block_number) - 500000 AS block_number --aprox 3 days FROM {{ ref("streamline__blocks") }} - - {% if var('STREAMLINE_RUN_HISTORY') %} - WHERE - block_number IS NOT NULL - {% else %} + {% endif %}), + tbl AS ( + SELECT + block_number, + block_number_hex + FROM + {{ ref("streamline__blocks") }} WHERE ( block_number >= ( @@ -34,22 +32,16 @@ tbl AS ( ) ) AND block_number IS NOT NULL - {% endif %} - EXCEPT - SELECT - block_number, - REPLACE( - concat_ws('', '0x', to_char(block_number, 'XXXXXXXX')), - ' ', - '' - ) AS block_number_hex - FROM - {{ ref("streamline__complete_blocks") }} - - {% if var('STREAMLINE_RUN_HISTORY') %} - WHERE - block_number IS NOT NULL - {% else %} + EXCEPT + SELECT + block_number, + REPLACE( + concat_ws('', '0x', to_char(block_number, 'XXXXXXXX')), + ' ', + '' + ) AS block_number_hex + FROM + {{ ref("streamline__complete_blocks") }} WHERE ( block_number >= ( @@ -60,8 +52,7 @@ tbl AS ( ) ) AND block_number IS NOT NULL - {% endif %} -) + ) SELECT block_number, 'eth_getBlockByNumber' AS method, @@ -70,5 +61,4 @@ SELECT '_-_', 'false' ) AS params -FROM - tbl +FROM tbl \ No newline at end of file diff --git a/models/silver/streamline/realtime/streamline__transactions_realtime.sql b/models/silver/streamline/realtime/streamline__transactions_realtime.sql index 0ac17bb..7a59483 100644 --- a/models/silver/streamline/realtime/streamline__transactions_realtime.sql +++ b/models/silver/streamline/realtime/streamline__transactions_realtime.sql @@ -12,7 +12,7 @@ WITH last_3_days AS ({% if var('STREAMLINE_RUN_HISTORY') %} 0 AS block_number {% else %} SELECT - MAX(block_number) - 50000 AS block_number --aprox 3 days + MAX(block_number) - 500000 AS block_number --aprox 3 days FROM {{ ref("streamline__blocks") }} {% endif %}), diff --git a/models/silver/streamline/realtime/streamline__tx_receipts_realtime.sql b/models/silver/streamline/realtime/streamline__tx_receipts_realtime.sql index ae28001..be26800 100644 --- a/models/silver/streamline/realtime/streamline__tx_receipts_realtime.sql +++ b/models/silver/streamline/realtime/streamline__tx_receipts_realtime.sql @@ -12,7 +12,7 @@ WITH last_3_days AS ({% if var('STREAMLINE_RUN_HISTORY') %} 0 AS block_number {% else %} SELECT - MAX(block_number) - 50000 AS block_number -- aprox 3 days + MAX(block_number) - 500000 AS block_number -- aprox 3 days FROM {{ ref("streamline__complete_transactions") }} {% endif %}), From d7216e23fd46d2f6d40fad3479ad5c707a63a311 Mon Sep 17 00:00:00 2001 From: xiuy001 Date: Mon, 27 Nov 2023 15:42:03 -0500 Subject: [PATCH 15/46] optimized the params --- .../silver/streamline/realtime/streamline__blocks_realtime.sql | 2 +- .../streamline/realtime/streamline__transactions_realtime.sql | 2 +- .../streamline/realtime/streamline__tx_receipts_realtime.sql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/models/silver/streamline/realtime/streamline__blocks_realtime.sql b/models/silver/streamline/realtime/streamline__blocks_realtime.sql index 349a917..e7c68bd 100644 --- a/models/silver/streamline/realtime/streamline__blocks_realtime.sql +++ b/models/silver/streamline/realtime/streamline__blocks_realtime.sql @@ -1,7 +1,7 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table','blocks', 'producer_batch_size',500000, 'producer_limit_size', 20000000, 'worker_batch_size',5000))", + func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table','blocks', 'producer_batch_size',10000, 'producer_limit_size',2000000, 'worker_batch_size',100))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} diff --git a/models/silver/streamline/realtime/streamline__transactions_realtime.sql b/models/silver/streamline/realtime/streamline__transactions_realtime.sql index 7a59483..09cd7cd 100644 --- a/models/silver/streamline/realtime/streamline__transactions_realtime.sql +++ b/models/silver/streamline/realtime/streamline__transactions_realtime.sql @@ -1,7 +1,7 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'transactions', 'exploded_key','[\"result\", \"transactions\"]', 'producer_batch_size',500000, 'producer_limit_size', 20000000, 'worker_batch_size',5000))", + func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'transactions', 'exploded_key','[\"result\", \"transactions\"]', 'producer_batch_size',10000, 'producer_limit_size',2000000, 'worker_batch_size',100))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} diff --git a/models/silver/streamline/realtime/streamline__tx_receipts_realtime.sql b/models/silver/streamline/realtime/streamline__tx_receipts_realtime.sql index be26800..906a62a 100644 --- a/models/silver/streamline/realtime/streamline__tx_receipts_realtime.sql +++ b/models/silver/streamline/realtime/streamline__tx_receipts_realtime.sql @@ -1,7 +1,7 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'tx_receipts', 'producer_batch_size',200000, 'producer_limit_size', 20000000, 'worker_batch_size',400))", + func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'tx_receipts', 'producer_batch_size',10000, 'producer_limit_size',2000000, 'worker_batch_size',100))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} From d38223a5dc4fd311a4f0c9257294c2511439e9b9 Mon Sep 17 00:00:00 2001 From: xiuy001 Date: Tue, 28 Nov 2023 15:30:06 -0500 Subject: [PATCH 16/46] updated --- models/silver/silver__blocks.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/models/silver/silver__blocks.sql b/models/silver/silver__blocks.sql index 8eb39e2..08993bf 100644 --- a/models/silver/silver__blocks.sql +++ b/models/silver/silver__blocks.sql @@ -1,6 +1,7 @@ -- depends_on: {{ ref('bronze__streamline_blocks') }} {{ config( materialized = 'incremental', + incremental_strategy = 'delete+insert', unique_key = "block_number", cluster_by = "block_timestamp::date", tags = ['core'] From 5d433380e86086deced7cdf08efdb15378092b17 Mon Sep 17 00:00:00 2001 From: xiuy001 Date: Wed, 29 Nov 2023 10:45:09 -0500 Subject: [PATCH 17/46] updated --- .../streamline/complete/streamline__complete_blocks.sql | 3 +++ models/silver/streamline/streamline__blocks.sql | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/models/silver/streamline/complete/streamline__complete_blocks.sql b/models/silver/streamline/complete/streamline__complete_blocks.sql index e40b6cb..4910a03 100644 --- a/models/silver/streamline/complete/streamline__complete_blocks.sql +++ b/models/silver/streamline/complete/streamline__complete_blocks.sql @@ -22,8 +22,11 @@ WHERE FROM {{ this }} ) + AND DATA != [] {% else %} {{ ref('bronze__streamline_FR_blocks') }} +WHERE + DATA != [] {% endif %} qualify(ROW_NUMBER() over (PARTITION BY id diff --git a/models/silver/streamline/streamline__blocks.sql b/models/silver/streamline/streamline__blocks.sql index bad8e4d..2603e94 100644 --- a/models/silver/streamline/streamline__blocks.sql +++ b/models/silver/streamline/streamline__blocks.sql @@ -18,4 +18,9 @@ SELECT '' ) AS block_number_hex FROM - TABLE(streamline.udtf_get_base_table({{ block_height }})) + TABLE( + streamline.udtf_get_base_table( + {{ block_height }} + - 2000 + ) + ) -- avoid the missing blocks at the tips of the chainhead, around 1 hour From 8d313e36500bc94d5426f0fd9cd309b8f99dc283 Mon Sep 17 00:00:00 2001 From: xiuy001 Date: Fri, 1 Dec 2023 14:28:30 -0500 Subject: [PATCH 18/46] updated --- models/silver/streamline/streamline__blocks.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/silver/streamline/streamline__blocks.sql b/models/silver/streamline/streamline__blocks.sql index 2603e94..289babd 100644 --- a/models/silver/streamline/streamline__blocks.sql +++ b/models/silver/streamline/streamline__blocks.sql @@ -21,6 +21,6 @@ FROM TABLE( streamline.udtf_get_base_table( {{ block_height }} - - 2000 + - 800 ) ) -- avoid the missing blocks at the tips of the chainhead, around 1 hour From d2168fcef28beee2a68e963d1c343f8ad31681db Mon Sep 17 00:00:00 2001 From: WHYTEWYLL Date: Mon, 11 Dec 2023 15:10:00 +0100 Subject: [PATCH 19/46] add: pk keys --- models/descriptions/pk_id.md | 5 +++++ models/gold/core/core__fact_blocks.sql | 6 +++++- models/gold/core/core__fact_blocks.yml | 8 ++++++++ models/gold/core/core__fact_logs.sql | 5 ++++- models/gold/core/core__fact_logs.yml | 6 ++++++ models/gold/core/core__fact_receipts.sql | 6 +++++- models/gold/core/core__fact_receipts.yml | 8 ++++++++ models/gold/core/core__fact_transactions.sql | 1 + models/gold/core/core__fact_transactions.yml | 8 ++++++++ models/silver/silver__blocks.sql | 8 +++++++- models/silver/silver__logs.sql | 3 +++ models/silver/silver__receipts.sql | 3 +++ models/silver/silver__relevant_contracts.sql | 5 ++++- models/silver/silver__transactions.sql | 5 ++++- 14 files changed, 71 insertions(+), 6 deletions(-) create mode 100644 models/descriptions/pk_id.md diff --git a/models/descriptions/pk_id.md b/models/descriptions/pk_id.md new file mode 100644 index 0000000..8d21a8a --- /dev/null +++ b/models/descriptions/pk_id.md @@ -0,0 +1,5 @@ +{% docs pk_id %} + +A uniquely generated identifier assigned by a surrogate key + +{% enddocs %} diff --git a/models/gold/core/core__fact_blocks.sql b/models/gold/core/core__fact_blocks.sql index 5df19b5..d525747 100644 --- a/models/gold/core/core__fact_blocks.sql +++ b/models/gold/core/core__fact_blocks.sql @@ -6,6 +6,7 @@ ) }} SELECT + block_id, block_number, block_timestamp, 'mainnet' AS network, @@ -61,6 +62,9 @@ SELECT transactions_root, 'uncles', uncles - ) AS block_header_json + ) AS block_header_json, + inserted_timestamp, + modified_timestamp, + invocation_id FROM {{ ref('silver__blocks') }} \ No newline at end of file diff --git a/models/gold/core/core__fact_blocks.yml b/models/gold/core/core__fact_blocks.yml index 742d1b9..d915f5a 100644 --- a/models/gold/core/core__fact_blocks.yml +++ b/models/gold/core/core__fact_blocks.yml @@ -4,6 +4,8 @@ models: description: '{{ doc("blocks_table_doc") }}' columns: + - name: BLOCK_ID + description: '{{ doc("pk_id") }}' - name: BLOCK_NUMBER description: '{{ doc("block_number") }}' - name: BLOCK_TIMESTAMP @@ -42,3 +44,9 @@ models: description: '{{ doc("uncle_blocks") }}' - name: BLOCK_HEADER_JSON description: '{{ doc("block_header_json") }}' + - name: INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + - name: MODIFIED_TIMESTAMP + description: "{{ doc('modified_timestamp') }}" + - name: INVOCACTION_ID + description: "{{ doc('invocation_id') }}" diff --git a/models/gold/core/core__fact_logs.sql b/models/gold/core/core__fact_logs.sql index 919e277..bbd4ebf 100644 --- a/models/gold/core/core__fact_logs.sql +++ b/models/gold/core/core__fact_logs.sql @@ -18,6 +18,9 @@ SELECT DATA, event_removed, tx_status, - _log_id + _log_id, + inserted_timestamp, + modified_timestamp, + invocation_id FROM {{ ref('silver__logs') }} \ No newline at end of file diff --git a/models/gold/core/core__fact_logs.yml b/models/gold/core/core__fact_logs.yml index 81d46fe..1e5a3fd 100644 --- a/models/gold/core/core__fact_logs.yml +++ b/models/gold/core/core__fact_logs.yml @@ -30,3 +30,9 @@ models: description: "The from address at the transaction level." - name: ORIGIN_TO_ADDRESS description: "The to address at the transaction level." + - name: INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + - name: MODIFIED_TIMESTAMP + description: "{{ doc('modified_timestamp') }}" + - name: INVOCACTION_ID + description: "{{ doc('invocation_id') }}" diff --git a/models/gold/core/core__fact_receipts.sql b/models/gold/core/core__fact_receipts.sql index acc9749..36dd073 100644 --- a/models/gold/core/core__fact_receipts.sql +++ b/models/gold/core/core__fact_receipts.sql @@ -6,6 +6,7 @@ ) }} SELECT + receipts_id, block_number, block_timestamp, block_hash, @@ -22,6 +23,9 @@ SELECT POSITION, TYPE, near_receipt_hash, - near_transaction_hash + near_transaction_hash, + inserted_timestamp, + modified_timestamp, + invocation_id FROM {{ ref('silver__receipts') }} \ No newline at end of file diff --git a/models/gold/core/core__fact_receipts.yml b/models/gold/core/core__fact_receipts.yml index 707d38e..bb095eb 100644 --- a/models/gold/core/core__fact_receipts.yml +++ b/models/gold/core/core__fact_receipts.yml @@ -4,6 +4,8 @@ models: description: '{{ doc("receipts_table_doc") }}' columns: + - name: RECEIPTS_ID + description: '{{ doc("pk_id") }}' - name: BLOCK_NUMBER description: '{{ doc("block_number") }}' - name: BLOCK_TIMESTAMP @@ -36,3 +38,9 @@ models: description: "Receipt hash on NEAR blockchain" - name: NEAR_TRANSACTION_HASH description: "Transaction hash on NEAR blockchain" + - name: INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + - name: MODIFIED_TIMESTAMP + description: "{{ doc('modified_timestamp') }}" + - name: INVOCACTION_ID + description: "{{ doc('invocation_id') }}" diff --git a/models/gold/core/core__fact_transactions.sql b/models/gold/core/core__fact_transactions.sql index 28b7205..4a0a33f 100644 --- a/models/gold/core/core__fact_transactions.sql +++ b/models/gold/core/core__fact_transactions.sql @@ -6,6 +6,7 @@ ) }} SELECT + tx_id, block_number, block_hash, block_timestamp, diff --git a/models/gold/core/core__fact_transactions.yml b/models/gold/core/core__fact_transactions.yml index 068a675..08ad524 100644 --- a/models/gold/core/core__fact_transactions.yml +++ b/models/gold/core/core__fact_transactions.yml @@ -4,6 +4,8 @@ models: description: '{{ doc("tx_table_doc") }}' columns: + - name: TX_ID + description: '{{ doc("pk_id") }}' - name: BLOCK_NUMBER description: '{{ doc("block_number") }}' - name: BLOCK_TIMESTAMP @@ -50,3 +52,9 @@ models: description: The s value of the transaction signature. - name: v description: The v value of the transaction signature. + - name: INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + - name: MODIFIED_TIMESTAMP + description: "{{ doc('modified_timestamp') }}" + - name: INVOCACTION_ID + description: "{{ doc('invocation_id') }}" diff --git a/models/silver/silver__blocks.sql b/models/silver/silver__blocks.sql index 08993bf..ffd676a 100644 --- a/models/silver/silver__blocks.sql +++ b/models/silver/silver__blocks.sql @@ -8,6 +8,9 @@ ) }} SELECT + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS block_id, block_number, utils.udf_hex_to_int( DATA :result :timestamp :: STRING @@ -47,7 +50,10 @@ SELECT DATA :result :stateRoot :: STRING AS state_root, DATA :result :transactionsRoot :: STRING AS transactions_root, _partition_by_block_id, - _inserted_timestamp + _inserted_timestamp, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS invocation_id FROM {% if is_incremental() %} diff --git a/models/silver/silver__logs.sql b/models/silver/silver__logs.sql index 0ccf606..c745550 100644 --- a/models/silver/silver__logs.sql +++ b/models/silver/silver__logs.sql @@ -175,6 +175,9 @@ FROM ) SELECT {{ dbt_utils.generate_surrogate_key(['BLOCK_NUMBER','tx_hash', '_LOG_ID']) }} AS logs_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS invocation_id, * FROM FINAL qualify(ROW_NUMBER() over (PARTITION BY tx_hash, _log_id diff --git a/models/silver/silver__receipts.sql b/models/silver/silver__receipts.sql index b14f03c..cd47225 100644 --- a/models/silver/silver__receipts.sql +++ b/models/silver/silver__receipts.sql @@ -92,6 +92,9 @@ FINAL AS ( SELECT {{ dbt_utils.generate_surrogate_key(['BLOCK_NUMBER', 'TX_HASH']) }} AS receipts_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS invocation_id, * FROM FINAL diff --git a/models/silver/silver__relevant_contracts.sql b/models/silver/silver__relevant_contracts.sql index 8130ff9..cdae72f 100644 --- a/models/silver/silver__relevant_contracts.sql +++ b/models/silver/silver__relevant_contracts.sql @@ -8,7 +8,10 @@ SELECT contract_address, 'Aurora' AS blockchain, COUNT(*) AS events, - MAX(block_number) AS latest_block + MAX(block_number) AS latest_block, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS invocation_id FROM {{ ref('silver__logs') }} GROUP BY diff --git a/models/silver/silver__transactions.sql b/models/silver/silver__transactions.sql index 84c6094..8385331 100644 --- a/models/silver/silver__transactions.sql +++ b/models/silver/silver__transactions.sql @@ -287,7 +287,10 @@ FROM ) SELECT *, - {{ dbt_utils.generate_surrogate_key(['BLOCK_NUMBER', 'TX_HASH', 'POSITION']) }} AS tx_id + {{ dbt_utils.generate_surrogate_key(['BLOCK_NUMBER', 'TX_HASH', 'POSITION']) }} AS tx_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS invocation_id FROM FINAL qualify(ROW_NUMBER() over (PARTITION BY block_number, tx_hash ORDER BY From e754536b8c31c734fb7bb2238ba48e236e9fa49a Mon Sep 17 00:00:00 2001 From: WHYTEWYLL Date: Mon, 11 Dec 2023 20:58:42 +0100 Subject: [PATCH 20/46] done --- models/gold/core/core__fact_blocks.sql | 3 +-- models/gold/core/core__fact_blocks.yml | 2 -- models/gold/core/core__fact_logs.sql | 3 +-- models/gold/core/core__fact_logs.yml | 2 -- models/gold/core/core__fact_receipts.sql | 3 +-- models/gold/core/core__fact_receipts.yml | 2 -- models/gold/core/core__fact_transactions.sql | 4 +++- models/gold/core/core__fact_transactions.yml | 2 -- 8 files changed, 6 insertions(+), 15 deletions(-) diff --git a/models/gold/core/core__fact_blocks.sql b/models/gold/core/core__fact_blocks.sql index d525747..eab3cf9 100644 --- a/models/gold/core/core__fact_blocks.sql +++ b/models/gold/core/core__fact_blocks.sql @@ -64,7 +64,6 @@ SELECT uncles ) AS block_header_json, inserted_timestamp, - modified_timestamp, - invocation_id + modified_timestamp FROM {{ ref('silver__blocks') }} \ No newline at end of file diff --git a/models/gold/core/core__fact_blocks.yml b/models/gold/core/core__fact_blocks.yml index d915f5a..df6f139 100644 --- a/models/gold/core/core__fact_blocks.yml +++ b/models/gold/core/core__fact_blocks.yml @@ -48,5 +48,3 @@ models: description: "{{ doc('inserted_timestamp') }}" - name: MODIFIED_TIMESTAMP description: "{{ doc('modified_timestamp') }}" - - name: INVOCACTION_ID - description: "{{ doc('invocation_id') }}" diff --git a/models/gold/core/core__fact_logs.sql b/models/gold/core/core__fact_logs.sql index bbd4ebf..b4c26f2 100644 --- a/models/gold/core/core__fact_logs.sql +++ b/models/gold/core/core__fact_logs.sql @@ -20,7 +20,6 @@ SELECT tx_status, _log_id, inserted_timestamp, - modified_timestamp, - invocation_id + modified_timestamp FROM {{ ref('silver__logs') }} \ No newline at end of file diff --git a/models/gold/core/core__fact_logs.yml b/models/gold/core/core__fact_logs.yml index 1e5a3fd..2bc1fb3 100644 --- a/models/gold/core/core__fact_logs.yml +++ b/models/gold/core/core__fact_logs.yml @@ -34,5 +34,3 @@ models: description: "{{ doc('inserted_timestamp') }}" - name: MODIFIED_TIMESTAMP description: "{{ doc('modified_timestamp') }}" - - name: INVOCACTION_ID - description: "{{ doc('invocation_id') }}" diff --git a/models/gold/core/core__fact_receipts.sql b/models/gold/core/core__fact_receipts.sql index 36dd073..04c5ec4 100644 --- a/models/gold/core/core__fact_receipts.sql +++ b/models/gold/core/core__fact_receipts.sql @@ -25,7 +25,6 @@ SELECT near_receipt_hash, near_transaction_hash, inserted_timestamp, - modified_timestamp, - invocation_id + modified_timestamp FROM {{ ref('silver__receipts') }} \ No newline at end of file diff --git a/models/gold/core/core__fact_receipts.yml b/models/gold/core/core__fact_receipts.yml index bb095eb..19465e5 100644 --- a/models/gold/core/core__fact_receipts.yml +++ b/models/gold/core/core__fact_receipts.yml @@ -42,5 +42,3 @@ models: description: "{{ doc('inserted_timestamp') }}" - name: MODIFIED_TIMESTAMP description: "{{ doc('modified_timestamp') }}" - - name: INVOCACTION_ID - description: "{{ doc('invocation_id') }}" diff --git a/models/gold/core/core__fact_transactions.sql b/models/gold/core/core__fact_transactions.sql index 4a0a33f..e69ca12 100644 --- a/models/gold/core/core__fact_transactions.sql +++ b/models/gold/core/core__fact_transactions.sql @@ -29,6 +29,8 @@ SELECT r, s, v, - tx_type + tx_type, + inserted_timestamp, + modified_timestamp FROM {{ ref('silver__transactions') }} \ No newline at end of file diff --git a/models/gold/core/core__fact_transactions.yml b/models/gold/core/core__fact_transactions.yml index 08ad524..1f6cbca 100644 --- a/models/gold/core/core__fact_transactions.yml +++ b/models/gold/core/core__fact_transactions.yml @@ -56,5 +56,3 @@ models: description: "{{ doc('inserted_timestamp') }}" - name: MODIFIED_TIMESTAMP description: "{{ doc('modified_timestamp') }}" - - name: INVOCACTION_ID - description: "{{ doc('invocation_id') }}" From 9fda0466dca9b360292851bf56f085434dc8c10e Mon Sep 17 00:00:00 2001 From: WHYTEWYLL Date: Mon, 11 Dec 2023 21:01:39 +0100 Subject: [PATCH 21/46] silver --- models/silver/atlas/silver__atlas_maa.sql | 2 +- models/silver/silver__blocks.sql | 2 +- models/silver/silver__logs.sql | 2 +- models/silver/silver__receipts.sql | 2 +- models/silver/silver__relevant_contracts.sql | 2 +- models/silver/silver__transactions.sql | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/models/silver/atlas/silver__atlas_maa.sql b/models/silver/atlas/silver__atlas_maa.sql index 36d1ae6..bdd3495 100644 --- a/models/silver/atlas/silver__atlas_maa.sql +++ b/models/silver/atlas/silver__atlas_maa.sql @@ -74,6 +74,6 @@ SELECT maa, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS invocation_id + '{{ invocation_id }}' AS _invocation_id FROM FINAL diff --git a/models/silver/silver__blocks.sql b/models/silver/silver__blocks.sql index ffd676a..08478a9 100644 --- a/models/silver/silver__blocks.sql +++ b/models/silver/silver__blocks.sql @@ -53,7 +53,7 @@ SELECT _inserted_timestamp, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS invocation_id + '{{ invocation_id }}' AS _invocation_id FROM {% if is_incremental() %} diff --git a/models/silver/silver__logs.sql b/models/silver/silver__logs.sql index c745550..1be61f6 100644 --- a/models/silver/silver__logs.sql +++ b/models/silver/silver__logs.sql @@ -177,7 +177,7 @@ SELECT {{ dbt_utils.generate_surrogate_key(['BLOCK_NUMBER','tx_hash', '_LOG_ID']) }} AS logs_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS invocation_id, + '{{ invocation_id }}' AS _invocation_id, * FROM FINAL qualify(ROW_NUMBER() over (PARTITION BY tx_hash, _log_id diff --git a/models/silver/silver__receipts.sql b/models/silver/silver__receipts.sql index cd47225..b9a37dc 100644 --- a/models/silver/silver__receipts.sql +++ b/models/silver/silver__receipts.sql @@ -94,7 +94,7 @@ SELECT {{ dbt_utils.generate_surrogate_key(['BLOCK_NUMBER', 'TX_HASH']) }} AS receipts_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS invocation_id, + '{{ invocation_id }}' AS _invocation_id, * FROM FINAL diff --git a/models/silver/silver__relevant_contracts.sql b/models/silver/silver__relevant_contracts.sql index cdae72f..cbd7fcc 100644 --- a/models/silver/silver__relevant_contracts.sql +++ b/models/silver/silver__relevant_contracts.sql @@ -11,7 +11,7 @@ SELECT MAX(block_number) AS latest_block, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS invocation_id + '{{ invocation_id }}' AS _invocation_id FROM {{ ref('silver__logs') }} GROUP BY diff --git a/models/silver/silver__transactions.sql b/models/silver/silver__transactions.sql index 8385331..7809180 100644 --- a/models/silver/silver__transactions.sql +++ b/models/silver/silver__transactions.sql @@ -290,7 +290,7 @@ SELECT {{ dbt_utils.generate_surrogate_key(['BLOCK_NUMBER', 'TX_HASH', 'POSITION']) }} AS tx_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS invocation_id + '{{ invocation_id }}' AS _invocation_id FROM FINAL qualify(ROW_NUMBER() over (PARTITION BY block_number, tx_hash ORDER BY From 98e2303fd0f34cedf1f2b191bb64e130f746fdc3 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 11 Dec 2023 13:05:41 -0700 Subject: [PATCH 22/46] upd atlas yml --- models/silver/atlas/silver__atlas_maa.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/models/silver/atlas/silver__atlas_maa.yml b/models/silver/atlas/silver__atlas_maa.yml index 644dc2e..f7c8f2e 100644 --- a/models/silver/atlas/silver__atlas_maa.yml +++ b/models/silver/atlas/silver__atlas_maa.yml @@ -25,15 +25,9 @@ models: - name: inserted_timestamp description: "{{ doc('inserted_timestamp') }}" - tests: - - not_null - name: modified_timestamp description: "{{ doc('modified_timestamp') }}" - tests: - - not_null - - name: invocation_id + - name: _invocation_id description: "{{ doc('invocation_id') }}" - tests: - - not_null From 43dcdb79f4c91162059e8de43834e8c9a882e8c5 Mon Sep 17 00:00:00 2001 From: WHYTEWYLL Date: Mon, 11 Dec 2023 22:17:29 +0100 Subject: [PATCH 23/46] add: coalesce --- models/gold/core/core__fact_blocks.sql | 15 ++++++++++++--- models/gold/core/core__fact_logs.sql | 14 ++++++++++++-- models/gold/core/core__fact_receipts.sql | 15 ++++++++++++--- models/gold/core/core__fact_transactions.sql | 15 ++++++++++++--- 4 files changed, 48 insertions(+), 11 deletions(-) diff --git a/models/gold/core/core__fact_blocks.sql b/models/gold/core/core__fact_blocks.sql index eab3cf9..dc217f7 100644 --- a/models/gold/core/core__fact_blocks.sql +++ b/models/gold/core/core__fact_blocks.sql @@ -6,7 +6,10 @@ ) }} SELECT - block_id, + COALESCE ( + block_id, + {{ dbt_utils.generate_surrogate_key(['block_number']) }} + ) AS block_number, block_number, block_timestamp, 'mainnet' AS network, @@ -63,7 +66,13 @@ SELECT 'uncles', uncles ) AS block_header_json, - inserted_timestamp, - modified_timestamp + COALESCE ( + inserted_timestamp, + _inserted_timestamp + ) AS inserted_timestamp, + COALESCE ( + modified_timestamp, + _inserted_timestamp + ) AS modified_timestamp FROM {{ ref('silver__blocks') }} \ No newline at end of file diff --git a/models/gold/core/core__fact_logs.sql b/models/gold/core/core__fact_logs.sql index b4c26f2..1a2d1a6 100644 --- a/models/gold/core/core__fact_logs.sql +++ b/models/gold/core/core__fact_logs.sql @@ -6,6 +6,10 @@ ) }} SELECT + COALESCE ( + logs_id, + {{ dbt_utils.generate_surrogate_key(['BLOCK_NUMBER','tx_hash', '_LOG_ID']) }} + ) AS block_number, block_number, block_timestamp, tx_hash, @@ -19,7 +23,13 @@ SELECT event_removed, tx_status, _log_id, - inserted_timestamp, - modified_timestamp + COALESCE ( + inserted_timestamp, + _inserted_timestamp + ) AS inserted_timestamp, + COALESCE ( + modified_timestamp, + _inserted_timestamp + ) AS modified_timestamp FROM {{ ref('silver__logs') }} \ No newline at end of file diff --git a/models/gold/core/core__fact_receipts.sql b/models/gold/core/core__fact_receipts.sql index 04c5ec4..73e02f4 100644 --- a/models/gold/core/core__fact_receipts.sql +++ b/models/gold/core/core__fact_receipts.sql @@ -6,7 +6,10 @@ ) }} SELECT - receipts_id, + COALESCE ( + receipts_id, + {{ dbt_utils.generate_surrogate_key(['BLOCK_NUMBER', 'TX_HASH']) }} + ) AS receipts_id, block_number, block_timestamp, block_hash, @@ -24,7 +27,13 @@ SELECT TYPE, near_receipt_hash, near_transaction_hash, - inserted_timestamp, - modified_timestamp + COALESCE ( + inserted_timestamp, + _inserted_timestamp + ) AS inserted_timestamp, + COALESCE ( + modified_timestamp, + _inserted_timestamp + ) AS modified_timestamp FROM {{ ref('silver__receipts') }} \ No newline at end of file diff --git a/models/gold/core/core__fact_transactions.sql b/models/gold/core/core__fact_transactions.sql index e69ca12..ce8bd32 100644 --- a/models/gold/core/core__fact_transactions.sql +++ b/models/gold/core/core__fact_transactions.sql @@ -6,7 +6,10 @@ ) }} SELECT - tx_id, + COALESCE ( + tx_id, + {{ dbt_utils.generate_surrogate_key(['BLOCK_NUMBER', 'TX_HASH', 'POSITION']) }} + ) AS tx_id, block_number, block_hash, block_timestamp, @@ -30,7 +33,13 @@ SELECT s, v, tx_type, - inserted_timestamp, - modified_timestamp + COALESCE ( + inserted_timestamp, + _inserted_timestamp + ) AS inserted_timestamp, + COALESCE ( + modified_timestamp, + _inserted_timestamp + ) AS modified_timestamp FROM {{ ref('silver__transactions') }} \ No newline at end of file From 76dc1dfbb8af6c263895b06f17f611951caf9c66 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 11 Dec 2023 16:47:59 -0700 Subject: [PATCH 24/46] fix gold pk ids --- models/gold/atlas/atlas__fact_maas.yml | 10 +++++----- models/gold/core/core__fact_blocks.sql | 10 +++++----- models/gold/core/core__fact_blocks.yml | 4 ++-- models/gold/core/core__fact_logs.sql | 10 +++++----- models/gold/core/core__fact_logs.yml | 2 ++ models/gold/core/core__fact_receipts.sql | 8 ++++---- models/gold/core/core__fact_receipts.yml | 4 ++-- models/gold/core/core__fact_transactions.sql | 10 +++++----- models/gold/core/core__fact_transactions.yml | 4 ++-- 9 files changed, 32 insertions(+), 30 deletions(-) diff --git a/models/gold/atlas/atlas__fact_maas.yml b/models/gold/atlas/atlas__fact_maas.yml index abdf8e0..4ab9263 100644 --- a/models/gold/atlas/atlas__fact_maas.yml +++ b/models/gold/atlas/atlas__fact_maas.yml @@ -6,29 +6,29 @@ models: Monthly Active Accounts (wallets) on Aurora, calculated over a rolling 30 day window. An active account, here, is defined as the signing of at least one transaction. columns: - - name: fact_maas_id + - name: FACT_MAAS_ID description: "{{ doc('id') }}" tests: - not_null - unique - - name: day + - name: DAY description: "{{ doc('active_day') }}" tests: - not_null - unique - - name: maa + - name: MAA description: "{{ doc('maa')}}" tests: - not_null - - name: inserted_timestamp + - name: INSERTED_TIMESTAMP description: "{{ doc('inserted_timestamp') }}" tests: - not_null - - name: modified_timestamp + - name: MODIFIED_TIMESTAMP description: "{{ doc('modified_timestamp') }}" tests: - not_null diff --git a/models/gold/core/core__fact_blocks.sql b/models/gold/core/core__fact_blocks.sql index dc217f7..c816661 100644 --- a/models/gold/core/core__fact_blocks.sql +++ b/models/gold/core/core__fact_blocks.sql @@ -6,10 +6,6 @@ ) }} SELECT - COALESCE ( - block_id, - {{ dbt_utils.generate_surrogate_key(['block_number']) }} - ) AS block_number, block_number, block_timestamp, 'mainnet' AS network, @@ -66,6 +62,10 @@ SELECT 'uncles', uncles ) AS block_header_json, + COALESCE ( + block_id, + {{ dbt_utils.generate_surrogate_key(['block_number']) }} + ) AS fact_blocks_id, COALESCE ( inserted_timestamp, _inserted_timestamp @@ -75,4 +75,4 @@ SELECT _inserted_timestamp ) AS modified_timestamp FROM - {{ ref('silver__blocks') }} \ No newline at end of file + {{ ref('silver__blocks') }} diff --git a/models/gold/core/core__fact_blocks.yml b/models/gold/core/core__fact_blocks.yml index df6f139..e835075 100644 --- a/models/gold/core/core__fact_blocks.yml +++ b/models/gold/core/core__fact_blocks.yml @@ -4,8 +4,6 @@ models: description: '{{ doc("blocks_table_doc") }}' columns: - - name: BLOCK_ID - description: '{{ doc("pk_id") }}' - name: BLOCK_NUMBER description: '{{ doc("block_number") }}' - name: BLOCK_TIMESTAMP @@ -44,6 +42,8 @@ models: description: '{{ doc("uncle_blocks") }}' - name: BLOCK_HEADER_JSON description: '{{ doc("block_header_json") }}' + - name: FACT_BLOCKS_ID + description: '{{ doc("pk_id") }}' - name: INSERTED_TIMESTAMP description: "{{ doc('inserted_timestamp') }}" - name: MODIFIED_TIMESTAMP diff --git a/models/gold/core/core__fact_logs.sql b/models/gold/core/core__fact_logs.sql index 1a2d1a6..5dbca57 100644 --- a/models/gold/core/core__fact_logs.sql +++ b/models/gold/core/core__fact_logs.sql @@ -6,10 +6,6 @@ ) }} SELECT - COALESCE ( - logs_id, - {{ dbt_utils.generate_surrogate_key(['BLOCK_NUMBER','tx_hash', '_LOG_ID']) }} - ) AS block_number, block_number, block_timestamp, tx_hash, @@ -23,6 +19,10 @@ SELECT event_removed, tx_status, _log_id, + COALESCE ( + logs_id, + {{ dbt_utils.generate_surrogate_key(['BLOCK_NUMBER','tx_hash', '_LOG_ID']) }} + ) AS fact_logs_id, COALESCE ( inserted_timestamp, _inserted_timestamp @@ -32,4 +32,4 @@ SELECT _inserted_timestamp ) AS modified_timestamp FROM - {{ ref('silver__logs') }} \ No newline at end of file + {{ ref('silver__logs') }} diff --git a/models/gold/core/core__fact_logs.yml b/models/gold/core/core__fact_logs.yml index 2bc1fb3..aac1146 100644 --- a/models/gold/core/core__fact_logs.yml +++ b/models/gold/core/core__fact_logs.yml @@ -30,6 +30,8 @@ models: description: "The from address at the transaction level." - name: ORIGIN_TO_ADDRESS description: "The to address at the transaction level." + - name: FACT_LOGS_ID + description: '{{ doc("pk_id") }}' - name: INSERTED_TIMESTAMP description: "{{ doc('inserted_timestamp') }}" - name: MODIFIED_TIMESTAMP diff --git a/models/gold/core/core__fact_receipts.sql b/models/gold/core/core__fact_receipts.sql index 73e02f4..751e891 100644 --- a/models/gold/core/core__fact_receipts.sql +++ b/models/gold/core/core__fact_receipts.sql @@ -6,10 +6,6 @@ ) }} SELECT - COALESCE ( - receipts_id, - {{ dbt_utils.generate_surrogate_key(['BLOCK_NUMBER', 'TX_HASH']) }} - ) AS receipts_id, block_number, block_timestamp, block_hash, @@ -27,6 +23,10 @@ SELECT TYPE, near_receipt_hash, near_transaction_hash, + COALESCE ( + receipts_id, + {{ dbt_utils.generate_surrogate_key(['BLOCK_NUMBER', 'TX_HASH']) }} + ) AS fact_receipts_id, COALESCE ( inserted_timestamp, _inserted_timestamp diff --git a/models/gold/core/core__fact_receipts.yml b/models/gold/core/core__fact_receipts.yml index 19465e5..7341550 100644 --- a/models/gold/core/core__fact_receipts.yml +++ b/models/gold/core/core__fact_receipts.yml @@ -4,8 +4,6 @@ models: description: '{{ doc("receipts_table_doc") }}' columns: - - name: RECEIPTS_ID - description: '{{ doc("pk_id") }}' - name: BLOCK_NUMBER description: '{{ doc("block_number") }}' - name: BLOCK_TIMESTAMP @@ -38,6 +36,8 @@ models: description: "Receipt hash on NEAR blockchain" - name: NEAR_TRANSACTION_HASH description: "Transaction hash on NEAR blockchain" + - name: FACT_RECEIPTS_ID + description: '{{ doc("pk_id") }}' - name: INSERTED_TIMESTAMP description: "{{ doc('inserted_timestamp') }}" - name: MODIFIED_TIMESTAMP diff --git a/models/gold/core/core__fact_transactions.sql b/models/gold/core/core__fact_transactions.sql index ce8bd32..4e741d5 100644 --- a/models/gold/core/core__fact_transactions.sql +++ b/models/gold/core/core__fact_transactions.sql @@ -6,10 +6,6 @@ ) }} SELECT - COALESCE ( - tx_id, - {{ dbt_utils.generate_surrogate_key(['BLOCK_NUMBER', 'TX_HASH', 'POSITION']) }} - ) AS tx_id, block_number, block_hash, block_timestamp, @@ -33,6 +29,10 @@ SELECT s, v, tx_type, + COALESCE ( + tx_id, + {{ dbt_utils.generate_surrogate_key(['BLOCK_NUMBER', 'TX_HASH', 'POSITION']) }} + ) AS fact_transactions_id, COALESCE ( inserted_timestamp, _inserted_timestamp @@ -42,4 +42,4 @@ SELECT _inserted_timestamp ) AS modified_timestamp FROM - {{ ref('silver__transactions') }} \ No newline at end of file + {{ ref('silver__transactions') }} diff --git a/models/gold/core/core__fact_transactions.yml b/models/gold/core/core__fact_transactions.yml index 1f6cbca..a010509 100644 --- a/models/gold/core/core__fact_transactions.yml +++ b/models/gold/core/core__fact_transactions.yml @@ -4,8 +4,6 @@ models: description: '{{ doc("tx_table_doc") }}' columns: - - name: TX_ID - description: '{{ doc("pk_id") }}' - name: BLOCK_NUMBER description: '{{ doc("block_number") }}' - name: BLOCK_TIMESTAMP @@ -52,6 +50,8 @@ models: description: The s value of the transaction signature. - name: v description: The v value of the transaction signature. + - name: FACT_TRANSACTIONS_ID + description: '{{ doc("pk_id") }}' - name: INSERTED_TIMESTAMP description: "{{ doc('inserted_timestamp') }}" - name: MODIFIED_TIMESTAMP From dc15263263d1d26e53e29b0d1ad65ab42a5bc2c3 Mon Sep 17 00:00:00 2001 From: Austin Date: Tue, 19 Dec 2023 16:43:25 -0500 Subject: [PATCH 25/46] traces models --- macros/create_udfs.sql | 1 - .../core/bronze__streamline_FR_traces.sql | 57 +++++++++++++++++++ .../bronze/core/bronze__streamline_traces.sql | 57 +++++++++++++++++++ .../complete/streamline__complete_traces.sql | 31 ++++++++++ .../history/streamline__traces_history.sql | 38 +++++++++++++ models/sources.yml | 1 + 6 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 models/bronze/core/bronze__streamline_FR_traces.sql create mode 100644 models/bronze/core/bronze__streamline_traces.sql create mode 100644 models/silver/streamline/complete/streamline__complete_traces.sql create mode 100644 models/silver/streamline/history/streamline__traces_history.sql diff --git a/macros/create_udfs.sql b/macros/create_udfs.sql index ae881f8..f816180 100644 --- a/macros/create_udfs.sql +++ b/macros/create_udfs.sql @@ -7,6 +7,5 @@ {% endset %} {% do run_query(sql) %} {{- fsc_utils.create_udfs() -}} - {% endif %} {% endmacro %} diff --git a/models/bronze/core/bronze__streamline_FR_traces.sql b/models/bronze/core/bronze__streamline_FR_traces.sql new file mode 100644 index 0000000..89a7c27 --- /dev/null +++ b/models/bronze/core/bronze__streamline_FR_traces.sql @@ -0,0 +1,57 @@ +{{ config ( + materialized = 'view' +) }} + +WITH meta AS ( + + SELECT + registered_on AS _inserted_timestamp, + file_name, + CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER) AS _partition_by_block_id + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", "traces") }}' + ) + ) A +) +SELECT + block_number, + tx_hash, + DATA, + _inserted_timestamp, + MD5( + CAST( + COALESCE(CAST(tx_hash AS text), '' :: STRING) AS text + ) + ) AS id, + s._partition_by_block_id, + s.value AS VALUE +FROM + {{ source( + "bronze_streamline", + "traces" + ) }} + s + JOIN meta b + ON b.file_name = metadata$filename + AND b._partition_by_block_id = s._partition_by_block_id +WHERE + b._partition_by_block_id = s._partition_by_block_id + AND ( + DATA :error :code IS NULL + OR DATA :error :code NOT IN ( + '-32000', + '-32001', + '-32002', + '-32003', + '-32004', + '-32005', + '-32006', + '-32007', + '-32008', + '-32009', + '-32010', + '-32608' + ) + ) diff --git a/models/bronze/core/bronze__streamline_traces.sql b/models/bronze/core/bronze__streamline_traces.sql new file mode 100644 index 0000000..d64f9e0 --- /dev/null +++ b/models/bronze/core/bronze__streamline_traces.sql @@ -0,0 +1,57 @@ +{{ config ( + materialized = 'view' +) }} + +WITH meta AS ( + + SELECT + last_modified AS _inserted_timestamp, + file_name, + CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER) AS _partition_by_block_id + FROM + TABLE( + information_schema.external_table_file_registration_history( + start_time => DATEADD('day', -3, CURRENT_TIMESTAMP()), + table_name => '{{ source( "bronze_streamline", "traces") }}') + ) A + ) + SELECT + block_number, + tx_hash, + DATA, + _inserted_timestamp, + MD5( + CAST( + COALESCE(CAST(tx_hash AS text), '' :: STRING) AS text + ) + ) AS id, + s._partition_by_block_id, + s.value AS VALUE + FROM + {{ source( + "bronze_streamline", + "traces" + ) }} + s + JOIN meta b + ON b.file_name = metadata$filename + AND b._partition_by_block_id = s._partition_by_block_id + WHERE + b._partition_by_block_id = s._partition_by_block_id + AND ( + DATA :error :code IS NULL + OR DATA :error :code NOT IN ( + '-32000', + '-32001', + '-32002', + '-32003', + '-32004', + '-32005', + '-32006', + '-32007', + '-32008', + '-32009', + '-32010', + '-32608' + ) + ) diff --git a/models/silver/streamline/complete/streamline__complete_traces.sql b/models/silver/streamline/complete/streamline__complete_traces.sql new file mode 100644 index 0000000..55bc355 --- /dev/null +++ b/models/silver/streamline/complete/streamline__complete_traces.sql @@ -0,0 +1,31 @@ +-- depends_on: {{ ref('bronze__streamline_transactions') }} +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)" +) }} + +SELECT + id, + block_number, + tx_hash, + _inserted_timestamp +FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_traces') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) +{% else %} + {{ ref('bronze__streamline_FR_traces') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/streamline/history/streamline__traces_history.sql b/models/silver/streamline/history/streamline__traces_history.sql new file mode 100644 index 0000000..9915832 --- /dev/null +++ b/models/silver/streamline/history/streamline__traces_history.sql @@ -0,0 +1,38 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'traces', 'exploded_key','[\"result\"]', 'producer_batch_size',10000, 'producer_limit_size',2000000, 'worker_batch_size',100))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH tbl AS ( + + SELECT + block_number, + tx_hash + FROM + {{ ref("silver__transactions") }} + WHERE + block_number IS NOT NULL + AND tx_hash IS NOT NULL + EXCEPT + SELECT + block_number, + tx_hash + FROM + {{ ref("streamline__complete_traces") }} + WHERE + block_number IS NOT NULL + AND tx_hash IS NOT NULL +) +SELECT + block_number, + 'debug_traceTransaction' AS method, + tx_hash AS params +FROM + tbl +ORDER BY + block_number ASC +LIMIT + 50 diff --git a/models/sources.yml b/models/sources.yml index 83ab44d..fa42ca7 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -18,6 +18,7 @@ sources: - name: blocks - name: transactions - name: tx_receipts + - name: traces - name: crosschain_silver database: crosschain From 5d19ecfb2e23039fc78fd9614997eb5981c9cd1f Mon Sep 17 00:00:00 2001 From: Austin Date: Tue, 19 Dec 2023 16:45:25 -0500 Subject: [PATCH 26/46] traces --- .../silver/streamline/complete/streamline__complete_traces.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/silver/streamline/complete/streamline__complete_traces.sql b/models/silver/streamline/complete/streamline__complete_traces.sql index 55bc355..d386fcd 100644 --- a/models/silver/streamline/complete/streamline__complete_traces.sql +++ b/models/silver/streamline/complete/streamline__complete_traces.sql @@ -1,4 +1,4 @@ --- depends_on: {{ ref('bronze__streamline_transactions') }} +-- depends_on: {{ ref('bronze__streamline_traces') }} {{ config ( materialized = "incremental", unique_key = "id", From db9e3b6886ed95fe4d6e593237785d8723b8edb5 Mon Sep 17 00:00:00 2001 From: Austin Date: Tue, 19 Dec 2023 20:52:10 -0500 Subject: [PATCH 27/46] history --- .../dbt_run_streamline_traces_history.yml | 34 +++++++++++++++++++ .../core/bronze__streamline_FR_traces.sql | 2 +- .../bronze/core/bronze__streamline_traces.sql | 2 +- .../complete/streamline__complete_traces.sql | 5 +-- .../history/streamline__traces_history.sql | 10 +++--- 5 files changed, 45 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/dbt_run_streamline_traces_history.yml diff --git a/.github/workflows/dbt_run_streamline_traces_history.yml b/.github/workflows/dbt_run_streamline_traces_history.yml new file mode 100644 index 0000000..65f8547 --- /dev/null +++ b/.github/workflows/dbt_run_streamline_traces_history.yml @@ -0,0 +1,34 @@ +name: dbt_run_streamline_traces_history +run-name: dbt_run_streamline_traces_history + +on: + workflow_dispatch: + schedule: + # Run every 2 hours + - cron: "0 */2 * * *" + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + with: + dbt_command: > + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/history/streamline__traces_history.sql + environment: workflow_prod + warehouse: ${{ vars.WAREHOUSE }} + secrets: inherit diff --git a/models/bronze/core/bronze__streamline_FR_traces.sql b/models/bronze/core/bronze__streamline_FR_traces.sql index 89a7c27..2cc38e5 100644 --- a/models/bronze/core/bronze__streamline_FR_traces.sql +++ b/models/bronze/core/bronze__streamline_FR_traces.sql @@ -17,7 +17,7 @@ WITH meta AS ( ) SELECT block_number, - tx_hash, + s.value :metadata :request :params [0] :: STRING AS tx_hash, DATA, _inserted_timestamp, MD5( diff --git a/models/bronze/core/bronze__streamline_traces.sql b/models/bronze/core/bronze__streamline_traces.sql index d64f9e0..b8060de 100644 --- a/models/bronze/core/bronze__streamline_traces.sql +++ b/models/bronze/core/bronze__streamline_traces.sql @@ -17,7 +17,7 @@ WITH meta AS ( ) SELECT block_number, - tx_hash, + s.value :metadata :request :params [0] :: STRING AS tx_hash, DATA, _inserted_timestamp, MD5( diff --git a/models/silver/streamline/complete/streamline__complete_traces.sql b/models/silver/streamline/complete/streamline__complete_traces.sql index d386fcd..8348091 100644 --- a/models/silver/streamline/complete/streamline__complete_traces.sql +++ b/models/silver/streamline/complete/streamline__complete_traces.sql @@ -16,9 +16,10 @@ FROM {% if is_incremental() %} {{ ref('bronze__streamline_traces') }} WHERE - _inserted_timestamp >= ( + _inserted_timestamp >= + ( SELECT - MAX(_inserted_timestamp) _inserted_timestamp + ifnull(MAX(_inserted_timestamp),'1900-01-01' :: timestamp_ntz ) _inserted_timestamp FROM {{ this }} ) diff --git a/models/silver/streamline/history/streamline__traces_history.sql b/models/silver/streamline/history/streamline__traces_history.sql index 9915832..33cdcc8 100644 --- a/models/silver/streamline/history/streamline__traces_history.sql +++ b/models/silver/streamline/history/streamline__traces_history.sql @@ -1,7 +1,7 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'traces', 'exploded_key','[\"result\"]', 'producer_batch_size',10000, 'producer_limit_size',2000000, 'worker_batch_size',100))", + func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'traces', 'producer_batch_size',10000, 'producer_limit_size',1000000, 'worker_batch_size',100))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} @@ -29,10 +29,12 @@ WITH tbl AS ( SELECT block_number, 'debug_traceTransaction' AS method, - tx_hash AS params + CONCAT( + tx_hash, + '_-_', + '{"tracer": "callTracer","timeout": "30s"}' + ) AS params FROM tbl ORDER BY block_number ASC -LIMIT - 50 From 7c4c488600248ace0b805b5b66fc50bdef4c1050 Mon Sep 17 00:00:00 2001 From: Austin Date: Wed, 20 Dec 2023 13:05:40 -0500 Subject: [PATCH 28/46] tune --- models/silver/streamline/history/streamline__traces_history.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/silver/streamline/history/streamline__traces_history.sql b/models/silver/streamline/history/streamline__traces_history.sql index 33cdcc8..4909cdf 100644 --- a/models/silver/streamline/history/streamline__traces_history.sql +++ b/models/silver/streamline/history/streamline__traces_history.sql @@ -1,7 +1,7 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'traces', 'producer_batch_size',10000, 'producer_limit_size',1000000, 'worker_batch_size',100))", + func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'traces', 'producer_batch_size',20000, 'producer_limit_size',5000000, 'worker_batch_size',200))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} From 8dee46a634155e81a80ff8c4a3627047c93e1f5e Mon Sep 17 00:00:00 2001 From: WHYTEWYLL <49686519+WHYTEWYLL@users.noreply.github.com> Date: Fri, 5 Jan 2024 21:08:22 +0700 Subject: [PATCH 29/46] add: traces and dbt update --- dbt_project.yml | 4 +- models/descriptions/precise_amounts.md | 17 + models/descriptions/traces_block_no.md | 5 + models/descriptions/traces_blocktime.md | 5 + models/descriptions/traces_call_data.md | 5 + models/descriptions/traces_from.md | 5 + models/descriptions/traces_gas.md | 5 + models/descriptions/traces_gas_used.md | 5 + models/descriptions/traces_identifier.md | 5 + models/descriptions/traces_index.md | 5 + models/descriptions/traces_input.md | 5 + models/descriptions/traces_output.md | 5 + models/descriptions/traces_sub.md | 5 + models/descriptions/traces_table_doc.md | 5 + models/descriptions/traces_to.md | 5 + models/descriptions/traces_tx_hash.md | 5 + models/descriptions/traces_type.md | 5 + models/descriptions/traces_value.md | 5 + models/gold/core/core__fact_blocks.sql | 3 +- models/gold/core/core__fact_logs.sql | 3 +- models/gold/core/core__fact_receipts.sql | 5 +- models/gold/core/core__fact_traces.sql | 46 ++ models/gold/core/core__fact_traces.yml | 52 +++ models/gold/core/core__fact_transactions.sql | 3 +- models/silver/silver__traces.sql | 413 ++++++++++++++++++ .../receipts/test_silver__receipts_full.sql | 2 +- .../test/traces/test_silver__traces_full.sql | 9 + .../test/traces/test_silver__traces_full.yml | 57 +++ .../traces/test_silver__traces_recent.sql | 27 ++ .../traces/test_silver__traces_recent.yml | 34 ++ 30 files changed, 743 insertions(+), 12 deletions(-) create mode 100644 models/descriptions/precise_amounts.md create mode 100644 models/descriptions/traces_block_no.md create mode 100644 models/descriptions/traces_blocktime.md create mode 100644 models/descriptions/traces_call_data.md create mode 100644 models/descriptions/traces_from.md create mode 100644 models/descriptions/traces_gas.md create mode 100644 models/descriptions/traces_gas_used.md create mode 100644 models/descriptions/traces_identifier.md create mode 100644 models/descriptions/traces_index.md create mode 100644 models/descriptions/traces_input.md create mode 100644 models/descriptions/traces_output.md create mode 100644 models/descriptions/traces_sub.md create mode 100644 models/descriptions/traces_table_doc.md create mode 100644 models/descriptions/traces_to.md create mode 100644 models/descriptions/traces_tx_hash.md create mode 100644 models/descriptions/traces_type.md create mode 100644 models/descriptions/traces_value.md create mode 100644 models/gold/core/core__fact_traces.sql create mode 100644 models/gold/core/core__fact_traces.yml create mode 100644 models/silver/silver__traces.sql create mode 100644 models/silver/test/traces/test_silver__traces_full.sql create mode 100644 models/silver/test/traces/test_silver__traces_full.yml create mode 100644 models/silver/test/traces/test_silver__traces_recent.sql create mode 100644 models/silver/test/traces/test_silver__traces_recent.yml diff --git a/dbt_project.yml b/dbt_project.yml index 109f150..8968aea 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -5,7 +5,7 @@ name: "aurora_models" version: "1.0.0" config-version: 2 -require-dbt-version: ">=1.4.0" +require-dbt-version: ">=1.7.0" # This setting configures which "profile" dbt uses for this project. profile: "aurora" @@ -68,5 +68,5 @@ dispatch: - dbt query-comment: - comment: '{{ dbt_snowflake_query_tags.get_query_comment(node) }}' + comment: "{{ dbt_snowflake_query_tags.get_query_comment(node) }}" append: true # Snowflake removes prefixed comments. diff --git a/models/descriptions/precise_amounts.md b/models/descriptions/precise_amounts.md new file mode 100644 index 0000000..60590ba --- /dev/null +++ b/models/descriptions/precise_amounts.md @@ -0,0 +1,17 @@ +{% docs precise_amount_unadjusted %} + +The precise, unadjusted amount of the transaction. This is returned as a string to avoid precision loss. + +{% enddocs %} + +{% docs precise_amount_adjusted %} + +The precise, adjusted amount of the transaction. This is returned as a string to avoid precision loss. + +{% enddocs %} + +{% docs tx_fee_precise %} + +The precise amount of the transaction fee. This is returned as a string to avoid precision loss. + +{% enddocs %} diff --git a/models/descriptions/traces_block_no.md b/models/descriptions/traces_block_no.md new file mode 100644 index 0000000..a9653a8 --- /dev/null +++ b/models/descriptions/traces_block_no.md @@ -0,0 +1,5 @@ +{% docs traces_block_no %} + +The block number of this transaction. + +{% enddocs %} diff --git a/models/descriptions/traces_blocktime.md b/models/descriptions/traces_blocktime.md new file mode 100644 index 0000000..4930f1f --- /dev/null +++ b/models/descriptions/traces_blocktime.md @@ -0,0 +1,5 @@ +{% docs traces_blocktime %} + +The block timestamp of this transaction. + +{% enddocs %} diff --git a/models/descriptions/traces_call_data.md b/models/descriptions/traces_call_data.md new file mode 100644 index 0000000..6651a22 --- /dev/null +++ b/models/descriptions/traces_call_data.md @@ -0,0 +1,5 @@ +{% docs traces_call_data %} + +The raw JSON data for this trace. + +{% enddocs %} diff --git a/models/descriptions/traces_from.md b/models/descriptions/traces_from.md new file mode 100644 index 0000000..b5bd708 --- /dev/null +++ b/models/descriptions/traces_from.md @@ -0,0 +1,5 @@ +{% docs traces_from %} + +The sending address of this trace. This is not necessarily the from address of the transaction. + +{% enddocs %} diff --git a/models/descriptions/traces_gas.md b/models/descriptions/traces_gas.md new file mode 100644 index 0000000..12390af --- /dev/null +++ b/models/descriptions/traces_gas.md @@ -0,0 +1,5 @@ +{% docs traces_gas %} + +The gas supplied for this trace. + +{% enddocs %} diff --git a/models/descriptions/traces_gas_used.md b/models/descriptions/traces_gas_used.md new file mode 100644 index 0000000..536f431 --- /dev/null +++ b/models/descriptions/traces_gas_used.md @@ -0,0 +1,5 @@ +{% docs traces_gas_used %} + +The gas used for this trace. + +{% enddocs %} diff --git a/models/descriptions/traces_identifier.md b/models/descriptions/traces_identifier.md new file mode 100644 index 0000000..d59b4d1 --- /dev/null +++ b/models/descriptions/traces_identifier.md @@ -0,0 +1,5 @@ +{% docs traces_identifier %} + +This field represents the position and type of the trace within the transaction. + +{% enddocs %} diff --git a/models/descriptions/traces_index.md b/models/descriptions/traces_index.md new file mode 100644 index 0000000..dccd2a2 --- /dev/null +++ b/models/descriptions/traces_index.md @@ -0,0 +1,5 @@ +{% docs trace_index %} + +The index of the trace within the transaction. + +{% enddocs %} diff --git a/models/descriptions/traces_input.md b/models/descriptions/traces_input.md new file mode 100644 index 0000000..5b136bc --- /dev/null +++ b/models/descriptions/traces_input.md @@ -0,0 +1,5 @@ +{% docs traces_input %} + +The input data for this trace. + +{% enddocs %} diff --git a/models/descriptions/traces_output.md b/models/descriptions/traces_output.md new file mode 100644 index 0000000..b2c7ddc --- /dev/null +++ b/models/descriptions/traces_output.md @@ -0,0 +1,5 @@ +{% docs traces_output %} + +The output data for this trace. + +{% enddocs %} diff --git a/models/descriptions/traces_sub.md b/models/descriptions/traces_sub.md new file mode 100644 index 0000000..c73d7ae --- /dev/null +++ b/models/descriptions/traces_sub.md @@ -0,0 +1,5 @@ +{% docs traces_sub %} + +The amount of nested sub traces for this trace. + +{% enddocs %} diff --git a/models/descriptions/traces_table_doc.md b/models/descriptions/traces_table_doc.md new file mode 100644 index 0000000..fd7807e --- /dev/null +++ b/models/descriptions/traces_table_doc.md @@ -0,0 +1,5 @@ +{% docs traces_table_doc %} + +This table contains flattened trace data for internal contract calls on the Ethereum blockchain. Hex encoded fields can be decoded to integers by using `TO_NUMBER(, 'XXXXXXXXXXXX')`, with the number of Xs being the max length of the encoded field. You must also remove the `0x` from your field to use the `TO_NUMBER()` function, if applicable. + +{% enddocs %} diff --git a/models/descriptions/traces_to.md b/models/descriptions/traces_to.md new file mode 100644 index 0000000..7849178 --- /dev/null +++ b/models/descriptions/traces_to.md @@ -0,0 +1,5 @@ +{% docs traces_to %} + +The receiving address of this trace. This is not necessarily the to address of the transaction. + +{% enddocs %} diff --git a/models/descriptions/traces_tx_hash.md b/models/descriptions/traces_tx_hash.md new file mode 100644 index 0000000..a61ed2f --- /dev/null +++ b/models/descriptions/traces_tx_hash.md @@ -0,0 +1,5 @@ +{% docs traces_tx_hash %} + +The transaction hash for the trace. Please note, this is not necessarily unique in this table as transactions frequently have multiple traces. + +{% enddocs %} diff --git a/models/descriptions/traces_type.md b/models/descriptions/traces_type.md new file mode 100644 index 0000000..012a074 --- /dev/null +++ b/models/descriptions/traces_type.md @@ -0,0 +1,5 @@ +{% docs traces_type %} + +The type of internal transaction. Common trace types are `CALL`, `DELEGATECALL`, and `STATICCALL`. + +{% enddocs %} diff --git a/models/descriptions/traces_value.md b/models/descriptions/traces_value.md new file mode 100644 index 0000000..a180be6 --- /dev/null +++ b/models/descriptions/traces_value.md @@ -0,0 +1,5 @@ +{% docs traces_value %} + +The amount of ETH transferred in this trace. + +{% enddocs %} diff --git a/models/gold/core/core__fact_blocks.sql b/models/gold/core/core__fact_blocks.sql index c816661..013d1a5 100644 --- a/models/gold/core/core__fact_blocks.sql +++ b/models/gold/core/core__fact_blocks.sql @@ -1,8 +1,7 @@ {{ config( materialized = 'view', persist_docs ={ "relation": true, - "columns": true }, - tags = ['core'] + "columns": true } ) }} SELECT diff --git a/models/gold/core/core__fact_logs.sql b/models/gold/core/core__fact_logs.sql index 5dbca57..4772694 100644 --- a/models/gold/core/core__fact_logs.sql +++ b/models/gold/core/core__fact_logs.sql @@ -1,8 +1,7 @@ {{ config( materialized = 'view', persist_docs ={ "relation": true, - "columns": true }, - tags = ['core'] + "columns": true } ) }} SELECT diff --git a/models/gold/core/core__fact_receipts.sql b/models/gold/core/core__fact_receipts.sql index 751e891..b376706 100644 --- a/models/gold/core/core__fact_receipts.sql +++ b/models/gold/core/core__fact_receipts.sql @@ -1,8 +1,7 @@ {{ config( materialized = 'view', persist_docs ={ "relation": true, - "columns": true }, - tags = ['core'] + "columns": true } ) }} SELECT @@ -36,4 +35,4 @@ SELECT _inserted_timestamp ) AS modified_timestamp FROM - {{ ref('silver__receipts') }} \ No newline at end of file + {{ ref('silver__receipts') }} diff --git a/models/gold/core/core__fact_traces.sql b/models/gold/core/core__fact_traces.sql new file mode 100644 index 0000000..7fe02b0 --- /dev/null +++ b/models/gold/core/core__fact_traces.sql @@ -0,0 +1,46 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + tx_hash, + block_number, + block_timestamp, + from_address, + to_address, + eth_value AS VALUE, + eth_value_precise_raw AS value_precise_raw, + eth_value_precise AS value_precise, + gas, + gas_used, + input, + output, + TYPE, + identifier, + DATA, + tx_status, + sub_traces, + trace_status, + error_reason, + trace_index, + COALESCE ( + traces_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'trace_index'] + ) }} + ) AS fact_traces_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp, + eth_value, + eth_value_precise_raw, + eth_value_precise +FROM + {{ ref('silver__traces') }} diff --git a/models/gold/core/core__fact_traces.yml b/models/gold/core/core__fact_traces.yml new file mode 100644 index 0000000..3f1783d --- /dev/null +++ b/models/gold/core/core__fact_traces.yml @@ -0,0 +1,52 @@ +version: 2 +models: + - name: core__fact_traces + description: '{{ doc("traces_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("traces_block_no") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("traces_blocktime") }}' + - name: TX_HASH + description: '{{ doc("traces_tx_hash") }}' + - name: FROM_ADDRESS + description: '{{ doc("traces_from") }}' + - name: TO_ADDRESS + description: '{{ doc("traces_to") }}' + - name: VALUE + description: '{{ doc("traces_value") }}' + - name: VALUE_PRECISE_RAW + description: '{{ doc("precise_amount_unadjusted") }}' + - name: VALUE_PRECISE + description: '{{ doc("precise_amount_adjusted") }}' + - name: GAS + description: '{{ doc("traces_gas") }}' + - name: GAS_USED + description: '{{ doc("traces_gas_used") }}' + - name: INPUT + description: '{{ doc("traces_input") }}' + - name: OUTPUT + description: '{{ doc("traces_output") }}' + - name: TYPE + description: '{{ doc("traces_type") }}' + - name: IDENTIFIER + description: '{{ doc("traces_identifier") }}' + - name: DATA + description: '{{ doc("traces_call_data") }}' + - name: TX_STATUS + description: '{{ doc("tx_status") }}' + - name: SUB_TRACES + description: '{{ doc("traces_sub") }}' + - name: TRACE_STATUS + description: The status of the trace, either `SUCCESS` or `FAIL` + - name: ERROR_REASON + description: The reason for the trace failure, if any. + - name: TRACE_INDEX + description: The index of the trace within the transaction. + - name: FACT_TRACES_ID + description: '{{ doc("pk_id") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' diff --git a/models/gold/core/core__fact_transactions.sql b/models/gold/core/core__fact_transactions.sql index 4e741d5..e488e27 100644 --- a/models/gold/core/core__fact_transactions.sql +++ b/models/gold/core/core__fact_transactions.sql @@ -1,8 +1,7 @@ {{ config( materialized = 'view', persist_docs ={ "relation": true, - "columns": true }, - tags = ['core'] + "columns": true } ) }} SELECT diff --git a/models/silver/silver__traces.sql b/models/silver/silver__traces.sql new file mode 100644 index 0000000..52abf8c --- /dev/null +++ b/models/silver/silver__traces.sql @@ -0,0 +1,413 @@ +-- depends_on: {{ ref('bronze__streamline_traces') }} +{{ config ( + materialized = "incremental", + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = "block_timestamp::date, _inserted_timestamp::date", + tags = ['core'] +) }} + +WITH bronze_traces AS ( + + SELECT + block_number, + VALUE :metadata :request :params [0] :: STRING AS tx_hash, + DATA :result AS full_traces, + _inserted_timestamp + FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_traces') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) +{% else %} + {{ ref('bronze__streamline_FR_traces') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY block_number, tx_hash +ORDER BY + _inserted_timestamp DESC)) = 1 +), +flatten_traces AS ( + SELECT + block_number, + tx_hash, + IFF( + path IN ( + 'result', + 'result.value', + 'result.type', + 'result.to', + 'result.input', + 'result.gasUsed', + 'result.gas', + 'result.from', + 'result.output', + 'result.error', + 'result.revertReason', + 'gasUsed', + 'gas', + 'type', + 'to', + 'from', + 'value', + 'input', + 'error', + 'output', + 'revertReason' + ), + 'ORIGIN', + REGEXP_REPLACE(REGEXP_REPLACE(path, '[^0-9]+', '_'), '^_|_$', '') + ) AS trace_address, + _inserted_timestamp, + OBJECT_AGG( + key, + VALUE + ) AS trace_json, + CASE + WHEN trace_address = 'ORIGIN' THEN NULL + WHEN POSITION( + '_' IN trace_address + ) = 0 THEN 'ORIGIN' + ELSE REGEXP_REPLACE( + trace_address, + '_[0-9]+$', + '', + 1, + 1 + ) + END AS parent_trace_address, + SPLIT( + trace_address, + '_' + ) AS str_array + FROM + bronze_traces txs, + TABLE( + FLATTEN( + input => PARSE_JSON( + txs.full_traces + ), + recursive => TRUE + ) + ) f + WHERE + f.index IS NULL + AND f.key != 'calls' + AND f.path != 'result' + GROUP BY + block_number, + tx_hash, + trace_address, + _inserted_timestamp +), +sub_traces AS ( + SELECT + block_number, + tx_hash, + parent_trace_address, + COUNT(*) AS sub_traces + FROM + flatten_traces + GROUP BY + block_number, + tx_hash, + parent_trace_address +), +num_array AS ( + SELECT + block_number, + tx_hash, + trace_address, + ARRAY_AGG(flat_value) AS num_array + FROM + ( + SELECT + block_number, + tx_hash, + trace_address, + IFF( + VALUE :: STRING = 'ORIGIN', + -1, + VALUE :: INT + ) AS flat_value + FROM + flatten_traces, + LATERAL FLATTEN ( + input => str_array + ) + ) + GROUP BY + block_number, + tx_hash, + trace_address +), +cleaned_traces AS ( + SELECT + b.block_number, + b.tx_hash, + b.trace_address, + IFNULL( + sub_traces, + 0 + ) AS sub_traces, + num_array, + ROW_NUMBER() over ( + PARTITION BY b.block_number, + b.tx_hash + ORDER BY + num_array ASC + ) - 1 AS trace_index, + trace_json, + b._inserted_timestamp + FROM + flatten_traces b + LEFT JOIN sub_traces s + ON b.block_number = s.block_number + AND b.tx_hash = s.tx_hash + AND b.trace_address = s.parent_trace_address + JOIN num_array n + ON b.block_number = n.block_number + AND b.tx_hash = n.tx_hash + AND b.trace_address = n.trace_address +), +final_traces AS ( + SELECT + tx_hash, + trace_index, + block_number, + trace_address, + trace_json :error :: STRING AS error_reason, + trace_json :from :: STRING AS from_address, + trace_json :to :: STRING AS to_address, + IFNULL( + utils.udf_hex_to_int( + trace_json :value :: STRING + ), + '0' + ) AS eth_value_precise_raw, + utils.udf_decimal_adjust( + eth_value_precise_raw, + 18 + ) AS eth_value_precise, + eth_value_precise :: FLOAT AS eth_value, + utils.udf_hex_to_int( + trace_json :gas :: STRING + ) :: INT AS gas, + utils.udf_hex_to_int( + trace_json :gasUsed :: STRING + ) :: INT AS gas_used, + trace_json :input :: STRING AS input, + trace_json :output :: STRING AS output, + trace_json :type :: STRING AS TYPE, + concat_ws( + '_', + TYPE, + trace_address + ) AS identifier, + concat_ws( + '-', + block_number, + tx_hash, + identifier + ) AS _call_id, + _inserted_timestamp, + trace_json AS DATA, + sub_traces + FROM + cleaned_traces +), +new_records AS ( + SELECT + f.block_number, + t.tx_hash, + t.block_timestamp, + t.tx_status, + t.position, + f.trace_index, + f.from_address, + f.to_address, + f.eth_value_precise_raw, + f.eth_value_precise, + f.eth_value, + f.gas, + f.gas_used, + f.input, + f.output, + f.type, + f.identifier, + f.sub_traces, + f.error_reason, + IFF( + f.error_reason IS NULL, + 'SUCCESS', + 'FAIL' + ) AS trace_status, + f.data, + IFF( + t.tx_hash IS NULL + OR t.block_timestamp IS NULL + OR t.tx_status IS NULL, + TRUE, + FALSE + ) AS is_pending, + f._call_id, + f._inserted_timestamp + FROM + final_traces f + LEFT OUTER JOIN {{ ref('silver__transactions') }} + t + ON f.tx_hash = t.tx_hash + AND f.block_number = t.block_number + +{% if is_incremental() %} +AND t._INSERTED_TIMESTAMP >= ( + SELECT + DATEADD('hour', -24, MAX(_inserted_timestamp)) + FROM + {{ this }}) + {% endif %} +) + +{% if is_incremental() %}, +missing_data AS ( + SELECT + t.block_number, + txs.tx_hash, + txs.block_timestamp, + txs.tx_status, + txs.position, + t.trace_index, + t.from_address, + t.to_address, + t.eth_value_precise_raw, + t.eth_value_precise, + t.eth_value, + t.gas, + t.gas_used, + t.input, + t.output, + t.type, + t.identifier, + t.sub_traces, + t.error_reason, + t.trace_status, + t.data, + FALSE AS is_pending, + t._call_id, + GREATEST( + t._inserted_timestamp, + txs._inserted_timestamp + ) AS _inserted_timestamp + FROM + {{ this }} + t + INNER JOIN {{ ref('silver__transactions') }} + txs + ON t.tx_hash = txs.tx_hash + AND t.block_number = txs.block_number + WHERE + t.is_pending +) +{% endif %}, +FINAL AS ( + SELECT + block_number, + tx_hash, + block_timestamp, + tx_status, + position, + trace_index, + from_address, + to_address, + eth_value_precise_raw, + eth_value_precise, + eth_value, + gas, + gas_used, + input, + output, + TYPE, + identifier, + sub_traces, + error_reason, + trace_status, + DATA, + is_pending, + _call_id, + _inserted_timestamp + FROM + new_records + +{% if is_incremental() %} +UNION +SELECT + block_number, + tx_hash, + block_timestamp, + tx_status, + position, + tx_hash, + trace_index, + from_address, + to_address, + eth_value_precise_raw, + eth_value_precise, + eth_value, + gas, + gas_used, + input, + output, + TYPE, + identifier, + sub_traces, + error_reason, + trace_status, + DATA, + is_pending, + _call_id, + _inserted_timestamp +FROM + missing_data +{% endif %} +) +SELECT + block_number, + tx_hash, + block_timestamp, + tx_status, + position AS tx_position, + trace_index, + from_address, + to_address, + eth_value_precise, + eth_value, + gas, + gas_used, + input, + output, + TYPE, + identifier, + sub_traces, + error_reason, + trace_status, + DATA, + is_pending, + _call_id, + _inserted_timestamp, + eth_value_precise_raw, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'trace_index'] + ) }} AS traces_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + FINAL qualify(ROW_NUMBER() over(PARTITION BY block_number, tx_hash, trace_index +ORDER BY + _inserted_timestamp DESC, is_pending ASC)) = 1 diff --git a/models/silver/test/receipts/test_silver__receipts_full.sql b/models/silver/test/receipts/test_silver__receipts_full.sql index a51ab58..6951fe0 100644 --- a/models/silver/test/receipts/test_silver__receipts_full.sql +++ b/models/silver/test/receipts/test_silver__receipts_full.sql @@ -6,4 +6,4 @@ SELECT * FROM - {{ ref('silver__receipts') }} \ No newline at end of file + {{ ref('silver__traces') }} \ No newline at end of file diff --git a/models/silver/test/traces/test_silver__traces_full.sql b/models/silver/test/traces/test_silver__traces_full.sql new file mode 100644 index 0000000..6951fe0 --- /dev/null +++ b/models/silver/test/traces/test_silver__traces_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__traces') }} \ No newline at end of file diff --git a/models/silver/test/traces/test_silver__traces_full.yml b/models/silver/test/traces/test_silver__traces_full.yml new file mode 100644 index 0000000..b925c16 --- /dev/null +++ b/models/silver/test/traces/test_silver__traces_full.yml @@ -0,0 +1,57 @@ +version: 2 +models: + - name: test_silver__traces_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - TX_POSITION + - TRACE_INDEX + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: FROM_ADDRESS + tests: + - not_null: + where: TYPE <> 'SELFDESTRUCT' + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: IDENTIFIER + tests: + - not_null + - name: ETH_VALUE + tests: + - not_null + - name: GAS + tests: + - not_null + - name: GAS_USED + tests: + - not_null diff --git a/models/silver/test/traces/test_silver__traces_recent.sql b/models/silver/test/traces/test_silver__traces_recent.sql new file mode 100644 index 0000000..7c3aae3 --- /dev/null +++ b/models/silver/test/traces/test_silver__traces_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__traces') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/test/traces/test_silver__traces_recent.yml b/models/silver/test/traces/test_silver__traces_recent.yml new file mode 100644 index 0000000..2adb205 --- /dev/null +++ b/models/silver/test/traces/test_silver__traces_recent.yml @@ -0,0 +1,34 @@ +version: 2 +models: + - name: test_silver__traces_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - TX_POSITION + - TRACE_INDEX + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ From 6803fb4a8fed1d2cfac20d2df39b911c7dec0651 Mon Sep 17 00:00:00 2001 From: WHYTEWYLL <49686519+WHYTEWYLL@users.noreply.github.com> Date: Wed, 10 Jan 2024 16:34:18 +0700 Subject: [PATCH 30/46] ticks --- .github/workflows/dbt_run_adhoc.yml | 12 +++--- models/gold/core/core__fact_blocks.sql | 3 +- models/gold/core/core__fact_logs.sql | 3 +- models/gold/core/core__fact_receipts.sql | 3 +- models/gold/core/core__fact_traces.sql | 14 +++---- models/gold/core/core__fact_transactions.sql | 3 +- models/silver/silver__traces.sql | 38 +++++++++---------- .../receipts/test_silver__receipts_full.sql | 2 +- 8 files changed, 40 insertions(+), 38 deletions(-) diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml index 8178217..0c0f963 100644 --- a/.github/workflows/dbt_run_adhoc.yml +++ b/.github/workflows/dbt_run_adhoc.yml @@ -7,7 +7,7 @@ on: - "main" inputs: environment: - type: choice + type: choice description: DBT Run Environment required: true options: @@ -15,9 +15,9 @@ on: - prod default: dev warehouse: - type: choice + type: choice description: Snowflake warehouse - required: true + required: true options: - DBT - DBT_CLOUD @@ -25,9 +25,9 @@ on: default: DBT dbt_command: type: string - description: 'DBT Run Command' + description: "DBT Run Command" required: true - + env: USE_VARS: "${{ vars.USE_VARS }}" DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" @@ -47,7 +47,7 @@ concurrency: jobs: run_dbt_jobs: runs-on: ubuntu-latest - environment: + environment: name: workflow_${{ inputs.environment }} steps: diff --git a/models/gold/core/core__fact_blocks.sql b/models/gold/core/core__fact_blocks.sql index 013d1a5..c816661 100644 --- a/models/gold/core/core__fact_blocks.sql +++ b/models/gold/core/core__fact_blocks.sql @@ -1,7 +1,8 @@ {{ config( materialized = 'view', persist_docs ={ "relation": true, - "columns": true } + "columns": true }, + tags = ['core'] ) }} SELECT diff --git a/models/gold/core/core__fact_logs.sql b/models/gold/core/core__fact_logs.sql index 4772694..5dbca57 100644 --- a/models/gold/core/core__fact_logs.sql +++ b/models/gold/core/core__fact_logs.sql @@ -1,7 +1,8 @@ {{ config( materialized = 'view', persist_docs ={ "relation": true, - "columns": true } + "columns": true }, + tags = ['core'] ) }} SELECT diff --git a/models/gold/core/core__fact_receipts.sql b/models/gold/core/core__fact_receipts.sql index b376706..e3569ff 100644 --- a/models/gold/core/core__fact_receipts.sql +++ b/models/gold/core/core__fact_receipts.sql @@ -1,7 +1,8 @@ {{ config( materialized = 'view', persist_docs ={ "relation": true, - "columns": true } + "columns": true }, + tags = ['core'] ) }} SELECT diff --git a/models/gold/core/core__fact_traces.sql b/models/gold/core/core__fact_traces.sql index 7fe02b0..ae434e0 100644 --- a/models/gold/core/core__fact_traces.sql +++ b/models/gold/core/core__fact_traces.sql @@ -1,7 +1,8 @@ {{ config( materialized = 'view', persist_docs ={ "relation": true, - "columns": true } + "columns": true }, + tags = ['core'] ) }} SELECT @@ -10,9 +11,9 @@ SELECT block_timestamp, from_address, to_address, - eth_value AS VALUE, - eth_value_precise_raw AS value_precise_raw, - eth_value_precise AS value_precise, + value, + value_precise_raw, + value_precise, gas, gas_used, input, @@ -38,9 +39,6 @@ SELECT COALESCE( modified_timestamp, '2000-01-01' - ) AS modified_timestamp, - eth_value, - eth_value_precise_raw, - eth_value_precise + ) AS modified_timestamp FROM {{ ref('silver__traces') }} diff --git a/models/gold/core/core__fact_transactions.sql b/models/gold/core/core__fact_transactions.sql index e488e27..4e741d5 100644 --- a/models/gold/core/core__fact_transactions.sql +++ b/models/gold/core/core__fact_transactions.sql @@ -1,7 +1,8 @@ {{ config( materialized = 'view', persist_docs ={ "relation": true, - "columns": true } + "columns": true }, + tags = ['core'] ) }} SELECT diff --git a/models/silver/silver__traces.sql b/models/silver/silver__traces.sql index 52abf8c..231f067 100644 --- a/models/silver/silver__traces.sql +++ b/models/silver/silver__traces.sql @@ -190,12 +190,12 @@ final_traces AS ( trace_json :value :: STRING ), '0' - ) AS eth_value_precise_raw, + ) AS value_precise_raw, utils.udf_decimal_adjust( - eth_value_precise_raw, + value_precise_raw, 18 - ) AS eth_value_precise, - eth_value_precise :: FLOAT AS eth_value, + ) AS value_precise, + value_precise :: FLOAT AS value, utils.udf_hex_to_int( trace_json :gas :: STRING ) :: INT AS gas, @@ -232,9 +232,9 @@ new_records AS ( f.trace_index, f.from_address, f.to_address, - f.eth_value_precise_raw, - f.eth_value_precise, - f.eth_value, + f.value_precise_raw, + f.value_precise, + f.value, f.gas, f.gas_used, f.input, @@ -285,9 +285,9 @@ missing_data AS ( t.trace_index, t.from_address, t.to_address, - t.eth_value_precise_raw, - t.eth_value_precise, - t.eth_value, + t.value_precise_raw, + t.value_precise, + t.value, t.gas, t.gas_used, t.input, @@ -325,9 +325,9 @@ FINAL AS ( trace_index, from_address, to_address, - eth_value_precise_raw, - eth_value_precise, - eth_value, + value_precise_raw, + value_precise, + value, gas, gas_used, input, @@ -356,9 +356,9 @@ SELECT trace_index, from_address, to_address, - eth_value_precise_raw, - eth_value_precise, - eth_value, + value_precise_raw, + value_precise, + value, gas, gas_used, input, @@ -385,8 +385,8 @@ SELECT trace_index, from_address, to_address, - eth_value_precise, - eth_value, + value_precise, + value, gas, gas_used, input, @@ -400,7 +400,7 @@ SELECT is_pending, _call_id, _inserted_timestamp, - eth_value_precise_raw, + value_precise_raw, {{ dbt_utils.generate_surrogate_key( ['tx_hash', 'trace_index'] ) }} AS traces_id, diff --git a/models/silver/test/receipts/test_silver__receipts_full.sql b/models/silver/test/receipts/test_silver__receipts_full.sql index 6951fe0..a51ab58 100644 --- a/models/silver/test/receipts/test_silver__receipts_full.sql +++ b/models/silver/test/receipts/test_silver__receipts_full.sql @@ -6,4 +6,4 @@ SELECT * FROM - {{ ref('silver__traces') }} \ No newline at end of file + {{ ref('silver__receipts') }} \ No newline at end of file From df01bc42aaedca70325cae395a01bc29844ce107 Mon Sep 17 00:00:00 2001 From: WHYTEWYLL <49686519+WHYTEWYLL@users.noreply.github.com> Date: Wed, 10 Jan 2024 16:36:48 +0700 Subject: [PATCH 31/46] reset dbt version --- dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_project.yml b/dbt_project.yml index 8968aea..26deae9 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -5,7 +5,7 @@ name: "aurora_models" version: "1.0.0" config-version: 2 -require-dbt-version: ">=1.7.0" +require-dbt-version: ">=1.4.0" # This setting configures which "profile" dbt uses for this project. profile: "aurora" From 0ac1297ec3f5cf170077af49db8e67333cf9c613 Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Wed, 10 Jan 2024 15:59:19 -0500 Subject: [PATCH 32/46] hotfix --- models/silver/silver__traces.sql | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/models/silver/silver__traces.sql b/models/silver/silver__traces.sql index 231f067..ebfa448 100644 --- a/models/silver/silver__traces.sql +++ b/models/silver/silver__traces.sql @@ -195,7 +195,7 @@ final_traces AS ( value_precise_raw, 18 ) AS value_precise, - value_precise :: FLOAT AS value, + value_precise :: FLOAT AS VALUE, utils.udf_hex_to_int( trace_json :gas :: STRING ) :: INT AS gas, @@ -321,13 +321,13 @@ FINAL AS ( tx_hash, block_timestamp, tx_status, - position, + POSITION, trace_index, from_address, to_address, value_precise_raw, value_precise, - value, + VALUE, gas, gas_used, input, @@ -351,14 +351,13 @@ SELECT tx_hash, block_timestamp, tx_status, - position, - tx_hash, + POSITION, trace_index, from_address, to_address, value_precise_raw, value_precise, - value, + VALUE, gas, gas_used, input, @@ -381,12 +380,12 @@ SELECT tx_hash, block_timestamp, tx_status, - position AS tx_position, + POSITION AS tx_position, trace_index, from_address, to_address, value_precise, - value, + VALUE, gas, gas_used, input, From 11050df943de5fbf3de7cca85dd0aaee0defffc4 Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Wed, 10 Jan 2024 15:59:34 -0500 Subject: [PATCH 33/46] hotfix for traces --- dbt_project.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dbt_project.yml b/dbt_project.yml index 26deae9..adf412e 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -49,6 +49,10 @@ vars: WAIT: 0 OBSERV_FULL_TEST: False START_GHA_TASKS: False + API_INTEGRATION: '{{ var("config")[target.name]["API_INTEGRATION"] if var("config")[target.name] else var("config")["dev"]["API_INTEGRATION"] }}' + EXTERNAL_FUNCTION_URI: '{{ var("config")[target.name]["EXTERNAL_FUNCTION_URI"] if var("config")[target.name] else var("config")["dev"]["EXTERNAL_FUNCTION_URI"] }}' + ROLES: | + ["INTERNAL_DEV"] tests: +store_failures: true # all tests From b8c3c34340b189ef7275e49d9a1b0c12c4def1f6 Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Wed, 10 Jan 2024 16:01:07 -0500 Subject: [PATCH 34/46] . --- dbt_project.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index adf412e..26deae9 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -49,10 +49,6 @@ vars: WAIT: 0 OBSERV_FULL_TEST: False START_GHA_TASKS: False - API_INTEGRATION: '{{ var("config")[target.name]["API_INTEGRATION"] if var("config")[target.name] else var("config")["dev"]["API_INTEGRATION"] }}' - EXTERNAL_FUNCTION_URI: '{{ var("config")[target.name]["EXTERNAL_FUNCTION_URI"] if var("config")[target.name] else var("config")["dev"]["EXTERNAL_FUNCTION_URI"] }}' - ROLES: | - ["INTERNAL_DEV"] tests: +store_failures: true # all tests From 573fd8cd26422b9a185e631f58d26da156c1846a Mon Sep 17 00:00:00 2001 From: WHYTEWYLL <49686519+WHYTEWYLL@users.noreply.github.com> Date: Fri, 26 Jan 2024 12:56:58 +0700 Subject: [PATCH 35/46] update 1.7 --- .github/workflows/dbt_alter_gha_task.yml | 12 ++++++------ .github/workflows/dbt_docs_update.yml | 4 ++-- .github/workflows/dbt_run_adhoc.yml | 4 ++-- .github/workflows/dbt_run_atlas.yml | 2 +- .github/workflows/dbt_run_deployment.yml | 2 +- .github/workflows/dbt_run_dev_refresh.yml | 2 +- .github/workflows/dbt_run_full_observability.yml | 4 ++-- .github/workflows/dbt_run_observability.yml | 4 ++-- .github/workflows/dbt_run_scheduled.yml | 2 +- .github/workflows/dbt_run_streamline_history.yml | 2 +- ...run_streamline_realtime_blocks_transactions.yml | 2 +- .../dbt_run_streamline_realtime_tx_receipts.yml | 2 +- .../dbt_run_streamline_traces_history.yml | 4 ++-- .github/workflows/dbt_test_daily.yml | 4 ++-- .github/workflows/dbt_test_tasks.yml | 6 +++--- .github/workflows/dbt_test_weekly.yml | 4 ++-- package-lock.yml | 14 ++++++++++++++ requirements.txt | 2 +- 18 files changed, 45 insertions(+), 31 deletions(-) create mode 100644 package-lock.yml diff --git a/.github/workflows/dbt_alter_gha_task.yml b/.github/workflows/dbt_alter_gha_task.yml index da996d8..d3094a6 100644 --- a/.github/workflows/dbt_alter_gha_task.yml +++ b/.github/workflows/dbt_alter_gha_task.yml @@ -11,14 +11,14 @@ on: description: Name of the workflow to perform the action on, no .yml extension required: true task_action: - type: choice + type: choice description: Action to perform required: true options: - SUSPEND - RESUME default: SUSPEND - + env: DBT_PROFILES_DIR: ./ @@ -36,11 +36,11 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@main + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@AN-4374/upgrade-dbt-1.7 with: workflow_name: | - ${{ inputs.workflow_name }} + ${{ inputs.workflow_name }} task_action: | - ${{ inputs.task_action }} + ${{ inputs.task_action }} environment: workflow_prod - secrets: inherit \ No newline at end of file + secrets: inherit diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml index 51f2d88..b6b463d 100644 --- a/.github/workflows/dbt_docs_update.yml +++ b/.github/workflows/dbt_docs_update.yml @@ -23,5 +23,5 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_docs_updates.yml@main - secrets: inherit + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_docs_updates.yml@AN-4374/upgrade-dbt-1.7 + secrets: inherit diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml index 0c0f963..96b5f32 100644 --- a/.github/workflows/dbt_run_adhoc.yml +++ b/.github/workflows/dbt_run_adhoc.yml @@ -53,9 +53,9 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "3.10" - name: install dependencies run: | diff --git a/.github/workflows/dbt_run_atlas.yml b/.github/workflows/dbt_run_atlas.yml index d4949d6..f8fbea0 100644 --- a/.github/workflows/dbt_run_atlas.yml +++ b/.github/workflows/dbt_run_atlas.yml @@ -25,7 +25,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run -s tag:atlas diff --git a/.github/workflows/dbt_run_deployment.yml b/.github/workflows/dbt_run_deployment.yml index 01ceb6b..dc5a8d5 100644 --- a/.github/workflows/dbt_run_deployment.yml +++ b/.github/workflows/dbt_run_deployment.yml @@ -38,7 +38,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_deployment_template.yml@main + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_deployment_template.yml@AN-4374/upgrade-dbt-1.7 with: dbt_command: | ${{ inputs.dbt_command }} diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml index e83e55c..f3232df 100644 --- a/.github/workflows/dbt_run_dev_refresh.yml +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -26,7 +26,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run-operation run_sp_create_prod_clone diff --git a/.github/workflows/dbt_run_full_observability.yml b/.github/workflows/dbt_run_full_observability.yml index 631aaaa..80177d6 100644 --- a/.github/workflows/dbt_run_full_observability.yml +++ b/.github/workflows/dbt_run_full_observability.yml @@ -32,9 +32,9 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "3.10" - name: install dependencies run: | diff --git a/.github/workflows/dbt_run_observability.yml b/.github/workflows/dbt_run_observability.yml index 3c9827c..8cfd82b 100644 --- a/.github/workflows/dbt_run_observability.yml +++ b/.github/workflows/dbt_run_observability.yml @@ -32,9 +32,9 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "3.10" - name: install dependencies run: | diff --git a/.github/workflows/dbt_run_scheduled.yml b/.github/workflows/dbt_run_scheduled.yml index 6af6314..c5bc87e 100644 --- a/.github/workflows/dbt_run_scheduled.yml +++ b/.github/workflows/dbt_run_scheduled.yml @@ -25,7 +25,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run -s tag:core diff --git a/.github/workflows/dbt_run_streamline_history.yml b/.github/workflows/dbt_run_streamline_history.yml index 07ac885..552af64 100644 --- a/.github/workflows/dbt_run_streamline_history.yml +++ b/.github/workflows/dbt_run_streamline_history.yml @@ -22,7 +22,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_RUN_HISTORY":True}' -m 1+models/silver/streamline/realtime/ --full-refresh diff --git a/.github/workflows/dbt_run_streamline_realtime_blocks_transactions.yml b/.github/workflows/dbt_run_streamline_realtime_blocks_transactions.yml index a4689ee..7e28767 100644 --- a/.github/workflows/dbt_run_streamline_realtime_blocks_transactions.yml +++ b/.github/workflows/dbt_run_streamline_realtime_blocks_transactions.yml @@ -25,7 +25,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/realtime/streamline__blocks_realtime.sql 1+models/silver/streamline/realtime/streamline__transactions_realtime.sql diff --git a/.github/workflows/dbt_run_streamline_realtime_tx_receipts.yml b/.github/workflows/dbt_run_streamline_realtime_tx_receipts.yml index 44b8a26..db0625d 100644 --- a/.github/workflows/dbt_run_streamline_realtime_tx_receipts.yml +++ b/.github/workflows/dbt_run_streamline_realtime_tx_receipts.yml @@ -25,7 +25,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/realtime/streamline__tx_receipts_realtime.sql diff --git a/.github/workflows/dbt_run_streamline_traces_history.yml b/.github/workflows/dbt_run_streamline_traces_history.yml index 65f8547..b725ff3 100644 --- a/.github/workflows/dbt_run_streamline_traces_history.yml +++ b/.github/workflows/dbt_run_streamline_traces_history.yml @@ -4,7 +4,7 @@ run-name: dbt_run_streamline_traces_history on: workflow_dispatch: schedule: - # Run every 2 hours + # Run every 2 hours - cron: "0 */2 * * *" env: @@ -25,7 +25,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/history/streamline__traces_history.sql diff --git a/.github/workflows/dbt_test_daily.yml b/.github/workflows/dbt_test_daily.yml index ca470a6..e3b06fa 100644 --- a/.github/workflows/dbt_test_daily.yml +++ b/.github/workflows/dbt_test_daily.yml @@ -32,9 +32,9 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "3.10" - name: install dependencies run: | diff --git a/.github/workflows/dbt_test_tasks.yml b/.github/workflows/dbt_test_tasks.yml index 976440b..594148b 100644 --- a/.github/workflows/dbt_test_tasks.yml +++ b/.github/workflows/dbt_test_tasks.yml @@ -5,7 +5,7 @@ on: workflow_dispatch: branches: - "main" - + env: DBT_PROFILES_DIR: ./ @@ -23,5 +23,5 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@main - secrets: inherit \ No newline at end of file + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@AN-4374/upgrade-dbt-1.7 + secrets: inherit diff --git a/.github/workflows/dbt_test_weekly.yml b/.github/workflows/dbt_test_weekly.yml index 9c73f86..ce504a4 100644 --- a/.github/workflows/dbt_test_weekly.yml +++ b/.github/workflows/dbt_test_weekly.yml @@ -32,9 +32,9 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "3.10" - name: install dependencies run: | diff --git a/package-lock.yml b/package-lock.yml new file mode 100644 index 0000000..36b5126 --- /dev/null +++ b/package-lock.yml @@ -0,0 +1,14 @@ +packages: +- package: calogica/dbt_expectations + version: 0.8.5 +- git: https://github.com/FlipsideCrypto/fsc-utils.git + revision: c84d623aa09ce2acb9451e6aedf5fa70c19b0b95 +- package: get-select/dbt_snowflake_query_tags + version: 2.3.1 +- package: calogica/dbt_date + version: 0.7.2 +- package: dbt-labs/dbt_utils + version: 1.0.0 +- git: https://github.com/FlipsideCrypto/livequery-models.git + revision: bca494102fbd2d621d32746e9a7fe780678044f8 +sha1_hash: 97973763ca6ea590de018e4df5195006f2827e27 diff --git a/requirements.txt b/requirements.txt index ec44b06..c61d3e1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -dbt-snowflake>=1.4,<1.5 \ No newline at end of file +dbt-snowflake>=1.7,<1.8 From b49f9abb7e6cdca555d4638fcd5e023f0fabe63a Mon Sep 17 00:00:00 2001 From: WHYTEWYLL <49686519+WHYTEWYLL@users.noreply.github.com> Date: Fri, 26 Jan 2024 13:27:56 +0700 Subject: [PATCH 36/46] profile --- dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_project.yml b/dbt_project.yml index 26deae9..8968aea 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -5,7 +5,7 @@ name: "aurora_models" version: "1.0.0" config-version: 2 -require-dbt-version: ">=1.4.0" +require-dbt-version: ">=1.7.0" # This setting configures which "profile" dbt uses for this project. profile: "aurora" From 5e32a86f34f1705693e720f0d9f8ed3c1a5a905d Mon Sep 17 00:00:00 2001 From: WHYTEWYLL <49686519+WHYTEWYLL@users.noreply.github.com> Date: Fri, 26 Jan 2024 13:31:03 +0700 Subject: [PATCH 37/46] @ fix --- .github/workflows/dbt_run_atlas.yml | 2 +- .github/workflows/dbt_run_dev_refresh.yml | 2 +- .github/workflows/dbt_run_scheduled.yml | 2 +- .github/workflows/dbt_run_streamline_history.yml | 2 +- .../dbt_run_streamline_realtime_blocks_transactions.yml | 2 +- .github/workflows/dbt_run_streamline_realtime_tx_receipts.yml | 2 +- .github/workflows/dbt_run_streamline_traces_history.yml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dbt_run_atlas.yml b/.github/workflows/dbt_run_atlas.yml index f8fbea0..cbbc3f2 100644 --- a/.github/workflows/dbt_run_atlas.yml +++ b/.github/workflows/dbt_run_atlas.yml @@ -25,7 +25,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run -s tag:atlas diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml index f3232df..cce0cc9 100644 --- a/.github/workflows/dbt_run_dev_refresh.yml +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -26,7 +26,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run-operation run_sp_create_prod_clone diff --git a/.github/workflows/dbt_run_scheduled.yml b/.github/workflows/dbt_run_scheduled.yml index c5bc87e..79393c9 100644 --- a/.github/workflows/dbt_run_scheduled.yml +++ b/.github/workflows/dbt_run_scheduled.yml @@ -25,7 +25,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run -s tag:core diff --git a/.github/workflows/dbt_run_streamline_history.yml b/.github/workflows/dbt_run_streamline_history.yml index 552af64..d42c955 100644 --- a/.github/workflows/dbt_run_streamline_history.yml +++ b/.github/workflows/dbt_run_streamline_history.yml @@ -22,7 +22,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_RUN_HISTORY":True}' -m 1+models/silver/streamline/realtime/ --full-refresh diff --git a/.github/workflows/dbt_run_streamline_realtime_blocks_transactions.yml b/.github/workflows/dbt_run_streamline_realtime_blocks_transactions.yml index 7e28767..3c77a92 100644 --- a/.github/workflows/dbt_run_streamline_realtime_blocks_transactions.yml +++ b/.github/workflows/dbt_run_streamline_realtime_blocks_transactions.yml @@ -25,7 +25,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/realtime/streamline__blocks_realtime.sql 1+models/silver/streamline/realtime/streamline__transactions_realtime.sql diff --git a/.github/workflows/dbt_run_streamline_realtime_tx_receipts.yml b/.github/workflows/dbt_run_streamline_realtime_tx_receipts.yml index db0625d..8d360ee 100644 --- a/.github/workflows/dbt_run_streamline_realtime_tx_receipts.yml +++ b/.github/workflows/dbt_run_streamline_realtime_tx_receipts.yml @@ -25,7 +25,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/realtime/streamline__tx_receipts_realtime.sql diff --git a/.github/workflows/dbt_run_streamline_traces_history.yml b/.github/workflows/dbt_run_streamline_traces_history.yml index b725ff3..d42c2ee 100644 --- a/.github/workflows/dbt_run_streamline_traces_history.yml +++ b/.github/workflows/dbt_run_streamline_traces_history.yml @@ -25,7 +25,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/history/streamline__traces_history.sql From 53ba6813cf8273c6543c81f8be4b4137684b2117 Mon Sep 17 00:00:00 2001 From: WHYTEWYLL <49686519+WHYTEWYLL@users.noreply.github.com> Date: Thu, 22 Feb 2024 16:42:15 +0100 Subject: [PATCH 38/46] stats model aurora --- .github/workflows/dbt_run_non_core.yml | 45 +++++++++++++ models/descriptions/__overview__.md | 41 ++++++++---- models/descriptions/stats_core.md | 65 +++++++++++++++++++ .../stats/stats__ez_core_metrics_hourly.sql | 60 +++++++++++++++++ .../stats/stats__ez_core_metrics_hourly.yml | 32 +++++++++ .../silver__hourly_prices_coin_gecko.sql | 37 +++++++++++ ...ilver_stats__core_metrics_block_hourly.sql | 43 ++++++++++++ ...ilver_stats__core_metrics_block_hourly.yml | 41 ++++++++++++ .../silver_stats__core_metrics_hourly.sql | 58 +++++++++++++++++ .../silver_stats__core_metrics_hourly.yml | 56 ++++++++++++++++ .../test/traces/test_silver__traces_full.yml | 3 - models/sources.yml | 1 + 12 files changed, 466 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/dbt_run_non_core.yml create mode 100644 models/descriptions/stats_core.md create mode 100644 models/gold/stats/stats__ez_core_metrics_hourly.sql create mode 100644 models/gold/stats/stats__ez_core_metrics_hourly.yml create mode 100644 models/silver/crossdb/silver__hourly_prices_coin_gecko.sql create mode 100644 models/silver/stats/silver_stats__core_metrics_block_hourly.sql create mode 100644 models/silver/stats/silver_stats__core_metrics_block_hourly.yml create mode 100644 models/silver/stats/silver_stats__core_metrics_hourly.sql create mode 100644 models/silver/stats/silver_stats__core_metrics_hourly.yml diff --git a/.github/workflows/dbt_run_non_core.yml b/.github/workflows/dbt_run_non_core.yml new file mode 100644 index 0000000..1fe0e1d --- /dev/null +++ b/.github/workflows/dbt_run_non_core.yml @@ -0,0 +1,45 @@ +name: dbt_run_observability +run-name: dbt_run_observability + +on: + workflow_dispatch: + schedule: + # Runs "1 times per day" (see https://crontab.guru) + - cron: "0 6 * * *" + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==${{ env.DBT_VERSION }} cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt run --threads 2 --vars '{"OBSERV_FULL_TEST":False}' -m tag:noncore diff --git a/models/descriptions/__overview__.md b/models/descriptions/__overview__.md index a895481..490e76f 100644 --- a/models/descriptions/__overview__.md +++ b/models/descriptions/__overview__.md @@ -28,6 +28,10 @@ There is more information on how to use dbt docs in the last section of this doc - [fact_logs](#!/model/model.aurora_models.core__fact_logs) - [fact_receipts](#!/model/model.aurora_models.core__fact_receipts) +**Stats EZ Tables:** + +- [ez_core_metrics_hourly](#!/model/model.aurora_models.ez_core_metrics_hourly) + ## **Data Model Overview** The Aurora models are built a few different ways, but the core fact tables are built using three layers of sql models: **bronze, silver, and gold (or core).** @@ -69,31 +73,42 @@ Note that you can also right-click on models to interactively filter and explore - [What is dbt?](https://docs.getdbt.com/docs/introduction) ## Archived Notices + ### **⚠️ Aurora Data Notes and Known Issues** -*Final Update* -We have switched to a node hosted by the Aurora team, and they pushed relevant fixes. The below issues are since resolved. -*Update: July 27, 2023:* +_Final Update_ +We have switched to a node hosted by the Aurora team, and they pushed relevant fixes. The below issues are since resolved. + +_Update: July 27, 2023:_ We have identified 5 primary issues with the data coming from the node, and a path forward to resolution after discussions with the Aurora Dev team: + 1. Duplicated Transactions - - Aurora Node issue, fix in progress by Aurora devs. Resolution timeline: 1-2 weeks + +- Aurora Node issue, fix in progress by Aurora devs. Resolution timeline: 1-2 weeks + 1. Reverted Transactions: Transactions or events that have been previously canceled or reverted are still being received from the chain. - - Aurora Node issue, fix in progress by Aurora devs. Resolution timeline: 1-2 weeks + +- Aurora Node issue, fix in progress by Aurora devs. Resolution timeline: 1-2 weeks + 1. Inaccurate Data: We've detected some inaccuracies in the data, namely incorrect timestamps (10M first block set to 1970) and transactions appearing in the wrong blocks. - - This is not a data integrity issue. Aurora contains pre-history and only launched as a public blockchain with **block 37,157,757**. Early blocks contain incomplete data, such as `0x0` as the block timestamp. + +- This is not a data integrity issue. Aurora contains pre-history and only launched as a public blockchain with **block 37,157,757**. Early blocks contain incomplete data, such as `0x0` as the block timestamp. + 1. Incomplete Data: We've noticed certain transaction data missing, particularly with regards to received transactions. Incomplete data includes some blocks get read with wrong txs count and txs info there, we believed that is from the out of sync status of the node which needs a full backfill again when the node is back sync - - This is likely due to our current node provider using an outdated version of the Aurora RPC package. Resolution: change node provider. + +- This is likely due to our current node provider using an outdated version of the Aurora RPC package. Resolution: change node provider. + 1. Block Confirmation Discrepancies: Transactions were confirmed on different blocks than those indicated in Explorer. - - This is likely due to our current node provider using an outdated version of the Aurora RPC package. Resolution: change node provider. -Our plan of action is (likely) to move to a dedicated node provided by Aurora which will solve the 4 major issues with a single decision. This timeline is dependent on the patch by Aurora, and our timeline will be updated as we learn more. +- This is likely due to our current node provider using an outdated version of the Aurora RPC package. Resolution: change node provider. +Our plan of action is (likely) to move to a dedicated node provided by Aurora which will solve the 4 major issues with a single decision. This timeline is dependent on the patch by Aurora, and our timeline will be updated as we learn more. -*Update: July 20, 2023:* -In onboarding Aurora data, our team has encountered several issues with data returned from the node. These are primarily associated with transactions that are either reverted or cancelled. At present, the node returns these transactions across multiple blocks and in different positions within the block at each time. This is uncommon, as the position should be constant. We may see pending transactions within a block on other EVMs, but on re-request the transaction would be finalized. These seem to be persistent across multiple blocks, even in subsequent requests. +_Update: July 20, 2023:_ +In onboarding Aurora data, our team has encountered several issues with data returned from the node. These are primarily associated with transactions that are either reverted or cancelled. At present, the node returns these transactions across multiple blocks and in different positions within the block at each time. This is uncommon, as the position should be constant. We may see pending transactions within a block on other EVMs, but on re-request the transaction would be finalized. These seem to be persistent across multiple blocks, even in subsequent requests. -At present, these transactions are included in our data. They will have `null` fields like status, fee, and others that are typically derived from receipts. These transactions do not have receipts, so we can identify them through their lack of receipt data. +At present, these transactions are included in our data. They will have `null` fields like status, fee, and others that are typically derived from receipts. These transactions do not have receipts, so we can identify them through their lack of receipt data. -Flipside is working closely with Near and Aurora to determine how this data should best be presented. +Flipside is working closely with Near and Aurora to determine how this data should best be presented. {% enddocs %} diff --git a/models/descriptions/stats_core.md b/models/descriptions/stats_core.md new file mode 100644 index 0000000..b827a4e --- /dev/null +++ b/models/descriptions/stats_core.md @@ -0,0 +1,65 @@ +{% docs ez_core_metrics_hourly_table_doc %} + +A convenience table that aggregates block and transaction related metrics using various aggregate functions such as SUM, COUNT, MIN and MAX from the fact_transactions table, on an hourly basis. Stats for the current hour will be updated as new data arrives. + +{% enddocs %} + +{% docs block_timestamp_hour %} + +The hour of the timestamp of the block. + +{% enddocs %} + +{% docs block_id_min %} + +The minimum block id in the hour. + +{% enddocs %} + +{% docs block_id_max %} + +The maximum block id in the hour. + +{% enddocs %} + +{% docs block_count %} + +The number of blocks in the hour. + +{% enddocs %} + +{% docs transaction_count %} + +The number of transactions in the hour. + +{% enddocs %} + +{% docs transaction_count_success %} + +The number of successful transactions in the hour. + +{% enddocs %} + +{% docs transaction_count_failed %} + +The number of failed transactions in the hour. + +{% enddocs %} + +{% docs unique_from_count %} + +The number of unique tx_from addresses in the hour. + +{% enddocs %} + +{% docs total_fees_native %} + +The sum of all fees in the hour, in the native fee currency. + +{% enddocs %} + +{% docs total_fees_usd %} + +The sum of all fees in the hour, in USD. + +{% enddocs %} diff --git a/models/gold/stats/stats__ez_core_metrics_hourly.sql b/models/gold/stats/stats__ez_core_metrics_hourly.sql new file mode 100644 index 0000000..c6934d3 --- /dev/null +++ b/models/gold/stats/stats__ez_core_metrics_hourly.sql @@ -0,0 +1,60 @@ +{{ config( + materialized = 'view', + tags = ['noncore'], + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'STATS, METRICS, CORE, HOURLY', + }} } +) }} + +WITH txs AS ( + + SELECT + block_timestamp_hour, + transaction_count, + transaction_count_success, + transaction_count_failed, + unique_from_count, + total_fees AS total_fees_native, + LAST_VALUE( + p.close ignore nulls + ) over ( + ORDER BY + block_timestamp_hour rows unbounded preceding + ) AS imputed_close, + core_metrics_hourly_id AS ez_core_metrics_hourly_id, + s.inserted_timestamp AS inserted_timestamp, + s.modified_timestamp AS modified_timestamp + FROM + {{ ref('silver_stats__core_metrics_hourly') }} + s + LEFT JOIN {{ ref('silver__hourly_prices_coin_gecko') }} + p + ON s.block_timestamp_hour = p.recorded_hour + AND p.id = 'sei-network' +) +SELECT + A.block_timestamp_hour, + A.block_number_min, + A.block_number_max, + A.block_count, + b.transaction_count, + b.transaction_count_success, + b.transaction_count_failed, + b.unique_from_count, + b.total_fees_native, + ROUND( + b.total_fees_native * b.imputed_close, + 2 + ) AS total_fees_usd, + A.core_metrics_block_hourly_id AS ez_core_metrics_hourly_id, + GREATEST( + A.inserted_timestamp, + b.inserted_timestamp + ) AS inserted_timestamp, + GREATEST( + A.modified_timestamp, + b.modified_timestamp + ) AS modified_timestamp +FROM + {{ ref('silver_stats__core_metrics_block_hourly') }} A + JOIN txs b + ON A.block_timestamp_hour = b.block_timestamp_hour \ No newline at end of file diff --git a/models/gold/stats/stats__ez_core_metrics_hourly.yml b/models/gold/stats/stats__ez_core_metrics_hourly.yml new file mode 100644 index 0000000..b601935 --- /dev/null +++ b/models/gold/stats/stats__ez_core_metrics_hourly.yml @@ -0,0 +1,32 @@ +version: 2 +models: + - name: stats__ez_core_metrics_hourly + description: '{{ doc("ez_core_metrics_hourly_table_doc") }}' + + columns: + - name: BLOCK_TIMESTAMP_HOUR + description: '{{ doc("block_timestamp_hour") }}' + - name: BLOCK_NUMBER_MIN + description: '{{ doc("block_id_min") }}' + - name: BLOCK_NUMBER_MAX + description: '{{ doc("block_id_max") }}' + - name: BLOCK_COUNT + description: '{{ doc("block_count") }}' + - name: TRANSACTION_COUNT + description: '{{ doc("transaction_count") }}' + - name: TRANSACTION_COUNT_SUCCESS + description: '{{ doc("transaction_count_success") }}' + - name: TRANSACTION_COUNT_FAILED + description: '{{ doc("transaction_count_failed") }}' + - name: UNIQUE_FROM_COUNT + description: '{{ doc("unique_from_count") }}' + - name: TOTAL_FEES_NATIVE + description: '{{ doc("total_fees_native") }}' + - name: TOTAL_FEES_USD + description: '{{ doc("total_fees_usd") }}' + - name: EZ_CORE_METRICS_HOURLY_ID + description: '{{ doc("pk_id") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' diff --git a/models/silver/crossdb/silver__hourly_prices_coin_gecko.sql b/models/silver/crossdb/silver__hourly_prices_coin_gecko.sql new file mode 100644 index 0000000..3d821c5 --- /dev/null +++ b/models/silver/crossdb/silver__hourly_prices_coin_gecko.sql @@ -0,0 +1,37 @@ +{{ config( + materialized = 'incremental', + unique_key = ['id','recorded_hour'], + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = 'recorded_hour::DATE', + tags = ['noncore'] +) }} + +SELECT + id, + recorded_hour, + OPEN, + high, + low, + CLOSE, + _INSERTED_TIMESTAMP, + hourly_prices_coin_gecko_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ source( + 'crosschain_silver', + 'hourly_prices_coin_gecko' + ) }} +WHERE + id = 'aurora' + +{% if is_incremental() %} +AND modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} \ No newline at end of file diff --git a/models/silver/stats/silver_stats__core_metrics_block_hourly.sql b/models/silver/stats/silver_stats__core_metrics_block_hourly.sql new file mode 100644 index 0000000..e7ac2d1 --- /dev/null +++ b/models/silver/stats/silver_stats__core_metrics_block_hourly.sql @@ -0,0 +1,43 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = "block_timestamp_hour", + cluster_by = ['block_timestamp_hour::DATE'], + tags = ['noncore'] +) }} + +SELECT + DATE_TRUNC( + 'hour', + block_timestamp + ) AS block_timestamp_hour, + MIN(block_id) AS block_number_min, + MAX(block_id) AS block_number_max, + COUNT( + 1 + ) AS block_count, + MAX(_inserted_timestamp) AS _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['block_timestamp_hour'] + ) }} AS core_metrics_block_hourly_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref('silver__blocks') }} +WHERE + block_timestamp_hour < DATE_TRUNC('hour', systimestamp()) + +{% if is_incremental() %} +AND DATE_TRUNC( + 'hour', + _inserted_timestamp +) >= ( + SELECT + MAX(DATE_TRUNC('hour', _inserted_timestamp)) - INTERVAL '12 hours' + FROM + {{ this }} +) +{% endif %} +GROUP BY + 1 \ No newline at end of file diff --git a/models/silver/stats/silver_stats__core_metrics_block_hourly.yml b/models/silver/stats/silver_stats__core_metrics_block_hourly.yml new file mode 100644 index 0000000..8d91718 --- /dev/null +++ b/models/silver/stats/silver_stats__core_metrics_block_hourly.yml @@ -0,0 +1,41 @@ +version: 2 +models: + - name: silver_stats__core_metrics_block_hourly + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_TIMESTAMP_HOUR + columns: + - name: BLOCK_TIMESTAMP_HOUR + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: BLOCK_NUMBER_MIN + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_NUMBER_MAX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_COUNT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: _INSERTED_TIMESTAMP + tests: + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 diff --git a/models/silver/stats/silver_stats__core_metrics_hourly.sql b/models/silver/stats/silver_stats__core_metrics_hourly.sql new file mode 100644 index 0000000..dc52de7 --- /dev/null +++ b/models/silver/stats/silver_stats__core_metrics_hourly.sql @@ -0,0 +1,58 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = "block_timestamp_hour", + cluster_by = ['block_timestamp_hour::DATE'], + tags = ['noncore'] +) }} + +SELECT + DATE_TRUNC( + 'hour', + block_timestamp + ) AS block_timestamp_hour, + COUNT( + DISTINCT tx_hash + ) AS transaction_count, + COUNT( + DISTINCT CASE + WHEN tx_success THEN tx_hash + END + ) AS transaction_count_success, + COUNT( + DISTINCT CASE + WHEN NOT tx_success THEN tx_hash + END + ) AS transaction_count_failed, + COUNT( + DISTINCT from_address + ) AS unique_from_count, + COUNT( + DISTINCT to_address + ) AS unique_to_count, + SUM(tx_fee) AS total_fees, + MAX(_inserted_timestamp) AS _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['block_timestamp_hour'] + ) }} AS core_metrics_hourly_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref('silver__transactions') }} +WHERE + block_timestamp_hour < DATE_TRUNC('hour', systimestamp()) + +{% if is_incremental() %} +AND DATE_TRUNC( + 'hour', + _inserted_timestamp +) >= ( + SELECT + MAX(DATE_TRUNC('hour', _inserted_timestamp)) - INTERVAL '12 hours' + FROM + {{ this }} +) +{% endif %} +GROUP BY + 1 \ No newline at end of file diff --git a/models/silver/stats/silver_stats__core_metrics_hourly.yml b/models/silver/stats/silver_stats__core_metrics_hourly.yml new file mode 100644 index 0000000..68d716c --- /dev/null +++ b/models/silver/stats/silver_stats__core_metrics_hourly.yml @@ -0,0 +1,56 @@ +version: 2 +models: + - name: silver_stats__core_metrics_hourly + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_TIMESTAMP_HOUR + columns: + - name: BLOCK_TIMESTAMP_HOUR + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TRANSACTION_COUNT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TRANSACTION_COUNT_SUCCESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TRANSACTION_COUNT_FAILED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: UNIQUE_FROM_COUNT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TOTAL_FEES + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - DECIMAL + - FLOAT + - NUMBER + - name: _INSERTED_TIMESTAMP + tests: + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 diff --git a/models/silver/test/traces/test_silver__traces_full.yml b/models/silver/test/traces/test_silver__traces_full.yml index b925c16..0774d8e 100644 --- a/models/silver/test/traces/test_silver__traces_full.yml +++ b/models/silver/test/traces/test_silver__traces_full.yml @@ -46,9 +46,6 @@ models: - name: IDENTIFIER tests: - not_null - - name: ETH_VALUE - tests: - - not_null - name: GAS tests: - not_null diff --git a/models/sources.yml b/models/sources.yml index fa42ca7..4b3f725 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -25,6 +25,7 @@ sources: schema: silver tables: - name: number_sequence + - name: hourly_prices_coin_gecko - name: github_actions database: aurora From d48215a72305e1993b3ed15b48a285dbea573baa Mon Sep 17 00:00:00 2001 From: WHYTEWYLL <49686519+WHYTEWYLL@users.noreply.github.com> Date: Thu, 22 Feb 2024 16:44:35 +0100 Subject: [PATCH 39/46] job name --- .github/workflows/dbt_run_non_core.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dbt_run_non_core.yml b/.github/workflows/dbt_run_non_core.yml index 1fe0e1d..3a5259f 100644 --- a/.github/workflows/dbt_run_non_core.yml +++ b/.github/workflows/dbt_run_non_core.yml @@ -1,5 +1,5 @@ -name: dbt_run_observability -run-name: dbt_run_observability +name: dbt_run_non_core +run-name: dbt_run_non_core on: workflow_dispatch: From 2de980d5af72bef3c028b6b78530001440b24ecb Mon Sep 17 00:00:00 2001 From: WHYTEWYLL <49686519+WHYTEWYLL@users.noreply.github.com> Date: Thu, 22 Feb 2024 16:53:35 +0100 Subject: [PATCH 40/46] del noncore --- .github/workflows/dbt_run_non_core.yml | 45 ------------------- .../stats/stats__ez_core_metrics_hourly.sql | 2 +- .../silver__hourly_prices_coin_gecko.sql | 2 +- ...ilver_stats__core_metrics_block_hourly.sql | 2 +- .../silver_stats__core_metrics_hourly.sql | 2 +- 5 files changed, 4 insertions(+), 49 deletions(-) delete mode 100644 .github/workflows/dbt_run_non_core.yml diff --git a/.github/workflows/dbt_run_non_core.yml b/.github/workflows/dbt_run_non_core.yml deleted file mode 100644 index 3a5259f..0000000 --- a/.github/workflows/dbt_run_non_core.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: dbt_run_non_core -run-name: dbt_run_non_core - -on: - workflow_dispatch: - schedule: - # Runs "1 times per day" (see https://crontab.guru) - - cron: "0 6 * * *" - -env: - USE_VARS: "${{ vars.USE_VARS }}" - DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - DBT_VERSION: "${{ vars.DBT_VERSION }}" - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: install dependencies - run: | - pip3 install dbt-snowflake==${{ env.DBT_VERSION }} cli_passthrough requests click - dbt deps - - name: Run DBT Jobs - run: | - dbt run --threads 2 --vars '{"OBSERV_FULL_TEST":False}' -m tag:noncore diff --git a/models/gold/stats/stats__ez_core_metrics_hourly.sql b/models/gold/stats/stats__ez_core_metrics_hourly.sql index c6934d3..467443e 100644 --- a/models/gold/stats/stats__ez_core_metrics_hourly.sql +++ b/models/gold/stats/stats__ez_core_metrics_hourly.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'view', - tags = ['noncore'], + tags = ['core'], meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'STATS, METRICS, CORE, HOURLY', }} } ) }} diff --git a/models/silver/crossdb/silver__hourly_prices_coin_gecko.sql b/models/silver/crossdb/silver__hourly_prices_coin_gecko.sql index 3d821c5..3de8790 100644 --- a/models/silver/crossdb/silver__hourly_prices_coin_gecko.sql +++ b/models/silver/crossdb/silver__hourly_prices_coin_gecko.sql @@ -4,7 +4,7 @@ incremental_strategy = 'merge', merge_exclude_columns = ["inserted_timestamp"], cluster_by = 'recorded_hour::DATE', - tags = ['noncore'] + tags = ['core'] ) }} SELECT diff --git a/models/silver/stats/silver_stats__core_metrics_block_hourly.sql b/models/silver/stats/silver_stats__core_metrics_block_hourly.sql index e7ac2d1..6fdecbf 100644 --- a/models/silver/stats/silver_stats__core_metrics_block_hourly.sql +++ b/models/silver/stats/silver_stats__core_metrics_block_hourly.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = "block_timestamp_hour", cluster_by = ['block_timestamp_hour::DATE'], - tags = ['noncore'] + tags = ['core'] ) }} SELECT diff --git a/models/silver/stats/silver_stats__core_metrics_hourly.sql b/models/silver/stats/silver_stats__core_metrics_hourly.sql index dc52de7..d63f209 100644 --- a/models/silver/stats/silver_stats__core_metrics_hourly.sql +++ b/models/silver/stats/silver_stats__core_metrics_hourly.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = "block_timestamp_hour", cluster_by = ['block_timestamp_hour::DATE'], - tags = ['noncore'] + tags = ['core'] ) }} SELECT From 1eeb6c363f3dbbadcd201685ff2246a6d0800b66 Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Thu, 22 Feb 2024 14:27:42 -0500 Subject: [PATCH 41/46] block_id to block_number and replace ref to sei --- models/gold/stats/stats__ez_core_metrics_hourly.sql | 4 ++-- .../stats/silver_stats__core_metrics_block_hourly.sql | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/models/gold/stats/stats__ez_core_metrics_hourly.sql b/models/gold/stats/stats__ez_core_metrics_hourly.sql index 467443e..13c0978 100644 --- a/models/gold/stats/stats__ez_core_metrics_hourly.sql +++ b/models/gold/stats/stats__ez_core_metrics_hourly.sql @@ -29,7 +29,7 @@ WITH txs AS ( LEFT JOIN {{ ref('silver__hourly_prices_coin_gecko') }} p ON s.block_timestamp_hour = p.recorded_hour - AND p.id = 'sei-network' + AND p.id = 'aurora' ) SELECT A.block_timestamp_hour, @@ -57,4 +57,4 @@ SELECT FROM {{ ref('silver_stats__core_metrics_block_hourly') }} A JOIN txs b - ON A.block_timestamp_hour = b.block_timestamp_hour \ No newline at end of file + ON A.block_timestamp_hour = b.block_timestamp_hour diff --git a/models/silver/stats/silver_stats__core_metrics_block_hourly.sql b/models/silver/stats/silver_stats__core_metrics_block_hourly.sql index 6fdecbf..e868bb0 100644 --- a/models/silver/stats/silver_stats__core_metrics_block_hourly.sql +++ b/models/silver/stats/silver_stats__core_metrics_block_hourly.sql @@ -11,8 +11,12 @@ SELECT 'hour', block_timestamp ) AS block_timestamp_hour, - MIN(block_id) AS block_number_min, - MAX(block_id) AS block_number_max, + MIN( + block_number + ) AS block_number_min, + MAX( + block_number + ) AS block_number_max, COUNT( 1 ) AS block_count, @@ -40,4 +44,4 @@ AND DATE_TRUNC( ) {% endif %} GROUP BY - 1 \ No newline at end of file + 1 From 9b8940fd7ac0b3468c8561cb1e85d9d77fef3ea5 Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Fri, 1 Mar 2024 13:12:27 -0500 Subject: [PATCH 42/46] fix stats inc logic --- ...ilver_stats__core_metrics_block_hourly.sql | 28 ++++++++++++----- .../silver_stats__core_metrics_hourly.sql | 30 ++++++++++++++----- 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/models/silver/stats/silver_stats__core_metrics_block_hourly.sql b/models/silver/stats/silver_stats__core_metrics_block_hourly.sql index e868bb0..3e1656a 100644 --- a/models/silver/stats/silver_stats__core_metrics_block_hourly.sql +++ b/models/silver/stats/silver_stats__core_metrics_block_hourly.sql @@ -5,7 +5,26 @@ cluster_by = ['block_timestamp_hour::DATE'], tags = ['core'] ) }} +/* run incremental timestamp value first then use it as a static value */ +{% if execute %} +{% if is_incremental() %} +{% set query %} + +SELECT + MIN(DATE_TRUNC('hour', block_timestamp)) block_timestamp_hour +FROM + {{ ref('silver__blocks') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) + FROM + {{ this }} + ) {% endset %} + {% set min_block_timestamp_hour = run_query(query).columns [0].values() [0] %} +{% endif %} +{% endif %} SELECT DATE_TRUNC( 'hour', @@ -35,13 +54,8 @@ WHERE {% if is_incremental() %} AND DATE_TRUNC( 'hour', - _inserted_timestamp -) >= ( - SELECT - MAX(DATE_TRUNC('hour', _inserted_timestamp)) - INTERVAL '12 hours' - FROM - {{ this }} -) + block_timestamp +) >= '{{ min_block_timestamp_hour }}' {% endif %} GROUP BY 1 diff --git a/models/silver/stats/silver_stats__core_metrics_hourly.sql b/models/silver/stats/silver_stats__core_metrics_hourly.sql index d63f209..0df7b02 100644 --- a/models/silver/stats/silver_stats__core_metrics_hourly.sql +++ b/models/silver/stats/silver_stats__core_metrics_hourly.sql @@ -5,7 +5,26 @@ cluster_by = ['block_timestamp_hour::DATE'], tags = ['core'] ) }} +/* run incremental timestamp value first then use it as a static value */ +{% if execute %} +{% if is_incremental() %} +{% set query %} + +SELECT + MIN(DATE_TRUNC('hour', block_timestamp)) block_timestamp_hour +FROM + {{ ref('silver__transactions') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) + FROM + {{ this }} + ) {% endset %} + {% set min_block_timestamp_hour = run_query(query).columns [0].values() [0] %} +{% endif %} +{% endif %} SELECT DATE_TRUNC( 'hour', @@ -46,13 +65,8 @@ WHERE {% if is_incremental() %} AND DATE_TRUNC( 'hour', - _inserted_timestamp -) >= ( - SELECT - MAX(DATE_TRUNC('hour', _inserted_timestamp)) - INTERVAL '12 hours' - FROM - {{ this }} -) + block_timestamp +) >= '{{ min_block_timestamp_hour }}' {% endif %} GROUP BY - 1 \ No newline at end of file + 1 From 6975d82f21ac1b4d7e5f3363424d94ea62f1dc42 Mon Sep 17 00:00:00 2001 From: WHYTEWYLL <49686519+WHYTEWYLL@users.noreply.github.com> Date: Wed, 13 Mar 2024 13:58:32 -0600 Subject: [PATCH 43/46] fix workflow and requ --- .github/workflows/dbt_alter_gha_task.yml | 35 +++++++++++++++++-- .github/workflows/dbt_run_atlas.yml | 35 ++++++++++++++----- .github/workflows/dbt_run_dev_refresh.yml | 35 ++++++++++++++----- .github/workflows/dbt_run_scheduled.yml | 35 ++++++++++++++----- .../workflows/dbt_run_streamline_history.yml | 35 ++++++++++++++----- ...treamline_realtime_blocks_transactions.yml | 35 ++++++++++++++----- ...bt_run_streamline_realtime_tx_receipts.yml | 35 ++++++++++++++----- .../dbt_run_streamline_traces_history.yml | 35 ++++++++++++++----- requirements.txt | 1 + 9 files changed, 222 insertions(+), 59 deletions(-) diff --git a/.github/workflows/dbt_alter_gha_task.yml b/.github/workflows/dbt_alter_gha_task.yml index d3094a6..d182e8a 100644 --- a/.github/workflows/dbt_alter_gha_task.yml +++ b/.github/workflows/dbt_alter_gha_task.yml @@ -34,13 +34,42 @@ env: concurrency: group: ${{ github.workflow }} +jobs: + dbt: + runs-on: ubuntu-latest + environment: + name: | + ${{ inputs.workflow_name }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Jobs + run: | + ${{ inputs.task_action }} + + - name: Store logs + uses: actions/upload-artifact@v3 + with: + name: dbt-logs + path: | + logs + target + jobs: called_workflow_template: uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@AN-4374/upgrade-dbt-1.7 with: workflow_name: | - ${{ inputs.workflow_name }} - task_action: | - ${{ inputs.task_action }} + + task_action: environment: workflow_prod secrets: inherit diff --git a/.github/workflows/dbt_run_atlas.yml b/.github/workflows/dbt_run_atlas.yml index cbbc3f2..249f03f 100644 --- a/.github/workflows/dbt_run_atlas.yml +++ b/.github/workflows/dbt_run_atlas.yml @@ -24,11 +24,30 @@ concurrency: group: ${{ github.workflow }} jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 - with: - dbt_command: > - dbt run -s tag:atlas - environment: workflow_prod - warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit + dbt: + runs-on: ubuntu-latest + environment: + name: workflow_prod + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Jobs + run: | + dbt run -s tag:atlas + + - name: Store logs + uses: actions/upload-artifact@v3 + with: + name: dbt-logs + path: | + logs + target diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml index cce0cc9..77ad10c 100644 --- a/.github/workflows/dbt_run_dev_refresh.yml +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -25,11 +25,30 @@ concurrency: group: ${{ github.workflow }} jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 - with: - dbt_command: > - dbt run-operation run_sp_create_prod_clone - environment: workflow_prod - warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit + dbt: + runs-on: ubuntu-latest + environment: + name: workflow_prod + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Jobs + run: | + dbt run-operation run_sp_create_prod_clone + + - name: Store logs + uses: actions/upload-artifact@v3 + with: + name: dbt-logs + path: | + logs + target diff --git a/.github/workflows/dbt_run_scheduled.yml b/.github/workflows/dbt_run_scheduled.yml index 79393c9..3361481 100644 --- a/.github/workflows/dbt_run_scheduled.yml +++ b/.github/workflows/dbt_run_scheduled.yml @@ -24,11 +24,30 @@ concurrency: group: ${{ github.workflow }} jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 - with: - dbt_command: > - dbt run -s tag:core - environment: workflow_prod - warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit + dbt: + runs-on: ubuntu-latest + environment: + name: workflow_prod + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Jobs + run: | + dbt run -s tag:core + + - name: Store logs + uses: actions/upload-artifact@v3 + with: + name: dbt-logs + path: | + logs + target diff --git a/.github/workflows/dbt_run_streamline_history.yml b/.github/workflows/dbt_run_streamline_history.yml index d42c955..5a3aefd 100644 --- a/.github/workflows/dbt_run_streamline_history.yml +++ b/.github/workflows/dbt_run_streamline_history.yml @@ -21,11 +21,30 @@ concurrency: group: ${{ github.workflow }} jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 - with: - dbt_command: > - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_RUN_HISTORY":True}' -m 1+models/silver/streamline/realtime/ --full-refresh - environment: workflow_prod - warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit + dbt: + runs-on: ubuntu-latest + environment: + name: workflow_prod + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Jobs + run: | + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_RUN_HISTORY":True}' -m 1+models/silver/streamline/realtime/ --full-refresh + + - name: Store logs + uses: actions/upload-artifact@v3 + with: + name: dbt-logs + path: | + logs + target diff --git a/.github/workflows/dbt_run_streamline_realtime_blocks_transactions.yml b/.github/workflows/dbt_run_streamline_realtime_blocks_transactions.yml index 3c77a92..7fa2e92 100644 --- a/.github/workflows/dbt_run_streamline_realtime_blocks_transactions.yml +++ b/.github/workflows/dbt_run_streamline_realtime_blocks_transactions.yml @@ -24,11 +24,30 @@ concurrency: group: ${{ github.workflow }} jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 - with: - dbt_command: > - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/realtime/streamline__blocks_realtime.sql 1+models/silver/streamline/realtime/streamline__transactions_realtime.sql - environment: workflow_prod - warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit + dbt: + runs-on: ubuntu-latest + environment: + name: workflow_prod + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Jobs + run: | + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/realtime/streamline__blocks_realtime.sql 1+models/silver/streamline/realtime/streamline__transactions_realtime.sql + + - name: Store logs + uses: actions/upload-artifact@v3 + with: + name: dbt-logs + path: | + logs + target diff --git a/.github/workflows/dbt_run_streamline_realtime_tx_receipts.yml b/.github/workflows/dbt_run_streamline_realtime_tx_receipts.yml index 8d360ee..552f037 100644 --- a/.github/workflows/dbt_run_streamline_realtime_tx_receipts.yml +++ b/.github/workflows/dbt_run_streamline_realtime_tx_receipts.yml @@ -24,11 +24,30 @@ concurrency: group: ${{ github.workflow }} jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 - with: - dbt_command: > - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/realtime/streamline__tx_receipts_realtime.sql - environment: workflow_prod - warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit + dbt: + runs-on: ubuntu-latest + environment: + name: workflow_prod + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Jobs + run: | + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/realtime/streamline__tx_receipts_realtime.sql + + - name: Store logs + uses: actions/upload-artifact@v3 + with: + name: dbt-logs + path: | + logs + target diff --git a/.github/workflows/dbt_run_streamline_traces_history.yml b/.github/workflows/dbt_run_streamline_traces_history.yml index d42c2ee..e4b7025 100644 --- a/.github/workflows/dbt_run_streamline_traces_history.yml +++ b/.github/workflows/dbt_run_streamline_traces_history.yml @@ -24,11 +24,30 @@ concurrency: group: ${{ github.workflow }} jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 - with: - dbt_command: > - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/history/streamline__traces_history.sql - environment: workflow_prod - warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit + dbt: + runs-on: ubuntu-latest + environment: + name: workflow_prod + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Jobs + run: | + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/history/streamline__traces_history.sql + + - name: Store logs + uses: actions/upload-artifact@v3 + with: + name: dbt-logs + path: | + logs + target diff --git a/requirements.txt b/requirements.txt index c61d3e1..39b82bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ dbt-snowflake>=1.7,<1.8 +protobuf==4.25.3 \ No newline at end of file From 42fe4b6113422ef487b7419dd85c90d05f607a5a Mon Sep 17 00:00:00 2001 From: WHYTEWYLL <49686519+WHYTEWYLL@users.noreply.github.com> Date: Wed, 13 Mar 2024 14:01:22 -0600 Subject: [PATCH 44/46] error --- .github/workflows/dbt_alter_gha_task.yml | 35 ++---------------------- 1 file changed, 3 insertions(+), 32 deletions(-) diff --git a/.github/workflows/dbt_alter_gha_task.yml b/.github/workflows/dbt_alter_gha_task.yml index d182e8a..d3094a6 100644 --- a/.github/workflows/dbt_alter_gha_task.yml +++ b/.github/workflows/dbt_alter_gha_task.yml @@ -34,42 +34,13 @@ env: concurrency: group: ${{ github.workflow }} -jobs: - dbt: - runs-on: ubuntu-latest - environment: - name: | - ${{ inputs.workflow_name }} - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Run DBT Jobs - run: | - ${{ inputs.task_action }} - - - name: Store logs - uses: actions/upload-artifact@v3 - with: - name: dbt-logs - path: | - logs - target - jobs: called_workflow_template: uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@AN-4374/upgrade-dbt-1.7 with: workflow_name: | - - task_action: + ${{ inputs.workflow_name }} + task_action: | + ${{ inputs.task_action }} environment: workflow_prod secrets: inherit From 993b2c4b2df21e0932278ed724f75d6e3ccc4108 Mon Sep 17 00:00:00 2001 From: WHYTEWYLL <49686519+WHYTEWYLL@users.noreply.github.com> Date: Wed, 13 Mar 2024 14:18:28 -0600 Subject: [PATCH 45/46] docs --- .github/workflows/dbt_docs_update.yml | 51 +++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml index b6b463d..0dda193 100644 --- a/.github/workflows/dbt_docs_update.yml +++ b/.github/workflows/dbt_docs_update.yml @@ -22,6 +22,51 @@ concurrency: group: ${{ github.workflow }} jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_docs_updates.yml@AN-4374/upgrade-dbt-1.7 - secrets: inherit + dbt: + runs-on: ubuntu-latest + environment: + name: workflow_prod + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: refresh ddl for datashare + run: | + cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi; + + - name: checkout docs branch + run: | + git checkout -B docs origin/main + + - name: generate dbt docs + run: dbt docs generate -t prod + + - name: move files to docs directory + run: | + mkdir -p ./docs + cp target/{catalog.json,manifest.json,index.html} docs/ + - name: clean up target directory + run: dbt clean + + - name: check for changes + run: git status + + - name: stage changed files + run: git add . + + - name: commit changed files + run: | + git config user.email "abc@xyz" + git config user.name "github-actions" + git commit -am "Auto-update docs" + - name: push changes to docs + run: | + git push -f --set-upstream origin docs From f75b88601868630fdedabab41cbc1f265acce43d Mon Sep 17 00:00:00 2001 From: WHYTEWYLL <49686519+WHYTEWYLL@users.noreply.github.com> Date: Wed, 13 Mar 2024 14:22:16 -0600 Subject: [PATCH 46/46] last one --- .github/workflows/dbt_alter_gha_task.yml | 64 ++++++++++++++---------- .github/workflows/dbt_run_deployment.yml | 39 ++++++++++++--- .github/workflows/dbt_test_tasks.yml | 39 ++++++++++----- 3 files changed, 95 insertions(+), 47 deletions(-) diff --git a/.github/workflows/dbt_alter_gha_task.yml b/.github/workflows/dbt_alter_gha_task.yml index d3094a6..4635655 100644 --- a/.github/workflows/dbt_alter_gha_task.yml +++ b/.github/workflows/dbt_alter_gha_task.yml @@ -1,46 +1,56 @@ -name: dbt_alter_gha_task -run-name: dbt_alter_gha_task +name: dbt_alter_gha_tasks on: - workflow_dispatch: - branches: - - "main" + workflow_call: inputs: workflow_name: type: string - description: Name of the workflow to perform the action on, no .yml extension + description: "Name of the workflow to perform the action on, no .yml extension" required: true task_action: - type: choice - description: Action to perform + type: string + description: "Action to perform" + required: true + environment: + type: string + description: "Environment to run the workflow in" required: true - options: - - SUSPEND - - RESUME - default: SUSPEND env: - DBT_PROFILES_DIR: ./ - + USE_VARS: "${{ vars.USE_VARS }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" ACCOUNT: "${{ vars.ACCOUNT }}" ROLE: "${{ vars.ROLE }}" USER: "${{ vars.USER }}" PASSWORD: "${{ secrets.PASSWORD }}" REGION: "${{ vars.REGION }}" DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} + WAREHOUSE: "${{ vars.WAREHOUSE }}" jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@AN-4374/upgrade-dbt-1.7 - with: - workflow_name: | - ${{ inputs.workflow_name }} - task_action: | - ${{ inputs.task_action }} - environment: workflow_prod - secrets: inherit + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: ${{ inputs.environment }} + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip3 install -r requirements.txt + dbt deps + + - name: Set up workflow name + run: echo "WORKFLOW_NAME_UPPER=$(echo '${{ inputs.workflow_name }}' | tr '[:lower:]' '[:upper:]')" >> $GITHUB_ENV + + - name: Run DBT Jobs + run: | + dbt run-operation fsc_utils.alter_gha_task --args '{ "task_name": "TRIGGER_${{ env.WORKFLOW_NAME_UPPER }}", "task_action": "${{ inputs.task_action }}" }' diff --git a/.github/workflows/dbt_run_deployment.yml b/.github/workflows/dbt_run_deployment.yml index dc5a8d5..6801df5 100644 --- a/.github/workflows/dbt_run_deployment.yml +++ b/.github/workflows/dbt_run_deployment.yml @@ -37,10 +37,35 @@ concurrency: group: ${{ github.workflow }} jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_deployment_template.yml@AN-4374/upgrade-dbt-1.7 - with: - dbt_command: | - ${{ inputs.dbt_command }} - warehouse: ${{ inputs.WAREHOUSE }} - secrets: inherit + dbt: + runs-on: ubuntu-latest + environment: + name: workflow_prod + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Jobs + run: | + ${{ inputs.dbt_command }} + + - name: Run datashare model + run: | + cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi; + dbt run-operation run_query --args "{sql: call admin.datashare.sp_grant_share_permissions('${{ env.DATABASE }}')}" + + - name: Store logs + uses: actions/upload-artifact@v3 + with: + name: dbt-logs + path: | + logs + target diff --git a/.github/workflows/dbt_test_tasks.yml b/.github/workflows/dbt_test_tasks.yml index 594148b..e56b17c 100644 --- a/.github/workflows/dbt_test_tasks.yml +++ b/.github/workflows/dbt_test_tasks.yml @@ -1,27 +1,40 @@ name: dbt_test_tasks -run-name: dbt_test_tasks on: - workflow_dispatch: - branches: - - "main" + workflow_call: env: - DBT_PROFILES_DIR: ./ - + USE_VARS: "${{ vars.USE_VARS }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" ACCOUNT: "${{ vars.ACCOUNT }}" ROLE: "${{ vars.ROLE }}" USER: "${{ vars.USER }}" PASSWORD: "${{ secrets.PASSWORD }}" REGION: "${{ vars.REGION }}" DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} + WAREHOUSE: "${{ vars.WAREHOUSE }}" jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@AN-4374/upgrade-dbt-1.7 - secrets: inherit + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Jobs + run: | + dbt test -m models/github_actions/github_actions__current_task_status.sql