diff --git a/.github/workflows/dbt_run_hourly.yml b/.github/workflows/dbt_run_daily.yml similarity index 87% rename from .github/workflows/dbt_run_hourly.yml rename to .github/workflows/dbt_run_daily.yml index b77bcde..51b1b75 100644 --- a/.github/workflows/dbt_run_hourly.yml +++ b/.github/workflows/dbt_run_daily.yml @@ -4,10 +4,11 @@ run-name: dbt_run_scheduled on: workflow_dispatch: schedule: - # Runs "every hour" (see https://crontab.guru) - - cron: '0 */1 * * *' + # Runs 0700 daily (see https://crontab.guru) + - cron: '0 7 * * *' env: + USE_VARS: "${{ vars.USE_VARS }}" DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" DBT_VERSION: "${{ vars.DBT_VERSION }}" ACCOUNT: "${{ vars.ACCOUNT }}" diff --git a/.github/workflows/dbt_run_daily_dev_refresh.yml b/.github/workflows/dbt_run_daily_dev_refresh.yml index c8d1239..cb65f37 100644 --- a/.github/workflows/dbt_run_daily_dev_refresh.yml +++ b/.github/workflows/dbt_run_daily_dev_refresh.yml @@ -8,6 +8,7 @@ on: - cron: '0 8 * * *' env: + USE_VARS: "${{ vars.USE_VARS }}" DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" DBT_VERSION: "${{ vars.DBT_VERSION }}" ACCOUNT: "${{ vars.ACCOUNT }}" diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml index 6b51e35..4bbbda3 100644 --- a/.github/workflows/dbt_test.yml +++ b/.github/workflows/dbt_test.yml @@ -4,9 +4,10 @@ run-name: dbt_test_scheduled on: workflow_dispatch: schedule: - # Run at 1600 UTC daily (see https://crontab.guru) - - cron: '0 16 * * *' + # Run at 0900 UTC daily (see https://crontab.guru) + - cron: '0 9 * * *' env: + USE_VARS: "${{ vars.USE_VARS }}" DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" DBT_VERSION: "${{ vars.DBT_VERSION }}" ACCOUNT: "${{ vars.ACCOUNT }}" diff --git a/.github/workflows/dbt_test_workflow.yml b/.github/workflows/dbt_test_workflow.yml deleted file mode 100644 index 0b46d48..0000000 --- a/.github/workflows/dbt_test_workflow.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: dbt_test_workflow -run-name: dbt_test_workflow - -on: - workflow_dispatch: - -env: - DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - DBT_VERSION: "${{ vars.DBT_VERSION }}" - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@alter_workflow_var - with: - dbt_command: > - dbt seed; dbt run - environment: workflow_prod - warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit diff --git a/macros/run_sp_create_prod_clone.sql b/macros/run_sp_create_prod_clone.sql index 97786ed..3db1ccc 100644 --- a/macros/run_sp_create_prod_clone.sql +++ b/macros/run_sp_create_prod_clone.sql @@ -3,7 +3,7 @@ call terra._internal.create_prod_clone( 'terra', 'terra_dev', - 'dbt_cloud' + 'internal_dev' ); {% endset %} {% do run_query(clone_query) %} diff --git a/models/core/core__ez_staking.yml b/models/core/core__ez_staking.yml index 88ac1b2..4ad0355 100644 --- a/models/core/core__ez_staking.yml +++ b/models/core/core__ez_staking.yml @@ -9,6 +9,7 @@ models: - ACTION - MESSAGE_INDEX - DELEGATOR_ADDRESS + - VALIDATOR_ADDRESS columns: - name: ACTION description: "{{ doc('action') }}" diff --git a/models/core/core__fact_blocks.yml b/models/core/core__fact_blocks.yml index 794d9f9..ea0cf2e 100644 --- a/models/core/core__fact_blocks.yml +++ b/models/core/core__fact_blocks.yml @@ -10,7 +10,7 @@ models: - block_id - sequence_gaps: column_name: block_id - where: BLOCK_TIMESTAMP < CURRENT_DATE + where: BLOCK_TIMESTAMP < CURRENT_DATE and BLOCK_ID > 4380273 columns: - name: BLOCK_ID @@ -161,7 +161,8 @@ models: - name: VALIDATOR_ADDRESS_ARRAY description: "{{ doc('validator_address_array') }}" tests: - - not_null + - not_null: + where: BLOCK_TIMESTAMP < CURRENT_DATE and BLOCK_ID > 4380273 - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - ARRAY diff --git a/models/silver/silver__blocks.sql b/models/silver/silver__blocks.sql index c282cb0..8a614ef 100644 --- a/models/silver/silver__blocks.sql +++ b/models/silver/silver__blocks.sql @@ -21,40 +21,33 @@ WITH base_blocks AS ( FROM {{ ref('bronze__blocks') }} WHERE - {{ incremental_load_filter('_inserted_timestamp') }} + {{ incremental_last_x_days( + '_inserted_timestamp', + 3 + ) }} qualify ROW_NUMBER() over ( PARTITION BY block_id ORDER BY _inserted_timestamp DESC ) = 1 ), -validator_signatures AS ( - SELECT - header :last_commit :height AS block_id, - header :last_commit :signatures AS signatures - FROM - base_blocks -), -validator_addresses AS ( - SELECT - validator_signatures.block_id AS block_id, - s0.value :validator_address AS validator_address - FROM - validator_signatures, - LATERAL FLATTEN( - input => validator_signatures.signatures - ) AS s0 -), validators_address_array AS ( SELECT - CAST(validator_addresses.block_id AS NUMBER(38, 0)) AS block_id, - ARRAY_AGG( - DISTINCT validator_addresses.validator_address - ) AS address_array + block_id, + validator_address_array, + _inserted_timestamp FROM - validator_addresses - GROUP BY - validator_addresses.block_id + {{ ref('silver__blocks_val_array') }} + WHERE + {{ incremental_last_x_days( + '_inserted_timestamp', + 3 + ) }} + qualify ROW_NUMBER() over ( + PARTITION BY block_id + ORDER BY + _inserted_timestamp DESC + ) = 1 ), FINAL AS ( SELECT @@ -77,7 +70,7 @@ FINAL AS ( base_blocks.header :validators_hash :: STRING AS validators_hash, base_blocks._ingested_at AS _ingested_at, base_blocks._inserted_timestamp AS _inserted_timestamp, - validators_address_array.address_array :: ARRAY AS validator_address_array + validators_address_array.validator_address_array :: ARRAY AS validator_address_array FROM base_blocks LEFT JOIN validators_address_array diff --git a/models/silver/silver__blocks_val_array.sql b/models/silver/silver__blocks_val_array.sql new file mode 100644 index 0000000..ac0f0de --- /dev/null +++ b/models/silver/silver__blocks_val_array.sql @@ -0,0 +1,61 @@ +{{ config( + materialized = 'incremental', + unique_key = 'block_id', + incremental_strategy = 'delete+insert', + cluster_by = ['_inserted_timestamp::DATE'], +) }} + +WITH base_blocks AS ( + + SELECT + header, + _inserted_timestamp + FROM + {{ ref('bronze__blocks') }} + WHERE + {{ incremental_last_x_days( + '_inserted_timestamp', + 3 + ) }} + qualify ROW_NUMBER() over ( + PARTITION BY block_id + ORDER BY + _inserted_timestamp DESC + ) = 1 +), +validator_signatures AS ( + SELECT + header :last_commit :height AS block_id, + header :last_commit :signatures AS signatures, + _inserted_timestamp + FROM + base_blocks +), +validator_addresses AS ( + SELECT + validator_signatures.block_id AS block_id, + s0.value :validator_address AS validator_address, + _inserted_timestamp + FROM + validator_signatures, + LATERAL FLATTEN( + input => validator_signatures.signatures + ) AS s0 +), +validators_address_array AS ( + SELECT + CAST(validator_addresses.block_id AS NUMBER(38, 0)) AS block_id, + ARRAY_AGG( + DISTINCT validator_addresses.validator_address + ) AS validator_address_array, + _inserted_timestamp + FROM + validator_addresses + GROUP BY + validator_addresses.block_id, + _inserted_timestamp +) +SELECT + * +FROM + validators_address_array diff --git a/models/silver/silver__blocks_val_array.yml b/models/silver/silver__blocks_val_array.yml new file mode 100644 index 0000000..5a50989 --- /dev/null +++ b/models/silver/silver__blocks_val_array.yml @@ -0,0 +1,38 @@ +version: 2 + +models: + - name: silver__blocks_val_array + description: |- + This table records all the blocks of Terra2 blockchain. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - block_id + columns: + - name: BLOCK_ID + description: "{{ doc('block_id')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + + + - name: VALIDATOR_ADDRESS_ARRAY + description: "{{ doc('validator_address_array') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - ARRAY + - VARIANT + - OBJECT + + - name: _INSERTED_TIMESTAMP + description: "{{ doc('_inserted_timestamp')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ