From 0a666f9beb939d9d71e0dfae1c28490532e11b8d Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Tue, 25 Apr 2023 10:42:48 -0400 Subject: [PATCH 01/12] initial updates --- README.md | 8 ++++---- dbt_project.yml | 4 ++-- macros/create_sps.sql | 2 +- macros/run_sp_create_prod_clone.sql | 2 +- macros/tags/add_database_or_schema_tags.sql | 2 +- models/descriptions/__overview__.md | 13 ++++++------- models/sources.yml | 2 +- profiles.yml | 2 +- 8 files changed, 17 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 1528017..ac8302a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -# Please find and replace all instances of `xyz` with your project name. +# Please find and replace all instances of `evmos` with your project name. ## Profile Set Up @@ -7,7 +7,7 @@ ---- ```yml -xyz: +evmos: target: dev outputs: dev: @@ -17,7 +17,7 @@ xyz: user: password: region: - database: xyz_DEV + database: evmos_DEV warehouse: schema: silver threads: 4 @@ -73,5 +73,5 @@ dbt run --var '{"UPDATE_SNOWFLAKE_TAGS":True}' -s models/core/core__fact_swaps.s ``` select * -from table(xyz.information_schema.tag_references('xyz.core.fact_blocks', 'table')); +from table(evmos.information_schema.tag_references('evmos.core.fact_blocks', 'table')); ``` \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 3a5b5f4..0f63641 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,14 +1,14 @@ # Name your project! Project names should contain only lowercase characters # and underscores. A good package name should reflect your organization's # name or the intended use of these models -name: "xyz_models" +name: "evmos_models" version: "1.0.0" config-version: 2 require-dbt-version: ">=1.4.0" # This setting configures which "profile" dbt uses for this project. -profile: "xyz" +profile: "evmos" # These configurations specify where dbt should look for different types of files. # The `model-paths` config, for example, states that models in this project can be diff --git a/macros/create_sps.sql b/macros/create_sps.sql index 8577716..b077b9e 100644 --- a/macros/create_sps.sql +++ b/macros/create_sps.sql @@ -1,5 +1,5 @@ {% macro create_sps() %} - {% if target.database == 'xyz' %} + {% if target.database == 'evmos' %} CREATE SCHEMA IF NOT EXISTS _internal; {{ sp_create_prod_clone('_internal') }}; {% endif %} diff --git a/macros/run_sp_create_prod_clone.sql b/macros/run_sp_create_prod_clone.sql index 1868683..4074835 100644 --- a/macros/run_sp_create_prod_clone.sql +++ b/macros/run_sp_create_prod_clone.sql @@ -1,6 +1,6 @@ {% macro run_sp_create_prod_clone() %} {% set clone_query %} -call xyz._internal.create_prod_clone('xyz', 'xyz_dev', 'internal_dev'); +call evmos._internal.create_prod_clone('evmos', 'evmos_dev', 'internal_dev'); {% endset %} {% do run_query(clone_query) %} diff --git a/macros/tags/add_database_or_schema_tags.sql b/macros/tags/add_database_or_schema_tags.sql index 184e0bd..cf8abf3 100644 --- a/macros/tags/add_database_or_schema_tags.sql +++ b/macros/tags/add_database_or_schema_tags.sql @@ -1,7 +1,7 @@ {% macro add_database_or_schema_tags() %} {{ set_database_tag_value( 'BLOCKCHAIN_NAME', - 'xyz' + 'EVMOS' ) }} {{ set_database_tag_value( 'BLOCKCHAIN_TYPE', diff --git a/models/descriptions/__overview__.md b/models/descriptions/__overview__.md index 588ffbd..b3e273a 100644 --- a/models/descriptions/__overview__.md +++ b/models/descriptions/__overview__.md @@ -1,10 +1,10 @@ {% docs __overview__ %} -# Welcome to the Flipside Crypto xyz Models Documentation +# Welcome to the Flipside Crypto EVMOS Models Documentation ## **What does this documentation cover?** -The documentation included here details the design of the xyz - tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/flipsideCrypto/xyz-models/) +The documentation included here details the design of the evmos + tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/flipsideCrypto/evmos-models/) ## **How do I use these docs?** The easiest way to navigate this documentation is to use the Quick Links below. These links will take you to the documentation for each table, which contains a description, a list of the columns, and other helpful information. @@ -17,7 +17,7 @@ There is more information on how to use dbt docs in the last section of this doc **Click on the links below to jump to the documentation for each schema.** -### Core Tables (`xyz`.`CORE`.``) +### Core Tables (`evmos`.`CORE`.``) **Dimension Tables:** @@ -33,8 +33,7 @@ There is more information on how to use dbt docs in the last section of this doc ## **Data Model Overview** -The xyz - models are built a few different ways, but the core fact tables are built using three layers of sql models: **bronze, silver, and gold (or core).** +The evmos models are built a few different ways, but the core fact tables are built using three layers of sql models: **bronze, silver, and gold (or core).** - Bronze: Data is loaded in from the source as a view - Silver: All necessary parsing, filtering, de-duping, and other transformations are done here @@ -68,7 +67,7 @@ Note that you can also right-click on models to interactively filter and explore - [Flipside](https://flipsidecrypto.xyz/) - [Velocity](https://app.flipsidecrypto.com/velocity?nav=Discover) - [Tutorials](https://docs.flipsidecrypto.com/our-data/tutorials) -- [Github](https://github.com/FlipsideCrypto/xyz-models) +- [Github](https://github.com/FlipsideCrypto/evmos-models) - [What is dbt?](https://docs.getdbt.com/docs/introduction) {% enddocs %} \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index e31d5cc..b19e00c 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -2,7 +2,7 @@ version: 2 sources: - name: crosschain - database: "{{ 'crosschain' if target.database == 'xyz' else 'crosschain_dev' }}" + database: "{{ 'crosschain' if target.database == 'evmos' else 'crosschain_dev' }}" schema: core tables: - name: dim_date_hours diff --git a/profiles.yml b/profiles.yml index 0661f98..6ff3582 100644 --- a/profiles.yml +++ b/profiles.yml @@ -1,4 +1,4 @@ -xyz: +evmos: target: dev outputs: dev: From 737b0a29989361bf059ad5d5e79dd9a98d2f0ade Mon Sep 17 00:00:00 2001 From: eric-laurello <102970824+eric-laurello@users.noreply.github.com> Date: Tue, 25 Apr 2023 10:43:56 -0400 Subject: [PATCH 02/12] Update README.md --- README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index ac8302a..4cef631 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,3 @@ - -# Please find and replace all instances of `evmos` with your project name. - ## Profile Set Up #### Use the following within profiles.yml @@ -74,4 +71,4 @@ dbt run --var '{"UPDATE_SNOWFLAKE_TAGS":True}' -s models/core/core__fact_swaps.s ``` select * from table(evmos.information_schema.tag_references('evmos.core.fact_blocks', 'table')); -``` \ No newline at end of file +``` From 9a2fb6fdb9e908edcc3d4dd5fa21a82519f63db2 Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Wed, 26 Apr 2023 08:52:42 -0400 Subject: [PATCH 03/12] add sample rpc calls --- analyses/sample_tendermint_rpc_calls.sql | 47 ++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 analyses/sample_tendermint_rpc_calls.sql diff --git a/analyses/sample_tendermint_rpc_calls.sql b/analyses/sample_tendermint_rpc_calls.sql new file mode 100644 index 0000000..8e36edc --- /dev/null +++ b/analyses/sample_tendermint_rpc_calls.sql @@ -0,0 +1,47 @@ +-- get chainhead +select ETHEREUM.STREAMLINE.UDF_API('GET','https://rpc-evmos.imperator.co/abci_info',{},{} ) + +--sample gen for last 1k blocks + create table sample_block_ids as ( + with gen as ( +select + row_number() over ( + order by + seq4() + ) as block_height +from + table(generator(rowcount => 100000000)) + ) + + + select top 10000 block_height from gen + -- except select 12872988 from sample_blocks + where block_height <= 12873408 + order by 1 desc) + + +--pull one block +select +block_height, +ETHEREUM.STREAMLINE.UDF_JSON_RPC_CALL('https://rpc-evmos.imperator.co/',{}, + [ + { 'id': block_height, 'jsonrpc': '2.0', 'method': 'block', 'params': [ block_height::STRING ] } + ] + ) data, + getdate() as _inserted_timestamp + from + (select 12889280 as block_height ) + + +--pull one block's transactions +select +block_height, +ETHEREUM.STREAMLINE.UDF_JSON_RPC_CALL('https://rpc-evmos.imperator.co/',{}, + [ + { 'id': block_height, 'jsonrpc': '2.0', 'method': 'tx_search', 'params': [ 'tx.height='||block_height::STRING , true, '1', '1000', 'asc' ] } + ] + ) data, + getdate() as _inserted_timestamp + from + (select 12889280 as block_height ) + \ No newline at end of file From 659d32fcfb8e59ef765929e4341e5ac65d3c5707 Mon Sep 17 00:00:00 2001 From: Jessica Huhnke <91915469+jhuhnke@users.noreply.github.com> Date: Tue, 9 May 2023 15:27:21 -0500 Subject: [PATCH 04/12] An 3099/silver (#1) * blocks table * transactions table * messages & message attributes * silver models - some tests still failing due to test data * core models / labels / validator & token metadata * updates to blocks & packages + formatting * rename to tendermint * updates * views to tables --- .github/workflows/dbt_run_dev_refresh.yml | 6 +- .github/workflows/dbt_run_incremental.yml | 41 ++ .github/workflows/dbt_run_streamline.yml | 48 +++ .../dbt_run_streamline_validators.yml | 48 +++ .github/workflows/dbt_test.yml | 41 ++ macros/tests/sequence_gaps.sql | 55 ++- macros/utils.sql | 35 ++ models/bronze/bronze__tendermint_blocks.sql | 60 +++ .../bronze__tendermint_transactions.sql | 61 +++ ...bronze_api__get_validator_metadata_lcd.sql | 46 +++ models/core/core__dim_labels.sql | 49 +++ models/core/core__dim_labels.yml | 33 ++ models/core/core__dim_tokens.sql | 26 ++ models/core/core__dim_tokens.yml | 49 +++ models/core/core__fact_blocks.sql | 13 + models/core/core__fact_blocks.yml | 29 ++ models/core/core__fact_msg_attributes.sql | 21 + models/core/core__fact_msg_attributes.yml | 42 ++ models/core/core__fact_msgs.sql | 19 + models/core/core__fact_msgs.yml | 38 ++ models/core/core__fact_transactions.sql | 75 ++++ models/core/core__fact_transactions.yml | 58 +++ models/core/core__fact_transfers.sql | 16 + models/core/core__fact_transfers.yml | 45 +++ models/core/core__fact_validators.sql | 23 ++ models/core/core__fact_validators.yml | 73 ++++ models/descriptions/_unique_key.md | 5 + models/descriptions/address.md | 5 + models/descriptions/amount.md | 5 + models/descriptions/block_id.md | 5 + models/descriptions/block_timestamp.md | 5 + models/descriptions/blockchain.md | 5 + models/descriptions/chain_id.md | 5 + models/descriptions/codespace.md | 5 + models/descriptions/creator.md | 5 + models/descriptions/currency.md | 5 + models/descriptions/decimal.md | 5 + models/descriptions/fee.md | 5 + models/descriptions/gas_used.md | 5 + models/descriptions/gas_wanted.md | 5 + models/descriptions/inserted_timestamp.md | 5 + models/descriptions/label.md | 5 + models/descriptions/label_subtype.md | 5 + models/descriptions/label_type.md | 5 + models/descriptions/msg_group.md | 5 + models/descriptions/msg_index.md | 5 + models/descriptions/msg_sub_group.md | 5 + models/descriptions/msg_type.md | 5 + models/descriptions/project_name.md | 5 + models/descriptions/proposer_address.md | 5 + models/descriptions/raw_metadata.md | 5 + models/descriptions/receiver.md | 5 + models/descriptions/sender.md | 5 + models/descriptions/transfer_type.md | 5 + models/descriptions/tx_code.md | 5 + models/descriptions/tx_count.md | 5 + models/descriptions/tx_from.md | 5 + models/descriptions/tx_id.md | 5 + models/descriptions/tx_log.md | 5 + models/descriptions/tx_succeeded.md | 5 + models/descriptions/validator_hash.md | 5 + models/silver/silver__blocks.sql | 46 +++ models/silver/silver__blocks.yml | 70 ++++ models/silver/silver__msg_attributes.sql | 41 ++ models/silver/silver__msg_attributes.yml | 70 ++++ models/silver/silver__msgs.sql | 134 +++++++ models/silver/silver__msgs.yml | 76 ++++ models/silver/silver__transactions.sql | 49 +++ models/silver/silver__transactions.yml | 87 +++++ models/silver/silver__transfers.sql | 362 ++++++++++++++++++ models/silver/silver__transfers.yml | 92 +++++ models/silver/silver__validators.sql | 26 ++ models/silver/silver__validators.yml | 139 +++++++ models/sources.yml | 18 + packages.yml | 6 +- 75 files changed, 2408 insertions(+), 33 deletions(-) create mode 100644 .github/workflows/dbt_run_incremental.yml create mode 100644 .github/workflows/dbt_run_streamline.yml create mode 100644 .github/workflows/dbt_run_streamline_validators.yml create mode 100644 .github/workflows/dbt_test.yml create mode 100644 macros/utils.sql create mode 100644 models/bronze/bronze__tendermint_blocks.sql create mode 100644 models/bronze/bronze__tendermint_transactions.sql create mode 100644 models/bronze_api/bronze_api__get_validator_metadata_lcd.sql create mode 100644 models/core/core__dim_labels.sql create mode 100644 models/core/core__dim_labels.yml create mode 100644 models/core/core__dim_tokens.sql create mode 100644 models/core/core__dim_tokens.yml create mode 100644 models/core/core__fact_blocks.sql create mode 100644 models/core/core__fact_blocks.yml create mode 100644 models/core/core__fact_msg_attributes.sql create mode 100644 models/core/core__fact_msg_attributes.yml create mode 100644 models/core/core__fact_msgs.sql create mode 100644 models/core/core__fact_msgs.yml create mode 100644 models/core/core__fact_transactions.sql create mode 100644 models/core/core__fact_transactions.yml create mode 100644 models/core/core__fact_transfers.sql create mode 100644 models/core/core__fact_transfers.yml create mode 100644 models/core/core__fact_validators.sql create mode 100644 models/core/core__fact_validators.yml create mode 100644 models/descriptions/_unique_key.md create mode 100644 models/descriptions/address.md create mode 100644 models/descriptions/amount.md create mode 100644 models/descriptions/block_id.md create mode 100644 models/descriptions/block_timestamp.md create mode 100644 models/descriptions/blockchain.md create mode 100644 models/descriptions/chain_id.md create mode 100644 models/descriptions/codespace.md create mode 100644 models/descriptions/creator.md create mode 100644 models/descriptions/currency.md create mode 100644 models/descriptions/decimal.md create mode 100644 models/descriptions/fee.md create mode 100644 models/descriptions/gas_used.md create mode 100644 models/descriptions/gas_wanted.md create mode 100644 models/descriptions/inserted_timestamp.md create mode 100644 models/descriptions/label.md create mode 100644 models/descriptions/label_subtype.md create mode 100644 models/descriptions/label_type.md create mode 100644 models/descriptions/msg_group.md create mode 100644 models/descriptions/msg_index.md create mode 100644 models/descriptions/msg_sub_group.md create mode 100644 models/descriptions/msg_type.md create mode 100644 models/descriptions/project_name.md create mode 100644 models/descriptions/proposer_address.md create mode 100644 models/descriptions/raw_metadata.md create mode 100644 models/descriptions/receiver.md create mode 100644 models/descriptions/sender.md create mode 100644 models/descriptions/transfer_type.md create mode 100644 models/descriptions/tx_code.md create mode 100644 models/descriptions/tx_count.md create mode 100644 models/descriptions/tx_from.md create mode 100644 models/descriptions/tx_id.md create mode 100644 models/descriptions/tx_log.md create mode 100644 models/descriptions/tx_succeeded.md create mode 100644 models/descriptions/validator_hash.md create mode 100644 models/silver/silver__blocks.sql create mode 100644 models/silver/silver__blocks.yml create mode 100644 models/silver/silver__msg_attributes.sql create mode 100644 models/silver/silver__msg_attributes.yml create mode 100644 models/silver/silver__msgs.sql create mode 100644 models/silver/silver__msgs.yml create mode 100644 models/silver/silver__transactions.sql create mode 100644 models/silver/silver__transactions.yml create mode 100644 models/silver/silver__transfers.sql create mode 100644 models/silver/silver__transfers.yml create mode 100644 models/silver/silver__validators.sql create mode 100644 models/silver/silver__validators.yml diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml index 2302c1a..9362e7a 100644 --- a/.github/workflows/dbt_run_dev_refresh.yml +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -3,9 +3,9 @@ run-name: dbt_run_dev_refresh on: workflow_dispatch: - # schedule: - # # Runs "at 9:00 UTC" (see https://crontab.guru) - # - cron: '0 9 * * *' + schedule: + # Runs "at 9:00 UTC" (see https://crontab.guru) + - cron: '0 9 * * *' env: USE_VARS: "${{ vars.USE_VARS }}" diff --git a/.github/workflows/dbt_run_incremental.yml b/.github/workflows/dbt_run_incremental.yml new file mode 100644 index 0000000..e6f108e --- /dev/null +++ b/.github/workflows/dbt_run_incremental.yml @@ -0,0 +1,41 @@ +name: dbt_run_scheduled +run-name: dbt_run_scheduled + +on: + workflow_dispatch: + schedule: + - cron: '0 1,7,13,19 * * *' + +env: + DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" + SCHEMA: "${{ secrets.SCHEMA }}" + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt run -s ./models diff --git a/.github/workflows/dbt_run_streamline.yml b/.github/workflows/dbt_run_streamline.yml new file mode 100644 index 0000000..3e12928 --- /dev/null +++ b/.github/workflows/dbt_run_streamline.yml @@ -0,0 +1,48 @@ +name: dbt_run_streamline +run-name: dbt_run_streamline + +on: + push: + branches: + - main + - turn-off-dev-turn-on-prod + schedule: + # Runs "every 6 hours" (see https://crontab.guru) + - cron: '0 */6 * * *' + +env: + DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" + SCHEMA: "${{ secrets.SCHEMA }}" + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt run -m ./models/streamline/* + + + diff --git a/.github/workflows/dbt_run_streamline_validators.yml b/.github/workflows/dbt_run_streamline_validators.yml new file mode 100644 index 0000000..ec08c55 --- /dev/null +++ b/.github/workflows/dbt_run_streamline_validators.yml @@ -0,0 +1,48 @@ +name: dbt_run_streamline_validators +run-name: dbt_run_streamline_validators + +on: + push: + branches: + - main + - turn-off-dev-turn-on-prod + schedule: + # Runs "every 6 hours" (see https://crontab.guru) + - cron: '0 0,12,23 * * *' + +env: + DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" + SCHEMA: "${{ secrets.SCHEMA }}" + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt run --full-refresh -x -m 1+models/streamline/streamline__validators_realtime.sql + + + diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml new file mode 100644 index 0000000..93241a8 --- /dev/null +++ b/.github/workflows/dbt_test.yml @@ -0,0 +1,41 @@ +name: dbt_test_scheduled +run-name: dbt_test_scheduled + +on: + workflow_dispatch: + schedule: + - cron: '0 4 * * *' + +env: + DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" + SCHEMA: "${{ secrets.SCHEMA }}" + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt test -m ./models diff --git a/macros/tests/sequence_gaps.sql b/macros/tests/sequence_gaps.sql index 84a9aa9..9425003 100644 --- a/macros/tests/sequence_gaps.sql +++ b/macros/tests/sequence_gaps.sql @@ -1,37 +1,34 @@ -{% macro sequence_gaps( - table, - partition_by, - column - ) %} - {%- set partition_sql = partition_by | join(", ") -%} - {%- set previous_column = "prev_" ~ column -%} - WITH source AS ( - SELECT - {{ partition_sql + "," if partition_sql }} - {{ column }}, - LAG( - {{ column }}, - 1 - ) over ( - {{ "PARTITION BY " ~ partition_sql if partition_sql }} - ORDER BY - {{ column }} ASC - ) AS {{ previous_column }} - FROM - {{ table }} - WHERE - block_timestamp::date <= current_date - 1 - ) +{% test sequence_gaps( + model, + partition_by, + column_name +) %} +{%- set partition_sql = partition_by | join(", ") -%} +{%- set previous_column = "prev_" ~ column_name -%} +WITH source AS ( + SELECT + {{ partition_sql + "," if partition_sql }} + {{ column_name }}, + LAG( + {{ column_name }}, + 1 + ) over ( + {{ "PARTITION BY " ~ partition_sql if partition_sql }} + ORDER BY + {{ column_name }} ASC + ) AS {{ previous_column }} + FROM + {{ model }} +) SELECT {{ partition_sql + "," if partition_sql }} {{ previous_column }}, - {{ column }}, - {{ column }} - {{ previous_column }} + {{ column_name }}, + {{ column_name }} - {{ previous_column }} - 1 AS gap FROM source WHERE - {{ column }} - {{ previous_column }} <> 1 + {{ column_name }} - {{ previous_column }} <> 1 ORDER BY - gap DESC -{% endmacro %} + gap DESC {% endtest %} diff --git a/macros/utils.sql b/macros/utils.sql new file mode 100644 index 0000000..85549f1 --- /dev/null +++ b/macros/utils.sql @@ -0,0 +1,35 @@ +{% macro if_data_call_function( + func, + target + ) %} + {% if var( + "STREAMLINE_INVOKE_STREAMS" + ) %} + {% if execute %} + {{ log( + "Running macro `if_data_call_function`: Calling udf " ~ func ~ " on " ~ target, + True + ) }} + {% endif %} + SELECT + {{ func }} + WHERE + EXISTS( + SELECT + 1 + FROM + {{ target }} + LIMIT + 1 + ) + {% else %} + {% if execute %} + {{ log( + "Running macro `if_data_call_function`: NOOP", + False + ) }} + {% endif %} + SELECT + NULL + {% endif %} +{% endmacro %} diff --git a/models/bronze/bronze__tendermint_blocks.sql b/models/bronze/bronze__tendermint_blocks.sql new file mode 100644 index 0000000..bacae1d --- /dev/null +++ b/models/bronze/bronze__tendermint_blocks.sql @@ -0,0 +1,60 @@ +{{ config( + materialized = 'incremental', + unique_key = 'block_id', + cluster_by = ['_inserted_timestamp::date'], + merge_update_columns = ["block_id"], +) }} + +WITH meta AS ( + + SELECT + registered_on, + last_modified, + LEAST( + last_modified, + registered_on + ) AS _inserted_timestamp, + file_name + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "streamline", "tendermint_block") }}' + ) + ) A + +{% if is_incremental() %} +WHERE + LEAST( + registered_on, + last_modified + ) >= ( + SELECT + COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP + FROM + {{ this }}) + ) +{% else %} +) +{% endif %} + +SELECT + value, + _partition_by_block_id, + block_number AS block_id, + DATA, + TO_TIMESTAMP( + m._inserted_timestamp + ) AS _inserted_timestamp +FROM + {{ source( + 'streamline', + 'tendermint_block' + ) }} +JOIN meta m +ON m.file_name = metadata$filename +WHERE + DATA: error IS NULL + +qualify(ROW_NUMBER() over (PARTITION BY block_number +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/bronze/bronze__tendermint_transactions.sql b/models/bronze/bronze__tendermint_transactions.sql new file mode 100644 index 0000000..c2ca96a --- /dev/null +++ b/models/bronze/bronze__tendermint_transactions.sql @@ -0,0 +1,61 @@ +{{ config( + materialized = 'incremental', + unique_key = 'tx_id', + cluster_by = ['_inserted_timestamp::date'], + merge_update_columns = ["data", "_inserted_timestamp"], +) }} + +WITH meta AS ( + + SELECT + registered_on, + last_modified, + LEAST( + last_modified, + registered_on + ) AS _inserted_timestamp, + file_name + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "streamline", "tendermint_tx_search") }}' + ) + ) A + +{% if is_incremental() %} +WHERE + LEAST( + registered_on, + last_modified + ) >= ( + SELECT + COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP + FROM + {{ this }}) + ) +{% else %} +) +{% endif %} + +SELECT + value, + _partition_by_block_id, + block_number as block_id, + value :data :hash :: STRING AS tx_id, + metadata, + array_index, + DATA, + TO_TIMESTAMP( + m._inserted_timestamp + ) AS _inserted_timestamp +FROM + {{ source( + 'streamline', + 'tendermint_tx_search' + ) }} +JOIN meta m +ON m.file_name = metadata$filename +WHERE + DATA: error IS NULL qualify(ROW_NUMBER() over (PARTITION BY value :data :hash :: STRING +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/bronze_api/bronze_api__get_validator_metadata_lcd.sql b/models/bronze_api/bronze_api__get_validator_metadata_lcd.sql new file mode 100644 index 0000000..42cf493 --- /dev/null +++ b/models/bronze_api/bronze_api__get_validator_metadata_lcd.sql @@ -0,0 +1,46 @@ +{{ config( + materialized = 'table' +) }} + +WITH call AS ( + + SELECT + ethereum.streamline.udf_api( + 'GET', + 'https://lcd-evmos.keplr.app/cosmos/staking/v1beta1/validators?pagination.limit=5000',{},{} + ) AS resp, + SYSDATE() AS _inserted_timestamp +), +keep_last_if_failed AS ( + SELECT + i.value :operator_address :: STRING AS address, + i.value AS DATA, + _inserted_timestamp, + 2 AS RANK + FROM + call, + LATERAL FLATTEN( + input => resp :data :validators + ) i + UNION ALL + SELECT + address, + DATA, + _inserted_timestamp, + 1 AS RANK + FROM + bronze_api.get_validator_metadata_lcd +) +SELECT + address, + DATA, + _inserted_timestamp +FROM + keep_last_if_failed A + JOIN ( + SELECT + MAX(RANK) max_rank + FROM + keep_last_if_failed + ) b + ON A.rank = b.max_rank \ No newline at end of file diff --git a/models/core/core__dim_labels.sql b/models/core/core__dim_labels.sql new file mode 100644 index 0000000..6d0a065 --- /dev/null +++ b/models/core/core__dim_labels.sql @@ -0,0 +1,49 @@ +{{ config( + materialized = 'table' +) }} + +SELECT + blockchain, + creator, + address, + label_type, + label_subtype, + project_name AS label, + address_name AS address_name, + NULL AS raw_metadata +FROM + {{ source( + 'crosschain', + 'address_labels' + ) }} +WHERE + blockchain = 'evmos' + +UNION +SELECT + blockchain, + creator, + address, + label_type, + label_subtype, + project_name AS label, + label AS address_name, + raw_metadata +FROM + {{ ref('core__dim_tokens') }} +WHERE + blockchain = 'evmos' +UNION +SELECT + blockchain, + creator, + address, + label_type, + label_subtype, + project_name AS label, + label AS address_name, + NULL AS raw_metadata +FROM + {{ ref('core__fact_validators') }} +WHERE + blockchain = 'evmos' \ No newline at end of file diff --git a/models/core/core__dim_labels.yml b/models/core/core__dim_labels.yml new file mode 100644 index 0000000..7c3a2f2 --- /dev/null +++ b/models/core/core__dim_labels.yml @@ -0,0 +1,33 @@ +version: 2 +models: + - name: core__dim_labels + description: A hand curated table containing address names / labels for popular contracts, validators, tokens, etc. + columns: + - name: BLOCKCHAIN + description: The name of the blockchain + tests: + - not_null + - name: CREATOR + description: The name of the creator of the label + tests: + - not_null + - name: LABEL_TYPE + description: A high-level category describing the addresses main function or ownership + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: ['flotsam', 'nft', 'defi', 'dex', 'cex', 'dapp', 'token', 'operator', 'layer2', 'chadmin'] + - name: LABEL_SUBTYPE + description: A sub-category nested within label type providing further detail + tests: + - not_null + - name: LABEL + description: Name of the controlling entity of the address + tests: + - not_null + - name: ADDRESS + description: Address that the label is for. This is the field that should be used to join other tables with labels. + tests: + - not_null + - name: RAW_METADATA + description: A field available for tokens that contains decimal information \ No newline at end of file diff --git a/models/core/core__dim_tokens.sql b/models/core/core__dim_tokens.sql new file mode 100644 index 0000000..0fe04b0 --- /dev/null +++ b/models/core/core__dim_tokens.sql @@ -0,0 +1,26 @@ +{{ config( + materialized = 'table' +) }} + +SELECT + 'evmos' AS blockchain, + address, + creator, + label_type, + label_subtype, + label, + project_name, + alias, + DECIMAL, + raw_metadata, + concat_ws( + '-', + address, + creator, + blockchain + ) AS unique_key +FROM + {{ source( + 'osmo', + 'asset_metadata' + ) }} diff --git a/models/core/core__dim_tokens.yml b/models/core/core__dim_tokens.yml new file mode 100644 index 0000000..272b770 --- /dev/null +++ b/models/core/core__dim_tokens.yml @@ -0,0 +1,49 @@ +version: 2 +models: + - name: core__dim_tokens + description: Contains token metadata for assets on the Evmos blockchain. This table was sourced from an API and may not contain every token. + columns: + - name: ADDRESS + description: "{{ doc('address') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: CREATOR + description: "{{ doc('creator') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: LABEL_TYPE + description: "{{ doc('label_type') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: LABEL_SUBTYPE + description: "{{ doc('label_subtype') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: LABEL + description: "{{ doc('label') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: PROJECT_NAME + description: "{{ doc('project_name') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: ALIAS + description: A secondary address for the token, where available + tests: + - dbt_expectations.expect_column_to_exist + - name: DECIMAL + description: "{{ doc('decimal') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: RAW_METADATA + description: "{{ doc('raw_metadata') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: UNIQUE_KEY + description: The unique key of the table + tests: + - dbt_expectations.expect_column_to_exist \ No newline at end of file diff --git a/models/core/core__fact_blocks.sql b/models/core/core__fact_blocks.sql new file mode 100644 index 0000000..9a6fc9c --- /dev/null +++ b/models/core/core__fact_blocks.sql @@ -0,0 +1,13 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + block_id, + block_timestamp, + chain_id, + tx_count, + proposer_address, + validator_hash +FROM + {{ ref('silver__blocks') }} \ No newline at end of file diff --git a/models/core/core__fact_blocks.yml b/models/core/core__fact_blocks.yml new file mode 100644 index 0000000..c05502c --- /dev/null +++ b/models/core/core__fact_blocks.yml @@ -0,0 +1,29 @@ +version: 2 +models: + - name: core__fact_blocks + description: Records of all blocks that have occurred on Evmos. This schema is currently in lite mode, and contains data back to DATE. + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: CHAIN_ID + description: "{{ doc('chain_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: TX_COUNT + description: "{{ doc('tx_count') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: PROPOSER_ADDRESS + description: "{{ doc('proposer_address') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: VALIDATOR_HASH + description: "{{ doc('validator_hash') }}" + tests: + - dbt_expectations.expect_column_to_exist \ No newline at end of file diff --git a/models/core/core__fact_msg_attributes.sql b/models/core/core__fact_msg_attributes.sql new file mode 100644 index 0000000..13e7743 --- /dev/null +++ b/models/core/core__fact_msg_attributes.sql @@ -0,0 +1,21 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + block_id, + block_timestamp, + tx_id, + tx_succeeded, + CONCAT( + msg_group, + ':', + msg_sub_group + ) AS msg_group, + msg_index, + msg_type, + attribute_index, + attribute_key, + attribute_value +FROM + {{ ref('silver__msg_attributes') }} diff --git a/models/core/core__fact_msg_attributes.yml b/models/core/core__fact_msg_attributes.yml new file mode 100644 index 0000000..c996d68 --- /dev/null +++ b/models/core/core__fact_msg_attributes.yml @@ -0,0 +1,42 @@ +version: 2 +models: + - name: core__fact_msg_attributes + description: Records of all message attributes associated to messages that have occurred on Evmos. This schema is currently in lite mode, and contains data back to DATE. + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: TX_SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: MSG_GROUP + description: "{{ doc('msg_group') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: MSG_INDEX + description: "{{ doc('msg_index') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: MSG_TYPE + description: "{{ doc('msg_type') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: ATTRIBUTE_KEY + description: "They key from the key-value pair from the message attribute" + tests: + - dbt_expectations.expect_column_to_exist + - name: ATTRIBUTE_VALUE + description: "They value from the key-value pair from the message attribute" + tests: + - dbt_expectations.expect_column_to_exist + diff --git a/models/core/core__fact_msgs.sql b/models/core/core__fact_msgs.sql new file mode 100644 index 0000000..3908d01 --- /dev/null +++ b/models/core/core__fact_msgs.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + block_id, + block_timestamp, + tx_id, + tx_succeeded, + CONCAT( + msg_group, + ':', + msg_sub_group + ) AS msg_group, + msg_index, + msg_type, + msg +FROM + {{ ref('silver__msgs') }} diff --git a/models/core/core__fact_msgs.yml b/models/core/core__fact_msgs.yml new file mode 100644 index 0000000..da1b8a4 --- /dev/null +++ b/models/core/core__fact_msgs.yml @@ -0,0 +1,38 @@ +version: 2 +models: + - name: core__fact_msgs + description: Records of all message attributes associated to messages that have occurred on Evmos. This schema is currently in lite mode, and contains data back to DATE. + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: TX_SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: MSG_GROUP + description: "{{ doc('msg_group') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: MSG_INDEX + description: "{{ doc('msg_index') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: MSG_TYPE + description: "{{ doc('msg_type') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: MSG + description: "A block of json that contains the message attributes in base64 encoding." + tests: + - dbt_expectations.expect_column_to_exist + diff --git a/models/core/core__fact_transactions.sql b/models/core/core__fact_transactions.sql new file mode 100644 index 0000000..784dc4a --- /dev/null +++ b/models/core/core__fact_transactions.sql @@ -0,0 +1,75 @@ +{{ config( + materialized = 'incremental', + unique_key = "tx_id", + incremental_strategy = 'merge', + cluster_by = ['block_timestamp::DATE'], +) }} + +WITH fee AS ( + + SELECT + tx_id, + attribute_value AS fee + FROM + {{ ref('silver__msg_attributes') }} + WHERE + attribute_key = 'fee' + +{% if is_incremental() %} +AND _inserted_timestamp :: DATE >= CURRENT_DATE -2 +{% endif %} +qualify(ROW_NUMBER() over(PARTITION BY tx_id +ORDER BY + msg_index)) = 1 +), +spender AS ( + SELECT + tx_id, + SPLIT_PART( + attribute_value, + '/', + 0 + ) AS tx_from + FROM + {{ ref('silver__msg_attributes') }} + WHERE + attribute_key = 'acc_seq' + OR (msg_type = 'transfer' + AND attribute_key = 'sender') + +{% if is_incremental() %} +AND _inserted_timestamp :: DATE >= CURRENT_DATE -2 +{% endif %} + +qualify(ROW_NUMBER() over(PARTITION BY tx_id +ORDER BY + msg_index)) = 1 +) +SELECT + t.block_id, + t.block_timestamp, + t.tx_id, + s.tx_from, + tx_succeeded, + codespace, + COALESCE( + fee, + '0aevmos' + ) AS fee, + gas_used, + gas_wanted, + tx_code, + tx_type, + msgs +FROM + {{ ref('silver__transactions') }} + t + LEFT OUTER JOIN fee f + ON t.tx_id = f.tx_id + LEFT OUTER JOIN spender s + ON t.tx_id = s.tx_id + +{% if is_incremental() %} +WHERE + _inserted_timestamp :: DATE >= CURRENT_DATE -2 +{% endif %} diff --git a/models/core/core__fact_transactions.yml b/models/core/core__fact_transactions.yml new file mode 100644 index 0000000..3e13d2d --- /dev/null +++ b/models/core/core__fact_transactions.yml @@ -0,0 +1,58 @@ +version: 2 +models: + - name: core__fact_transactions + description: Records of all transactions that have occurred on Evmos. This schema is currently in lite mode, and contains data back to DATE. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: TX_FROM + description: "{{ doc('tx_from') }}" + tests: + - not_null: + where: tx_succeeded = 'TRUE' + - name: TX_SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: CODESPACE + description: "{{ doc('codespace') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: FEE + description: "{{ doc('fee') }}" + tests: + - not_null + - name: GAS_USED + description: "{{ doc('gas_used') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: GAS_WANTED + description: "{{ doc('gas_wanted') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: TX_CODE + description: "{{ doc('tx_code') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: TX_TYPE + description: "The type of transaction" + tests: + - dbt_expectations.expect_column_to_exist + - name: MSGS + description: "The underlying json from the messages or events within the transactions" + tests: + - dbt_expectations.expect_column_to_exist diff --git a/models/core/core__fact_transfers.sql b/models/core/core__fact_transfers.sql new file mode 100644 index 0000000..d7bc2dc --- /dev/null +++ b/models/core/core__fact_transfers.sql @@ -0,0 +1,16 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + block_id, + block_timestamp, + tx_id, + tx_succeeded, + transfer_type, + sender, + amount, + currency, + receiver +FROM + {{ ref('silver__transfers') }} \ No newline at end of file diff --git a/models/core/core__fact_transfers.yml b/models/core/core__fact_transfers.yml new file mode 100644 index 0000000..d04153a --- /dev/null +++ b/models/core/core__fact_transfers.yml @@ -0,0 +1,45 @@ +version: 2 +models: + - name: core__fact_transfers + description: Records of all transfers on Evmos. This schema is currently in lite mode and contains data back to DATE. + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: TX_SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: TRANSFER_TYPE + description: "{{ doc('transfer_type') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: SENDER + description: "{{ doc('sender') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: AMOUNT + description: "{{ doc('amount') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: CURRENCY + description: "{{ doc('currency') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: RECEIVER + description: "{{ doc('receiver') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: UNIQUE_KEY + description: "{{ doc('_unique_key') }}" + tests: + - dbt_expectations.expect_column_to_exist \ No newline at end of file diff --git a/models/core/core__fact_validators.sql b/models/core/core__fact_validators.sql new file mode 100644 index 0000000..38a6f7e --- /dev/null +++ b/models/core/core__fact_validators.sql @@ -0,0 +1,23 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + address, + blockchain, + creator, + label_type, + label_subtype, + label, + project_name, + delegator_shares, + jailed, + rate, + max_change_rate, + max_rate, + min_self_delegation, + RANK, + raw_metadata, + unique_key +FROM + {{ ref('silver__validators') }} diff --git a/models/core/core__fact_validators.yml b/models/core/core__fact_validators.yml new file mode 100644 index 0000000..76e391c --- /dev/null +++ b/models/core/core__fact_validators.yml @@ -0,0 +1,73 @@ +version: 2 +models: + - name: core__fact_validators + description: Information about all active and inactive set validators on Evmos. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - UNIQUE_KEY + columns: + - name: ADDRESS + description: "{{ doc('address') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: CREATOR + description: "{{ doc('creator') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: LABEL_TYPE + description: "{{ doc('label_type') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: LABEL_SUBTYPE + description: "{{ doc('label_subtype') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: LABEL + description: "{{ doc('label') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: PROJECT_NAME + description: "{{ doc('project_name') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: DELEGATOR_SHARES + description: The number of tokens staked to the validator. + tests: + - dbt_expectations.expect_column_to_exist + - name: JAILED + description: FALSE when a validator is not jailed, TRUE when a validator is jailed + tests: + - dbt_expectations.expect_column_to_exist + - name: RATE + description: The current commission rate the validator is charging stakers. + tests: + - dbt_expectations.expect_column_to_exist + - name: MAX_CHANGE_RATE + description: The maximum rate at which a validator can change their commission per day. + tests: + - dbt_expectations.expect_column_to_exist + - name: MAX_RATE + description: The maximum commission rate that the validator can charge + tests: + - dbt_expectations.expect_column_to_exist + - name: MIN_SELF_DELEGATION + description: The minimum number of Evmos tokens that the operator must be staking with their own validator + tests: + - dbt_expectations.expect_column_to_exist + - name: RANK + description: The rank of the validator in the validator set. Rank is determined by the number of Evmos tokens staked to the validator + tests: + - dbt_expectations.expect_column_to_exist + - name: RAW_METADATA + description: "{{ doc('raw_metadata') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: UNIQUE_KEY + description: The unique key of the table + tests: + - dbt_expectations.expect_column_to_exist \ No newline at end of file diff --git a/models/descriptions/_unique_key.md b/models/descriptions/_unique_key.md new file mode 100644 index 0000000..f602192 --- /dev/null +++ b/models/descriptions/_unique_key.md @@ -0,0 +1,5 @@ +{% docs _unique_key %} + +The unique key for the table. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/address.md b/models/descriptions/address.md new file mode 100644 index 0000000..36215fe --- /dev/null +++ b/models/descriptions/address.md @@ -0,0 +1,5 @@ +{% docs address %} + +Address unique to an individual wallet, validator, or token. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/amount.md b/models/descriptions/amount.md new file mode 100644 index 0000000..1c7f7a0 --- /dev/null +++ b/models/descriptions/amount.md @@ -0,0 +1,5 @@ +{% docs amount %} + +The amount that was used in the transaction message. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/block_id.md b/models/descriptions/block_id.md new file mode 100644 index 0000000..95c8389 --- /dev/null +++ b/models/descriptions/block_id.md @@ -0,0 +1,5 @@ +{% docs block_id %} + +The block height the block was recorded at. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/block_timestamp.md b/models/descriptions/block_timestamp.md new file mode 100644 index 0000000..907f6a3 --- /dev/null +++ b/models/descriptions/block_timestamp.md @@ -0,0 +1,5 @@ +{% docs block_timestamp %} + +The date and time at which the block began. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/blockchain.md b/models/descriptions/blockchain.md new file mode 100644 index 0000000..dc243a7 --- /dev/null +++ b/models/descriptions/blockchain.md @@ -0,0 +1,5 @@ +{% docs blockchain %} + +In this table, always Cosmos. Used to join to cross-chain tables. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/chain_id.md b/models/descriptions/chain_id.md new file mode 100644 index 0000000..0981eb6 --- /dev/null +++ b/models/descriptions/chain_id.md @@ -0,0 +1,5 @@ +{% docs chain_id %} + +The name and version of the blockchain. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/codespace.md b/models/descriptions/codespace.md new file mode 100644 index 0000000..7537e19 --- /dev/null +++ b/models/descriptions/codespace.md @@ -0,0 +1,5 @@ +{% docs codespace %} + +Namespace for the code. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/creator.md b/models/descriptions/creator.md new file mode 100644 index 0000000..ff72c36 --- /dev/null +++ b/models/descriptions/creator.md @@ -0,0 +1,5 @@ +{% docs creator %} + +Name of the label creator - for now, this will always be "Flipside." + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/currency.md b/models/descriptions/currency.md new file mode 100644 index 0000000..93f0327 --- /dev/null +++ b/models/descriptions/currency.md @@ -0,0 +1,5 @@ +{% docs currency %} + +The currency that was used in the transaction message. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/decimal.md b/models/descriptions/decimal.md new file mode 100644 index 0000000..eb9c154 --- /dev/null +++ b/models/descriptions/decimal.md @@ -0,0 +1,5 @@ +{% docs decimal %} + +Divide the amount by POW(10, decimal) to get the amount used in the transaction. This value can be NULL, as decimals are hand curated from an outside source and not found on-chain. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/fee.md b/models/descriptions/fee.md new file mode 100644 index 0000000..8247bd4 --- /dev/null +++ b/models/descriptions/fee.md @@ -0,0 +1,5 @@ +{% docs fee %} + +The fee is paid by the initiator of the transaction. Fee = gas * gas price and is given in micro-OSMO. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/gas_used.md b/models/descriptions/gas_used.md new file mode 100644 index 0000000..ae688ae --- /dev/null +++ b/models/descriptions/gas_used.md @@ -0,0 +1,5 @@ +{% docs gas_used %} + +The amount of gas consumed by the transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/gas_wanted.md b/models/descriptions/gas_wanted.md new file mode 100644 index 0000000..937d08a --- /dev/null +++ b/models/descriptions/gas_wanted.md @@ -0,0 +1,5 @@ +{% docs gas_wanted %} + +Amount of gas requested for a transaction. It is provided by users when the transaction is generated. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/inserted_timestamp.md b/models/descriptions/inserted_timestamp.md new file mode 100644 index 0000000..374d1f5 --- /dev/null +++ b/models/descriptions/inserted_timestamp.md @@ -0,0 +1,5 @@ +{% docs inserted_timestamp %} + +The date and time at which the block or transaction was inserted in the bronze tables. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/label.md b/models/descriptions/label.md new file mode 100644 index 0000000..51af3dc --- /dev/null +++ b/models/descriptions/label.md @@ -0,0 +1,5 @@ +{% docs label %} + +The label or name of the address. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/label_subtype.md b/models/descriptions/label_subtype.md new file mode 100644 index 0000000..113de57 --- /dev/null +++ b/models/descriptions/label_subtype.md @@ -0,0 +1,5 @@ +{% docs label_subtype %} + +Adds more detail to the label type. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/label_type.md b/models/descriptions/label_type.md new file mode 100644 index 0000000..c81472b --- /dev/null +++ b/models/descriptions/label_type.md @@ -0,0 +1,5 @@ +{% docs label_type %} + +A broad category that describes what a label is representing. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/msg_group.md b/models/descriptions/msg_group.md new file mode 100644 index 0000000..ef4c5c1 --- /dev/null +++ b/models/descriptions/msg_group.md @@ -0,0 +1,5 @@ +{% docs msg_group %} + +Value grouping different messages together to represent a single action. Format will include the numeric msg_group and msg_sub_group with a ":" seperator. The subgroup will always be 0 except for "Exec" actions. NULL group means messages are related to the header (overall transaction) + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/msg_index.md b/models/descriptions/msg_index.md new file mode 100644 index 0000000..8af9065 --- /dev/null +++ b/models/descriptions/msg_index.md @@ -0,0 +1,5 @@ +{% docs msg_index %} + +Short for "message index," the position in which messages occur in a transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/msg_sub_group.md b/models/descriptions/msg_sub_group.md new file mode 100644 index 0000000..8b5a36c --- /dev/null +++ b/models/descriptions/msg_sub_group.md @@ -0,0 +1,5 @@ +{% docs msg_sub_group %} + +Silver only -- Numeric value grouping different messages together to represent a single action within a group. This is relevent for exec actions that contain mutiple underlying actions. NULL sub group means messages are related to the header (overall transaction) + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/msg_type.md b/models/descriptions/msg_type.md new file mode 100644 index 0000000..e16302b --- /dev/null +++ b/models/descriptions/msg_type.md @@ -0,0 +1,5 @@ +{% docs msg_type %} + +A string containing information about the type of message occurring. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/project_name.md b/models/descriptions/project_name.md new file mode 100644 index 0000000..a8a553d --- /dev/null +++ b/models/descriptions/project_name.md @@ -0,0 +1,5 @@ +{% docs project_name %} + +The name of the project the label belongs to. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/proposer_address.md b/models/descriptions/proposer_address.md new file mode 100644 index 0000000..0dcaa94 --- /dev/null +++ b/models/descriptions/proposer_address.md @@ -0,0 +1,5 @@ +{% docs proposer_address %} + +The address of the validator that proposed the block. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/raw_metadata.md b/models/descriptions/raw_metadata.md new file mode 100644 index 0000000..539e992 --- /dev/null +++ b/models/descriptions/raw_metadata.md @@ -0,0 +1,5 @@ +{% docs raw_metadata %} + +Additional details about the validator or token in json format. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/receiver.md b/models/descriptions/receiver.md new file mode 100644 index 0000000..e82f2da --- /dev/null +++ b/models/descriptions/receiver.md @@ -0,0 +1,5 @@ +{% docs receiver %} + +The wallet address of the individual received tokens in the transfer. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/sender.md b/models/descriptions/sender.md new file mode 100644 index 0000000..1b55aab --- /dev/null +++ b/models/descriptions/sender.md @@ -0,0 +1,5 @@ +{% docs sender %} + +The wallet address of the individual sent tokens in the transfer. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/transfer_type.md b/models/descriptions/transfer_type.md new file mode 100644 index 0000000..b8630ce --- /dev/null +++ b/models/descriptions/transfer_type.md @@ -0,0 +1,5 @@ +{% docs transfer_type %} + +Details on the type of transfer occurring during the transaction. "Transfer_In" = depositing tokens onto the Cosmos Hub. "Transfer_out" = withdrawing tokens from the Cosmos Hub. "Cosmos" = wallet to wallet transfer on the Cosmos Hub. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/tx_code.md b/models/descriptions/tx_code.md new file mode 100644 index 0000000..5200c7e --- /dev/null +++ b/models/descriptions/tx_code.md @@ -0,0 +1,5 @@ +{% docs tx_code %} + +A number that corresponds to various error codes. When "0", the transaction is successful. Non-zero numbers signify different types of transaction failures. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/tx_count.md b/models/descriptions/tx_count.md new file mode 100644 index 0000000..750dccb --- /dev/null +++ b/models/descriptions/tx_count.md @@ -0,0 +1,5 @@ +{% docs tx_count %} + +The number of transactions that occurred during a block. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/tx_from.md b/models/descriptions/tx_from.md new file mode 100644 index 0000000..d9fd624 --- /dev/null +++ b/models/descriptions/tx_from.md @@ -0,0 +1,5 @@ +{% docs tx_from %} + +The wallet address of the individual who initiated the transaction + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/tx_id.md b/models/descriptions/tx_id.md new file mode 100644 index 0000000..9232e98 --- /dev/null +++ b/models/descriptions/tx_id.md @@ -0,0 +1,5 @@ +{% docs tx_id %} + +A unique key that identifies a transaction. Called "TxHash" on block explorers. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/tx_log.md b/models/descriptions/tx_log.md new file mode 100644 index 0000000..f4024b9 --- /dev/null +++ b/models/descriptions/tx_log.md @@ -0,0 +1,5 @@ +{% docs tx_log %} + +A string that contains the transaction logs, which are logs written by the program interacted with during the transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/tx_succeeded.md b/models/descriptions/tx_succeeded.md new file mode 100644 index 0000000..a429659 --- /dev/null +++ b/models/descriptions/tx_succeeded.md @@ -0,0 +1,5 @@ +{% docs tx_succeeded %} + +Transaction status is "TRUE" if the transaction went through, "FALSE" if the transaction failed. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/validator_hash.md b/models/descriptions/validator_hash.md new file mode 100644 index 0000000..4536422 --- /dev/null +++ b/models/descriptions/validator_hash.md @@ -0,0 +1,5 @@ +{% docs validator_hash %} + +The root hash of the new validator set. + +{% enddocs %} \ No newline at end of file diff --git a/models/silver/silver__blocks.sql b/models/silver/silver__blocks.sql new file mode 100644 index 0000000..8ef46e7 --- /dev/null +++ b/models/silver/silver__blocks.sql @@ -0,0 +1,46 @@ +{{ config( + materialized = 'incremental', + unique_key = "CONCAT_WS('-', chain_id, block_id)", + incremental_strategy = 'delete+insert', + cluster_by = ['block_timestamp::DATE'], +) }} + +SELECT + block_id, + COALESCE( + DATA [0] :result :block :header :time :: TIMESTAMP, + DATA :result :block :header :time :: TIMESTAMP + ) AS block_timestamp, + COALESCE( + DATA [0] :result :block :header :chain_id :: STRING, + DATA :result :block :header :chain_id :: STRING + ) AS chain_id, + ARRAY_SIZE( + COALESCE( + data [0] :result :block :data :txs, + data :result :block :data :txs + ) + ) AS tx_count, + COALESCE( + data [0] :result :block :header :proposer_address :: STRING, + data :result :block :header :proposer_address :: STRING + ) AS proposer_address, + COALESCE( + DATA [0] :result :block :header :validators_hash :: STRING, + DATA :result :block :header :validators_hash :: STRING + ) AS validator_hash, + _inserted_timestamp :: TIMESTAMP AS _inserted_timestamp +FROM + {{ ref('bronze__tendermint_blocks') }} +WHERE + data [0] :error IS NULL + AND DATA :error IS NULL + +{% if is_incremental() %} +AND _inserted_timestamp :: DATE >= ( + SELECT + MAX(_inserted_timestamp) :: DATE - 2 + FROM + {{ this }} +) +{% endif %} diff --git a/models/silver/silver__blocks.yml b/models/silver/silver__blocks.yml new file mode 100644 index 0000000..532fafa --- /dev/null +++ b/models/silver/silver__blocks.yml @@ -0,0 +1,70 @@ +version: 2 +models: + - name: silver__blocks + description: Records of all blocks that have occurred on Evmos, dating back to the genesis block. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - CHAIN_ID + - BLOCK_ID + - sequence_gaps: + column_name: block_ID + where: BLOCK_TIMESTAMP <= CURRENT_DATE - 1 + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: CHAIN_ID + description: "{{ doc('chain_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: TX_COUNT + description: "{{ doc('tx_count') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: PROPOSER_ADDRESS + description: "{{ doc('proposer_address') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: VALIDATOR_HASH + description: "{{ doc('validator_hash') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: _INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ diff --git a/models/silver/silver__msg_attributes.sql b/models/silver/silver__msg_attributes.sql new file mode 100644 index 0000000..4885a9b --- /dev/null +++ b/models/silver/silver__msg_attributes.sql @@ -0,0 +1,41 @@ +{{ config( + materialized = 'incremental', + unique_key = "CONCAT_WS('-', tx_id, msg_index, attribute_index)", + incremental_strategy = 'delete+insert', + cluster_by = ['block_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", +) }} + +SELECT + block_id, + block_timestamp, + tx_id, + tx_succeeded, + msg_group, + msg_sub_group, + msg_index, + msg_type, + b.index AS attribute_index, + TRY_BASE64_DECODE_STRING( + b.value :key :: STRING + ) AS attribute_key, + TRY_BASE64_DECODE_STRING( + b.value :value :: STRING + ) AS attribute_value, + _inserted_timestamp +FROM + {{ ref('silver__msgs') }} A, + LATERAL FLATTEN( + input => A.msg, + path => 'attributes' + ) b + +{% if is_incremental() %} +WHERE + _inserted_timestamp :: DATE >= ( + SELECT + MAX(_inserted_timestamp) :: DATE - 2 + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/silver__msg_attributes.yml b/models/silver/silver__msg_attributes.yml new file mode 100644 index 0000000..913079c --- /dev/null +++ b/models/silver/silver__msg_attributes.yml @@ -0,0 +1,70 @@ +version: 2 +models: + - name: silver__msg_attributes + description: Records of all messages associated to transactions that have occurred on Evmos, dating back to the genesis block. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - MSG_INDEX + - ATTRIBUTE_INDEX + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: MSG_GROUP + description: "{{ doc('msg_group') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: MSG_SUB_GROUP + description: "{{ doc('msg_sub_group') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: MSG_INDEX + description: "{{ doc('msg_index') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: MSG_TYPE + description: "{{ doc('msg_type') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: MSG + description: "The underlying json from the message or event within the transactions" + \ No newline at end of file diff --git a/models/silver/silver__msgs.sql b/models/silver/silver__msgs.sql new file mode 100644 index 0000000..7023b86 --- /dev/null +++ b/models/silver/silver__msgs.sql @@ -0,0 +1,134 @@ +{{ config( + materialized = 'incremental', + unique_key = "CONCAT_WS('-', tx_id, msg_index)", + incremental_strategy = 'delete+insert', + cluster_by = ['block_timestamp::DATE'], +) }} + +WITH base AS ( + + SELECT + t.block_id, + t.block_timestamp, + t.tx_id, + t.gas_used, + t.gas_wanted, + t.tx_succeeded, + f.value AS msg, + f.index :: INT AS msg_index, + msg :type :: STRING AS msg_type, + IFF( + TRY_BASE64_DECODE_STRING( + msg :attributes [0] :key :: STRING + ) = 'action', + TRUE, + FALSE + ) AS is_action, + NULLIF( + (conditional_true_event(is_action) over (PARTITION BY tx_id + ORDER BY + msg_index ASC) -1), + -1 + ) AS msg_group, + IFF( + TRY_BASE64_DECODE_STRING( + msg :attributes [0] :key :: STRING + ) = 'module', + TRUE, + FALSE + ) AS is_module, + TRY_BASE64_DECODE_STRING( + msg :attributes [0] :key :: STRING + ) AS attribute_key, + TRY_BASE64_DECODE_STRING( + msg :attributes [0] :value :: STRING + ) AS attribute_value, + t._inserted_timestamp + FROM + {{ ref('silver__transactions') }} t, + LATERAL FLATTEN(input => msgs) f + +{% if is_incremental() %} +WHERE + _inserted_timestamp :: DATE >= ( + SELECT + MAX(_inserted_timestamp) :: DATE - 2 + FROM + {{ this }} + ) +{% endif %} +), +exec_actions AS ( + SELECT + DISTINCT tx_id, + msg_group + FROM + base + WHERE + msg_type = 'message' + AND attribute_key = 'action' + AND LOWER(attribute_value) LIKE '%exec%' +), +GROUPING AS ( + SELECT + base.tx_id, + base.msg_index, + RANK() over( + PARTITION BY base.tx_id, + base.msg_group + ORDER BY + base.msg_index + ) -1 AS msg_sub_group + FROM + base + INNER JOIN exec_actions e + ON base.tx_id = e.tx_id + AND base.msg_group = e.msg_group + WHERE + base.is_module = 'TRUE' + AND base.msg_type = 'message' +), +FINAL AS ( + SELECT + block_id, + block_timestamp, + A.tx_id, + tx_succeeded, + msg_group, + CASE + WHEN msg_group IS NULL THEN NULL + ELSE COALESCE( + LAST_VALUE( + b.msg_sub_group ignore nulls + ) over( + PARTITION BY A.tx_id, + msg_group + ORDER BY + A.msg_index DESC rows unbounded preceding + ), + 0 + ) + END AS msg_sub_group, + A.msg_index, + msg_type, + msg, + _inserted_timestamp + FROM + base A + LEFT JOIN GROUPING b + ON A.tx_id = b.tx_id + AND A.msg_index = b.msg_index +) +SELECT + block_id, + block_timestamp, + tx_id, + tx_succeeded, + msg_group, + msg_sub_group, + msg_index, + msg_type, + msg, + _inserted_timestamp +FROM + FINAL diff --git a/models/silver/silver__msgs.yml b/models/silver/silver__msgs.yml new file mode 100644 index 0000000..fc07ac5 --- /dev/null +++ b/models/silver/silver__msgs.yml @@ -0,0 +1,76 @@ +version: 2 +models: + - name: silver__msgs + description: Records of all messages associated to transactions that have occurred on Evmos, dating back to the genesis block. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - MSG_INDEX + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: TX_SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - BOOLEAN + - name: MSG_GROUP + description: "{{ doc('msg_group') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: MSG_SUB_GROUP + description: "{{ doc('msg_sub_group') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: MSG_INDEX + description: "{{ doc('msg_index') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: MSG_TYPE + description: "{{ doc('msg_type') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: MSG + description: "The underlying json from the message or event within the transactions" + \ No newline at end of file diff --git a/models/silver/silver__transactions.sql b/models/silver/silver__transactions.sql new file mode 100644 index 0000000..4d28b3d --- /dev/null +++ b/models/silver/silver__transactions.sql @@ -0,0 +1,49 @@ +{{ config( + materialized = 'incremental', + unique_key = "tx_id", + incremental_strategy = 'delete+insert', + cluster_by = 'block_timestamp::DATE', +) }} + +SELECT + b.block_id, + block_timestamp, + tx_id, + b.data :tx_result :codespace :: STRING AS codespace, + b.data :tx_result :gas_used :: NUMBER AS gas_used, + b.data :tx_result :gas_wanted :: NUMBER AS gas_wanted, + CASE + WHEN b.data :tx_result :code :: NUMBER = 0 THEN TRUE + ELSE FALSE + END AS tx_succeeded, + b.data :tx_result :code :: NUMBER AS tx_code, + b.data :tx_result :events AS msgs, + TRY_BASE64_DECODE_STRING( + DATA :tx_result :data + ) AS tx_type, + TRY_PARSE_JSON( + b.data :tx_result :log + ) AS tx_log, + b._inserted_timestamp +FROM + {{ ref('bronze__tendermint_transactions') }} + b + LEFT OUTER JOIN {{ ref('silver__blocks') }} + bb + ON b.block_id = bb.block_id + +{% if is_incremental() %} +WHERE + bb._inserted_timestamp :: DATE >= ( + SELECT + MAX(_inserted_timestamp) :: DATE - 2 + FROM + {{ this }} + ) + AND b._inserted_timestamp :: DATE >= ( + SELECT + MAX(_inserted_timestamp) :: DATE - 2 + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/silver__transactions.yml b/models/silver/silver__transactions.yml new file mode 100644 index 0000000..bd1a1fb --- /dev/null +++ b/models/silver/silver__transactions.yml @@ -0,0 +1,87 @@ +version: 2 +models: + - name: silver__transactions + description: Records of all transactions that have occurred on Evmos, dating back to the genesis block. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: CODESPACE + description: "{{ doc('codespace') }}" + - name: GAS_USED + description: "{{ doc('gas_used') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS_WANTED + description: "{{ doc('gas_wanted') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - BOOLEAN + - name: TX_CODE + description: "{{ doc('tx_code') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: MSGS + description: "The underlying json from the messages or events within the transactions" + tests: + - not_null + - name: TX_TYPE + description: "The transaction type" + tests: + - not_null + - name: TX_LOG + description: "{{ doc('tx_log') }}" + tests: + - not_null + - name: _INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ diff --git a/models/silver/silver__transfers.sql b/models/silver/silver__transfers.sql new file mode 100644 index 0000000..24458ee --- /dev/null +++ b/models/silver/silver__transfers.sql @@ -0,0 +1,362 @@ +{{ config( + materialized = 'incremental', + unique_key = "CONCAT_WS('-', tx_id, block_id, msg_index, currency)", + incremental_strategy = 'delete+insert', + cluster_by = 'block_timestamp::DATE', +) }} + +WITH evmos_txs AS ( + + SELECT + DISTINCT tx_id + FROM + {{ ref('silver__msg_attributes') }} + WHERE + attribute_value IN ( + '/cosmos.bank.v1beta1.MsgSend', + '/cosmos.bank.v1beta1.MsgMultiSend', + '/ibc.applications.transfer.v1.MsgTransfer' + ) + +{% if is_incremental() %} +AND _inserted_timestamp :: DATE >= ( + SELECT + MAX(_inserted_timestamp) :: DATE - 2 + FROM + {{ this }} +) +{% endif %} +), +sender_index AS ( + SELECT + tx_id, + MIN(msg_index) AS msg_index + FROM + {{ ref('silver__msg_attributes') }} + WHERE + msg_type = 'tx' + AND attribute_key = 'acc_seq' + +{% if is_incremental() %} +AND _inserted_timestamp :: DATE >= ( + SELECT + MAX(_inserted_timestamp) :: DATE - 2 + FROM + {{ this }} +) +{% endif %} +GROUP BY + tx_id +), +sender AS ( + SELECT + m.block_id, + m.tx_id, + s.msg_index, + SPLIT_PART( + attribute_value, + '/', + 0 + ) AS sender + FROM + {{ ref('silver__msg_attributes') }} + m + INNER JOIN sender_index s + ON m.tx_id = s.tx_id + AND m.msg_index = s.msg_index + WHERE + msg_type = 'tx' + AND attribute_key = 'acc_seq' + +{% if is_incremental() %} +AND _inserted_timestamp :: DATE >= ( + SELECT + MAX(_inserted_timestamp) :: DATE - 2 + FROM + {{ this }} +) +{% endif %} +), +msg_index AS ( + SELECT + m.block_id, + v.tx_id, + attribute_key, + m.msg_index + FROM + evmos_txs v + LEFT OUTER JOIN {{ ref('silver__msg_attributes') }} + m + ON v.tx_id = m.tx_id + INNER JOIN sender s + ON v.tx_id = s.tx_id + AND m.block_id = s.block_id + WHERE + msg_type = 'transfer' + AND attribute_key = 'amount' + AND m.msg_index > s.msg_index + +{% if is_incremental() %} +AND _inserted_timestamp :: DATE >= ( + SELECT + MAX(_inserted_timestamp) :: DATE - 2 + FROM + {{ this }} +) +{% endif %} +), +receiver AS ( + SELECT + m.block_id, + v.tx_id, + m.msg_index, + attribute_value AS receiver + FROM + evmos_txs v + LEFT OUTER JOIN {{ ref('silver__msg_attributes') }} + m + ON v.tx_id = m.tx_id + INNER JOIN sender s + ON v.tx_id = s.tx_id + AND m.block_id = s.block_id + WHERE + msg_type = 'ibc_transfer' + OR msg_type = 'transfer' + AND attribute_key = 'recipient' + AND m.msg_index > s.msg_index + +{% if is_incremental() %} +AND _inserted_timestamp :: DATE >= ( + SELECT + MAX(_inserted_timestamp) :: DATE - 2 + FROM + {{ this }} +) +{% endif %} +), +amount AS ( + SELECT + m.block_id, + v.tx_id, + m.msg_index, + COALESCE( + SPLIT_PART( + TRIM( + REGEXP_REPLACE( + attribute_value, + '[^[:digit:]]', + ' ' + ) + ), + ' ', + 0 + ), + TRY_PARSE_JSON(attribute_value) :amount + ) AS amount, + COALESCE( + RIGHT(attribute_value, LENGTH(attribute_value) - LENGTH(SPLIT_PART(TRIM(REGEXP_REPLACE(attribute_value, '[^[:digit:]]', ' ')), ' ', 0))), + TRY_PARSE_JSON(attribute_value) [1] :denom + ) AS currency + FROM + evmos_txs v + LEFT OUTER JOIN {{ ref('silver__msg_attributes') }} + m + ON v.tx_id = m.tx_id + INNER JOIN sender s + ON v.tx_id = s.tx_id + AND m.block_id = s.block_id + WHERE + msg_type = 'transfer' + AND attribute_key = 'amount' + AND m.msg_index > s.msg_index + +{% if is_incremental() %} +AND _inserted_timestamp :: DATE >= ( + SELECT + MAX(_inserted_timestamp) :: DATE - 2 + FROM + {{ this }} +) +{% endif %} +), +evmos_txs_final AS ( + SELECT + r.block_id, + block_timestamp, + r.tx_id, + tx_succeeded, + 'EVMOS' AS transfer_type, + r.msg_index, + sender, + amount, + currency, + receiver, + _inserted_timestamp + FROM + receiver r + LEFT OUTER JOIN amount C + ON r.tx_id = C.tx_id + AND r.block_id = C.block_id + AND r.msg_index = C.msg_index + LEFT OUTER JOIN sender s + ON r.tx_id = s.tx_id + AND r.block_id = s.block_id + LEFT OUTER JOIN {{ ref('silver__transactions') }} + t + ON r.tx_id = t.tx_id + AND r.block_id = t.block_id + WHERE + amount IS NOT NULL + AND sender IS NOT NULL + +{% if is_incremental() %} +AND _inserted_timestamp :: DATE >= ( + SELECT + MAX(_inserted_timestamp) :: DATE - 2 + FROM + {{ this }} +) +{% endif %} +), +ibc_in_tx AS ( + SELECT + block_id, + block_timestamp, + tx_id, + tx_succeeded, + msg_index, + 'IBC_TRANSFER_IN' AS transfer_type, + TRY_PARSE_JSON(attribute_value) :sender :: STRING AS sender, + TRY_PARSE_JSON(attribute_value) :amount :: INT AS amount, + CASE + WHEN TRY_PARSE_JSON(attribute_value) :denom :: STRING LIKE '%/%' THEN SPLIT(TRY_PARSE_JSON(attribute_value) :denom :: STRING, '/') [array_size(split(try_parse_json(attribute_value):denom::string, '/')) - 1] + ELSE TRY_PARSE_JSON(attribute_value) :denom :: STRING + END AS currency, + TRY_PARSE_JSON(attribute_value) :receiver :: STRING AS receiver, + _inserted_timestamp + FROM + {{ ref('silver__msg_attributes') }} + WHERE + msg_type = 'write_acknowledgement' + AND attribute_key = 'packet_data' + AND TRY_PARSE_JSON(attribute_value): amount IS NOT NULL + +{% if is_incremental() %} +AND _inserted_timestamp :: DATE >= ( + SELECT + MAX(_inserted_timestamp) :: DATE - 2 + FROM + {{ this }} +) +{% endif %} +), +ibc_out_txid AS ( + SELECT + tx_id + FROM + {{ ref('silver__msg_attributes') }} + WHERE + msg_type = 'ibc_transfer' + +{% if is_incremental() %} +AND _inserted_timestamp :: DATE >= ( + SELECT + MAX(_inserted_timestamp) :: DATE - 2 + FROM + {{ this }} +) +{% endif %} +), +ibc_out_tx AS ( + SELECT + block_id, + block_timestamp, + tx_id, + tx_succeeded, + msg_index, + 'IBC_TRANSFER_OUT' AS transfer_type, + TRY_PARSE_JSON(attribute_value) :sender :: STRING AS sender, + TRY_PARSE_JSON(attribute_value) :amount :: INT AS amount, + CASE + WHEN TRY_PARSE_JSON(attribute_value) :denom :: STRING LIKE '%/%' THEN SPLIT(TRY_PARSE_JSON(attribute_value) :denom :: STRING, '/') [array_size(split(try_parse_json(attribute_value):denom::string, '/')) - 1] + ELSE TRY_PARSE_JSON(attribute_value) :denom :: STRING + END AS currency, + TRY_PARSE_JSON(attribute_value) :receiver :: STRING AS receiver, + _inserted_timestamp + FROM + {{ ref('silver__msg_attributes') }} + WHERE + tx_id IN ( + SELECT + tx_id + FROM + ibc_out_txid + ) + AND msg_type = 'send_packet' + AND attribute_key = 'packet_data' + +{% if is_incremental() %} +AND _inserted_timestamp :: DATE >= ( + SELECT + MAX(_inserted_timestamp) :: DATE - 2 + FROM + {{ this }} +) +{% endif %} +), +ibc_transfers_agg AS ( + SELECT + * + FROM + ibc_out_tx + UNION ALL + SELECT + * + FROM + ibc_in_tx +), +ibc_tx_final AS ( + SELECT + i.block_id, + i.block_timestamp, + i.tx_id, + i.tx_succeeded, + i.transfer_type, + i.sender, + i.amount, + i.currency, + i.receiver, + msg_index, + _inserted_timestamp + FROM + ibc_transfers_agg i +) +SELECT + block_id, + block_timestamp, + tx_id, + tx_succeeded, + transfer_type, + sender, + amount, + currency, + receiver, + msg_index, + _inserted_timestamp +FROM + ibc_tx_final +UNION ALL +SELECT + block_id, + block_timestamp, + tx_id, + tx_succeeded, + transfer_type, + sender, + amount, + currency, + receiver, + msg_index, + _inserted_timestamp +FROM + evmos_txs_final diff --git a/models/silver/silver__transfers.yml b/models/silver/silver__transfers.yml new file mode 100644 index 0000000..c2bf4a0 --- /dev/null +++ b/models/silver/silver__transfers.yml @@ -0,0 +1,92 @@ +version: 2 +models: + - name: silver__transfers + description: Records of all transfers on the Evmos, including IBC transfers as on- and off-ramps to Evmos and wallet to wallet transfers. This schema is in lite mode and contains data back to DATE. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - BLOCK_ID + - MSG_INDEX + - CURRENCY + columns: + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: TX_SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - BOOLEAN + - name: TRANSFER_TYPE + description: "{{ doc('transfer_type') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: SENDER + description: "{{ doc('sender') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: AMOUNT + description: "{{ doc('amount') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: CURRENCY + description: "{{ doc('currency') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: RECEIVER + description: "{{ doc('receiver') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: _INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ \ No newline at end of file diff --git a/models/silver/silver__validators.sql b/models/silver/silver__validators.sql new file mode 100644 index 0000000..242640d --- /dev/null +++ b/models/silver/silver__validators.sql @@ -0,0 +1,26 @@ +{{ config( + materialized = 'table' +) }} + +SELECT + address, + 'evmos' AS blockchain, + 'flipside' AS creator, + 'operator' AS label_type, + 'validator' AS label_subtype, + DATA :description :moniker :: STRING AS label, + DATA :description :identity :: STRING AS project_name, + DATA :delegator_shares :: INT AS delegator_shares, + DATA :jailed :: BOOLEAN AS jailed, + DATA :commission :commission_rates :rate :: FLOAT AS rate, + DATA :commission :commission_rates :max_change_rate :: FLOAT AS max_change_rate, + DATA :commission :commission_rates :max_rate :: FLOAT AS max_rate, + DATA :min_self_delegation :: INT AS min_self_delegation, + RANK() over ( + PARTITION BY address + ORDER BY + DATA :delegator_shares :: INT DESC + ) AS RANK, + DATA AS raw_metadata +FROM + {{ ref('bronze_api__get_validator_metadata_lcd') }} diff --git a/models/silver/silver__validators.yml b/models/silver/silver__validators.yml new file mode 100644 index 0000000..94b3bdb --- /dev/null +++ b/models/silver/silver__validators.yml @@ -0,0 +1,139 @@ +version: 2 +models: + - name: silver__validators + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ADDRESS + - CREATOR + - BLOCKCHAIN + columns: + - name: ADDRESS + description: "{{ doc('address') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: ['evmos'] + - name: CREATOR + description: "{{ doc('creator') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: LABEL_TYPE + description: "{{ doc('label_type') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: ['operator'] + - name: LABEL_SUBTYPE + description: "{{ doc('label_subtype') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: ['validator'] + - name: LABEL + description: "{{ doc('label') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: PROJECT_NAME + description: "{{ doc('project_name') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: DELEGATOR_SHARES + description: The number of tokens staked to the validator. + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: JAILED + description: FALSE when a validator is not jailed, TRUE when a validator is jailed + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - BOOLEAN + - name: RATE + description: The current commission rate the validator is charging stakers. + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: MAX_CHANGE_RATE + description: The maximum rate at which a validator can change their commission per day. + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: MAX_RATE + description: The maximum commission rate that the validator can charge + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: MIN_SELF_DELEGATION + description: The minimum number of Evmos tokens that the operator must be staking with their own validator + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: RANK + description: The rank of the validator in the validator set. Rank is determined by the number of Evmos tokens staked to the validator + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: RAW_METADATA + description: "{{ doc('raw_metadata') }}" + tests: + - not_null + - name: UNIQUE_KEY + description: The unique key of the table + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index b19e00c..f59ad90 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -8,3 +8,21 @@ sources: - name: dim_date_hours - name: address_tags - name: dim_dates + - name: address_labels + - name: bronze + database: evmos + schema: bronze + tables: + - name: sample_blocks + - name: sample_txs + - name: streamline + database: streamline + schema: evmos_dev + tables: + - name: tendermint_block + - name: tendermint_tx_search + - name: osmo + database: osmosis + schema: silver + tables: + - name: asset_metadata diff --git a/packages.yml b/packages.yml index 9159ebb..58d1209 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,7 @@ packages: - package: calogica/dbt_expectations - version: [">=0.4.0", "<0.9.0"] + version: [">=0.8.0", "<0.9.0"] + - package: dbt-labs/dbt_external_tables + version: [">=0.8.0", "<0.9.0"] + - package: dbt-labs/dbt_utils + version: [">=1.0.0", "<1.1.0"] From ed7af7452f1c5e267ad0edd7a8db0fd106ed4b99 Mon Sep 17 00:00:00 2001 From: Jessica Huhnke <91915469+jhuhnke@users.noreply.github.com> Date: Wed, 10 May 2023 15:11:32 -0500 Subject: [PATCH 05/12] change to prod streamline table (#2) --- models/bronze/bronze__tendermint_blocks.sql | 4 ++-- models/bronze/bronze__tendermint_transactions.sql | 5 ++--- models/silver/silver__transactions.sql | 3 ++- models/silver/silver__transactions.yml | 8 ++++---- models/sources.yml | 6 +++--- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/models/bronze/bronze__tendermint_blocks.sql b/models/bronze/bronze__tendermint_blocks.sql index bacae1d..40ec94e 100644 --- a/models/bronze/bronze__tendermint_blocks.sql +++ b/models/bronze/bronze__tendermint_blocks.sql @@ -18,7 +18,7 @@ WITH meta AS ( FROM TABLE( information_schema.external_table_files( - table_name => '{{ source( "streamline", "tendermint_block") }}' + table_name => '{{ source( "streamline", "tendermint_blocks") }}' ) ) A @@ -48,7 +48,7 @@ SELECT FROM {{ source( 'streamline', - 'tendermint_block' + 'tendermint_blocks' ) }} JOIN meta m ON m.file_name = metadata$filename diff --git a/models/bronze/bronze__tendermint_transactions.sql b/models/bronze/bronze__tendermint_transactions.sql index c2ca96a..9dc7827 100644 --- a/models/bronze/bronze__tendermint_transactions.sql +++ b/models/bronze/bronze__tendermint_transactions.sql @@ -18,7 +18,7 @@ WITH meta AS ( FROM TABLE( information_schema.external_table_files( - table_name => '{{ source( "streamline", "tendermint_tx_search") }}' + table_name => '{{ source( "streamline", "tendermint_transactions") }}' ) ) A @@ -43,7 +43,6 @@ SELECT block_number as block_id, value :data :hash :: STRING AS tx_id, metadata, - array_index, DATA, TO_TIMESTAMP( m._inserted_timestamp @@ -51,7 +50,7 @@ SELECT FROM {{ source( 'streamline', - 'tendermint_tx_search' + 'tendermint_transactions' ) }} JOIN meta m ON m.file_name = metadata$filename diff --git a/models/silver/silver__transactions.sql b/models/silver/silver__transactions.sql index 4d28b3d..da974c7 100644 --- a/models/silver/silver__transactions.sql +++ b/models/silver/silver__transactions.sql @@ -32,8 +32,9 @@ FROM bb ON b.block_id = bb.block_id +WHERE tx_id is not null {% if is_incremental() %} -WHERE +AND bb._inserted_timestamp :: DATE >= ( SELECT MAX(_inserted_timestamp) :: DATE - 2 diff --git a/models/silver/silver__transactions.yml b/models/silver/silver__transactions.yml index bd1a1fb..a792c9b 100644 --- a/models/silver/silver__transactions.yml +++ b/models/silver/silver__transactions.yml @@ -61,7 +61,8 @@ models: - name: TX_CODE description: "{{ doc('tx_code') }}" tests: - - not_null + - not_null: + where: TX_SUCCEEDED - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER @@ -72,12 +73,11 @@ models: - not_null - name: TX_TYPE description: "The transaction type" - tests: - - not_null - name: TX_LOG description: "{{ doc('tx_log') }}" tests: - - not_null + - not_null: + where: TX_SUCCEEDED - name: _INSERTED_TIMESTAMP description: "{{ doc('inserted_timestamp') }}" tests: diff --git a/models/sources.yml b/models/sources.yml index f59ad90..cce853d 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -17,10 +17,10 @@ sources: - name: sample_txs - name: streamline database: streamline - schema: evmos_dev + schema: evmos tables: - - name: tendermint_block - - name: tendermint_tx_search + - name: tendermint_blocks + - name: tendermint_transactions - name: osmo database: osmosis schema: silver From d5577d9602ea430a18fcb4cac5174dc438dea8df Mon Sep 17 00:00:00 2001 From: xiuy001 <97179309+xiuy001@users.noreply.github.com> Date: Thu, 11 May 2023 20:32:32 -0400 Subject: [PATCH 06/12] An 3060 get streamline in dbt (#3) * updated * update * added all * updated * updated create udfs * updated api_integrations --- .github/workflows/dbt_run_streamline.yml | 48 ----- .../workflows/dbt_run_streamline_history.yml | 44 +++++ .../workflows/dbt_run_streamline_realtime.yml | 44 +++++ .../dbt_run_streamline_validators.yml | 48 ----- dbt_project.yml | 7 +- macros/create_sps.sql | 2 +- macros/create_udfs.sql | 7 + macros/streamline/api_integrations.sql | 11 ++ macros/streamline/get_base_table_udtf.sql | 24 +++ macros/streamline/models.sql | 171 ++++++++++++++++++ macros/streamline/streamline_udfs.sql | 40 ++++ macros/utils.sql | 43 +++++ .../bronze__streamline_FR_eth_blocks.sql | 10 + ...bronze__streamline_FR_eth_transactions.sql | 10 + ...ronze__streamline_FR_tendermint_blocks.sql | 10 + ..._streamline_FR_tendermint_transactions.sql | 10 + ...e__streamline_FR_tendermint_validators.sql | 10 + .../bronze__streamline_eth_blocks.sql | 12 ++ .../bronze__streamline_eth_transactions.sql | 12 ++ .../bronze__streamline_tendermint_blocks.sql | 12 ++ ...ze__streamline_tendermint_transactions.sql | 12 ++ ...onze__streamline_tendermint_validators.sql | 12 ++ models/sources.yml | 11 ++ .../streamline__complete_eth_blocks.sql | 30 +++ .../streamline__complete_eth_transactions.sql | 30 +++ ...streamline__complete_tendermint_blocks.sql | 30 +++ ...line__complete_tendermint_transactions.sql | 30 +++ ...amline__complete_tendermint_validators.sql | 30 +++ .../streamline__eth_blocks_realtime.sql | 62 +++++++ .../streamline__eth_transactions_realtime.sql | 62 +++++++ ...streamline__tendermint_blocks_realtime.sql | 52 ++++++ ...line__tendermint_transactions_realtime.sql | 50 +++++ ...amline__tendermint_validators_realtime.sql | 50 +++++ models/streamline/streamline__blocks.sql | 21 +++ 34 files changed, 959 insertions(+), 98 deletions(-) delete mode 100644 .github/workflows/dbt_run_streamline.yml create mode 100644 .github/workflows/dbt_run_streamline_history.yml create mode 100644 .github/workflows/dbt_run_streamline_realtime.yml delete mode 100644 .github/workflows/dbt_run_streamline_validators.yml create mode 100644 macros/streamline/api_integrations.sql create mode 100644 macros/streamline/get_base_table_udtf.sql create mode 100644 macros/streamline/models.sql create mode 100644 macros/streamline/streamline_udfs.sql create mode 100644 models/bronze/streamline/bronze__streamline_FR_eth_blocks.sql create mode 100644 models/bronze/streamline/bronze__streamline_FR_eth_transactions.sql create mode 100644 models/bronze/streamline/bronze__streamline_FR_tendermint_blocks.sql create mode 100644 models/bronze/streamline/bronze__streamline_FR_tendermint_transactions.sql create mode 100644 models/bronze/streamline/bronze__streamline_FR_tendermint_validators.sql create mode 100644 models/bronze/streamline/bronze__streamline_eth_blocks.sql create mode 100644 models/bronze/streamline/bronze__streamline_eth_transactions.sql create mode 100644 models/bronze/streamline/bronze__streamline_tendermint_blocks.sql create mode 100644 models/bronze/streamline/bronze__streamline_tendermint_transactions.sql create mode 100644 models/bronze/streamline/bronze__streamline_tendermint_validators.sql create mode 100644 models/streamline/complete/streamline__complete_eth_blocks.sql create mode 100644 models/streamline/complete/streamline__complete_eth_transactions.sql create mode 100644 models/streamline/complete/streamline__complete_tendermint_blocks.sql create mode 100644 models/streamline/complete/streamline__complete_tendermint_transactions.sql create mode 100644 models/streamline/complete/streamline__complete_tendermint_validators.sql create mode 100644 models/streamline/realtime/streamline__eth_blocks_realtime.sql create mode 100644 models/streamline/realtime/streamline__eth_transactions_realtime.sql create mode 100644 models/streamline/realtime/streamline__tendermint_blocks_realtime.sql create mode 100644 models/streamline/realtime/streamline__tendermint_transactions_realtime.sql create mode 100644 models/streamline/realtime/streamline__tendermint_validators_realtime.sql create mode 100644 models/streamline/streamline__blocks.sql diff --git a/.github/workflows/dbt_run_streamline.yml b/.github/workflows/dbt_run_streamline.yml deleted file mode 100644 index 3e12928..0000000 --- a/.github/workflows/dbt_run_streamline.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: dbt_run_streamline -run-name: dbt_run_streamline - -on: - push: - branches: - - main - - turn-off-dev-turn-on-prod - schedule: - # Runs "every 6 hours" (see https://crontab.guru) - - cron: '0 */6 * * *' - -env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" - - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" - SCHEMA: "${{ secrets.SCHEMA }}" - - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v1 - with: - python-version: "3.7.x" - - - name: install dependencies - run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click - dbt deps - - name: Run DBT Jobs - run: | - dbt run -m ./models/streamline/* - - - diff --git a/.github/workflows/dbt_run_streamline_history.yml b/.github/workflows/dbt_run_streamline_history.yml new file mode 100644 index 0000000..5b3254b --- /dev/null +++ b/.github/workflows/dbt_run_streamline_history.yml @@ -0,0 +1,44 @@ +name: dbt_run_streamline_history +run-name: dbt_run_streamline_history + +on: + workflow_dispatch: + schedule: + # Runs "every 6 hours" (see https://crontab.guru) + - cron: '0 1-23/6 * * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_RUN_HISTORY":True}' -m 1+models/streamline/realtime \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_realtime.yml b/.github/workflows/dbt_run_streamline_realtime.yml new file mode 100644 index 0000000..f7cf2bc --- /dev/null +++ b/.github/workflows/dbt_run_streamline_realtime.yml @@ -0,0 +1,44 @@ +name: dbt_run_streamline_realtime +run-name: dbt_run_streamline_realtime + +on: + workflow_dispatch: + schedule: + # Runs "every 1 hour at min 40" (see https://crontab.guru) + - cron: '40 */1 * * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod_backfill + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/streamline/realtime \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_validators.yml b/.github/workflows/dbt_run_streamline_validators.yml deleted file mode 100644 index ec08c55..0000000 --- a/.github/workflows/dbt_run_streamline_validators.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: dbt_run_streamline_validators -run-name: dbt_run_streamline_validators - -on: - push: - branches: - - main - - turn-off-dev-turn-on-prod - schedule: - # Runs "every 6 hours" (see https://crontab.guru) - - cron: '0 0,12,23 * * *' - -env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" - - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" - SCHEMA: "${{ secrets.SCHEMA }}" - - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v1 - with: - python-version: "3.7.x" - - - name: install dependencies - run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click - dbt deps - - name: Run DBT Jobs - run: | - dbt run --full-refresh -x -m 1+models/streamline/streamline__validators_realtime.sql - - - diff --git a/dbt_project.yml b/dbt_project.yml index 0f63641..3b9ebd4 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -41,7 +41,12 @@ models: vars: "dbt_date:time_zone": GMT - "UPDATE_SNOWFLAKE_TAGS": TRUE + STREAMLINE_INVOKE_STREAMS: False + STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: False + UPDATE_UDFS_AND_SPS: False + STREAMLINE_RUN_HISTORY: False + UPDATE_SNOWFLAKE_TAGS: True + WAIT: 0 tests: +store_failures: true # all tests diff --git a/macros/create_sps.sql b/macros/create_sps.sql index b077b9e..04ae357 100644 --- a/macros/create_sps.sql +++ b/macros/create_sps.sql @@ -1,5 +1,5 @@ {% macro create_sps() %} - {% if target.database == 'evmos' %} + {% if target.database == 'EVMOS' %} CREATE SCHEMA IF NOT EXISTS _internal; {{ sp_create_prod_clone('_internal') }}; {% endif %} diff --git a/macros/create_udfs.sql b/macros/create_udfs.sql index 56fe0ff..1f1e9bb 100644 --- a/macros/create_udfs.sql +++ b/macros/create_udfs.sql @@ -1,2 +1,9 @@ {% macro create_udfs() %} + {{ create_udtf_get_base_table( + schema = "streamline" + ) }} + {{ create_udf_get_chainhead() }} + {{ create_udf_json_rpc() }} + {{ create_udf_get_tendermint_transactions() }} + {{ create_udf_get_tendermint_validators() }} {% endmacro %} diff --git a/macros/streamline/api_integrations.sql b/macros/streamline/api_integrations.sql new file mode 100644 index 0000000..1c374b3 --- /dev/null +++ b/macros/streamline/api_integrations.sql @@ -0,0 +1,11 @@ +{% macro create_aws_ethereum_api() %} + {% if target.name == "prod" %} + {% set sql %} + CREATE api integration IF NOT EXISTS aws_evmos_api api_provider = aws_api_gateway api_aws_role_arn = 'arn:aws:iam::490041342817:role/snowflake-api-evmos' api_allowed_prefixes = ( + 'https://55h4rahr50.execute-api.us-east-1.amazonaws.com/dev/', + 'https://n0reh6ugbf.execute-api.us-east-1.amazonaws.com/prod/' + ) enabled = TRUE; +{% endset %} + {% do run_query(sql) %} + {% endif %} +{% endmacro %} diff --git a/macros/streamline/get_base_table_udtf.sql b/macros/streamline/get_base_table_udtf.sql new file mode 100644 index 0000000..a488d14 --- /dev/null +++ b/macros/streamline/get_base_table_udtf.sql @@ -0,0 +1,24 @@ +{% macro create_udtf_get_base_table(schema) %} +create or replace function {{ schema }}.udtf_get_base_table(max_height integer) +returns table (height number) +as +$$ + with base as ( + select + row_number() over ( + order by + seq4() + ) as id + from + table(generator(rowcount => 100000000)) + ) +select + id as height +from + base +where + id <= max_height +$$ +; + +{% endmacro %} \ No newline at end of file diff --git a/macros/streamline/models.sql b/macros/streamline/models.sql new file mode 100644 index 0000000..02cf854 --- /dev/null +++ b/macros/streamline/models.sql @@ -0,0 +1,171 @@ + +{% macro decode_logs_history( + start, + stop + ) %} + WITH look_back AS ( + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 1 + ) +SELECT + l.block_number, + l._log_id, + abi.data AS abi, + l.data +FROM + {{ ref("streamline__decode_logs") }} + l + INNER JOIN {{ ref("silver__abis") }} + abi + ON l.abi_address = abi.contract_address +WHERE + ( + l.block_number BETWEEN {{ start }} + AND {{ stop }} + ) + AND l.block_number <= ( + SELECT + block_number + FROM + look_back + ) + AND _log_id NOT IN ( + SELECT + _log_id + FROM + {{ ref("streamline__complete_decode_logs") }} + WHERE + ( + block_number BETWEEN {{ start }} + AND {{ stop }} + ) + AND block_number <= ( + SELECT + block_number + FROM + look_back + ) + ) +{% endmacro %} + +{% macro streamline_external_table_query( + model, + partition_function, + partition_name, + unique_key + ) %} + WITH meta AS ( + SELECT + job_created_time AS _inserted_timestamp, + file_name, + {{ partition_function }} AS {{ partition_name }} + FROM + TABLE( + information_schema.external_table_file_registration_history( + start_time => DATEADD('day', -7, CURRENT_TIMESTAMP()), + table_name => '{{ source( "bronze_streamline", model) }}') + ) A + ) + SELECT + {{ unique_key }}, + DATA, + _inserted_timestamp, + MD5( + CAST( + COALESCE(CAST({{ unique_key }} AS text), '' :: STRING) AS text + ) + ) AS id, + s.{{ partition_name }}, + s.value AS VALUE + FROM + {{ source( + "bronze_streamline", + model + ) }} + s + JOIN meta b + ON b.file_name = metadata$filename + AND b.{{ partition_name }} = s.{{ partition_name }} + WHERE + b.{{ partition_name }} = s.{{ partition_name }} + AND ( + DATA :error :code IS NULL + OR DATA :error :code NOT IN ( + '-32000', + '-32001', + '-32002', + '-32003', + '-32004', + '-32005', + '-32006', + '-32007', + '-32008', + '-32009', + '-32010' + ) + ) +{% endmacro %} + +{% macro streamline_external_table_FR_query( + model, + partition_function, + partition_name, + unique_key + ) %} + WITH meta AS ( + SELECT + registered_on AS _inserted_timestamp, + file_name, + {{ partition_function }} AS {{ partition_name }} + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", model) }}' + ) + ) A + ) +SELECT + {{ unique_key }}, + DATA, + _inserted_timestamp, + MD5( + CAST( + COALESCE(CAST({{ unique_key }} AS text), '' :: STRING) AS text + ) + ) AS id, + s.{{ partition_name }}, + s.value AS VALUE +FROM + {{ source( + "bronze_streamline", + model + ) }} + s + JOIN meta b + ON b.file_name = metadata$filename + AND b.{{ partition_name }} = s.{{ partition_name }} +WHERE + b.{{ partition_name }} = s.{{ partition_name }} + AND ( + DATA :error :code IS NULL + OR DATA :error :code NOT IN ( + '-32000', + '-32001', + '-32002', + '-32003', + '-32004', + '-32005', + '-32006', + '-32007', + '-32008', + '-32009', + '-32010' + ) + ) +{% endmacro %} \ No newline at end of file diff --git a/macros/streamline/streamline_udfs.sql b/macros/streamline/streamline_udfs.sql new file mode 100644 index 0000000..823d8b9 --- /dev/null +++ b/macros/streamline/streamline_udfs.sql @@ -0,0 +1,40 @@ +{% macro create_udf_get_chainhead() %} + CREATE EXTERNAL FUNCTION IF NOT EXISTS streamline.udf_get_chainhead() returns variant api_integration = aws_evmos_api AS {% if target.name == "prod" %} + 'https://n0reh6ugbf.execute-api.us-east-1.amazonaws.com/prod/get_chainhead' + {% else %} + 'https://55h4rahr50.execute-api.us-east-1.amazonaws.com/dev/get_chainhead' + {%- endif %}; +{% endmacro %} + +{% macro create_udf_json_rpc() %} + CREATE + OR REPLACE EXTERNAL FUNCTION streamline.udf_json_rpc( + json OBJECT + ) returns ARRAY api_integration = aws_evmos_api AS {% if target.name == "prod" %} + 'https://n0reh6ugbf.execute-api.us-east-1.amazonaws.com/prod/bulk_get_json_rpc' + {% else %} + 'https://55h4rahr50.execute-api.us-east-1.amazonaws.com/dev/bulk_get_json_rpc' + {%- endif %}; +{% endmacro %} + +{% macro create_udf_get_tendermint_transactions() %} + CREATE + OR REPLACE EXTERNAL FUNCTION streamline.bulk_get_tendermint_transactions( + json OBJECT + ) returns ARRAY api_integration = aws_evmos_api AS {% if target.name == "prod" %} + 'https://n0reh6ugbf.execute-api.us-east-1.amazonaws.com/prod/bulk_get_tendermint_transactions' + {% else %} + 'https://55h4rahr50.execute-api.us-east-1.amazonaws.com/dev/bulk_get_tendermint_transactions' + {%- endif %}; +{% endmacro %} + +{% macro create_udf_get_tendermint_validators() %} + CREATE + OR REPLACE EXTERNAL FUNCTION streamline.bulk_get_tendermint_validators( + json OBJECT + ) returns ARRAY api_integration = aws_evmos_api AS {% if target.name == "prod" %} + 'https://n0reh6ugbf.execute-api.us-east-1.amazonaws.com/prod/bulk_get_tendermint_validators' + {% else %} + 'https://55h4rahr50.execute-api.us-east-1.amazonaws.com/dev/bulk_get_tendermint_validators' + {%- endif %}; +{% endmacro %} \ No newline at end of file diff --git a/macros/utils.sql b/macros/utils.sql index 85549f1..0b800b8 100644 --- a/macros/utils.sql +++ b/macros/utils.sql @@ -33,3 +33,46 @@ NULL {% endif %} {% endmacro %} + +{% macro if_data_call_wait() %} + {% if var( + "STREAMLINE_INVOKE_STREAMS" + ) %} + {% set query %} + SELECT + 1 + WHERE + EXISTS( + SELECT + 1 + FROM + {{ model.schema ~ "." ~ model.alias }} + LIMIT + 1 + ) {% endset %} + {% if execute %} + {% set results = run_query( + query + ) %} + {% if results %} + {{ log( + "Waiting...", + info = True + ) }} + + {% set wait_query %} + SELECT + system$wait( + {{ var( + "WAIT", + 600 + ) }} + ) {% endset %} + {% do run_query(wait_query) %} + {% else %} + SELECT + NULL; + {% endif %} + {% endif %} + {% endif %} +{% endmacro %} diff --git a/models/bronze/streamline/bronze__streamline_FR_eth_blocks.sql b/models/bronze/streamline/bronze__streamline_FR_eth_blocks.sql new file mode 100644 index 0000000..c190e9e --- /dev/null +++ b/models/bronze/streamline/bronze__streamline_FR_eth_blocks.sql @@ -0,0 +1,10 @@ +{{ config ( + materialized = 'view' +) }} + +{{ streamline_external_table_FR_query( + "eth_blocks", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", + partition_name = "_partition_by_block_id", + unique_key = "block_number" +) }} \ No newline at end of file diff --git a/models/bronze/streamline/bronze__streamline_FR_eth_transactions.sql b/models/bronze/streamline/bronze__streamline_FR_eth_transactions.sql new file mode 100644 index 0000000..8c54a36 --- /dev/null +++ b/models/bronze/streamline/bronze__streamline_FR_eth_transactions.sql @@ -0,0 +1,10 @@ +{{ config ( + materialized = 'view' +) }} + +{{ streamline_external_table_FR_query( + "eth_transactions", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", + partition_name = "_partition_by_block_id", + unique_key = "block_number" +) }} \ No newline at end of file diff --git a/models/bronze/streamline/bronze__streamline_FR_tendermint_blocks.sql b/models/bronze/streamline/bronze__streamline_FR_tendermint_blocks.sql new file mode 100644 index 0000000..c199622 --- /dev/null +++ b/models/bronze/streamline/bronze__streamline_FR_tendermint_blocks.sql @@ -0,0 +1,10 @@ +{{ config ( + materialized = 'view' +) }} + +{{ streamline_external_table_FR_query( + "tendermint_blocks", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", + partition_name = "_partition_by_block_id", + unique_key = "block_number" +) }} \ No newline at end of file diff --git a/models/bronze/streamline/bronze__streamline_FR_tendermint_transactions.sql b/models/bronze/streamline/bronze__streamline_FR_tendermint_transactions.sql new file mode 100644 index 0000000..85ea40f --- /dev/null +++ b/models/bronze/streamline/bronze__streamline_FR_tendermint_transactions.sql @@ -0,0 +1,10 @@ +{{ config ( + materialized = 'view' +) }} + +{{ streamline_external_table_FR_query( + "tendermint_transactions", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", + partition_name = "_partition_by_block_id", + unique_key = "block_number" +) }} \ No newline at end of file diff --git a/models/bronze/streamline/bronze__streamline_FR_tendermint_validators.sql b/models/bronze/streamline/bronze__streamline_FR_tendermint_validators.sql new file mode 100644 index 0000000..a186098 --- /dev/null +++ b/models/bronze/streamline/bronze__streamline_FR_tendermint_validators.sql @@ -0,0 +1,10 @@ +{{ config ( + materialized = 'view' +) }} + +{{ streamline_external_table_FR_query( + "tendermint_validators", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", + partition_name = "_partition_by_block_id", + unique_key = "block_number" +) }} \ No newline at end of file diff --git a/models/bronze/streamline/bronze__streamline_eth_blocks.sql b/models/bronze/streamline/bronze__streamline_eth_blocks.sql new file mode 100644 index 0000000..9de31ba --- /dev/null +++ b/models/bronze/streamline/bronze__streamline_eth_blocks.sql @@ -0,0 +1,12 @@ + +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_") [-1] %} +{{ streamline_external_table_query( + "eth_blocks", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", + partition_name = "_partition_by_block_id", + unique_key = "block_number" +) }} \ No newline at end of file diff --git a/models/bronze/streamline/bronze__streamline_eth_transactions.sql b/models/bronze/streamline/bronze__streamline_eth_transactions.sql new file mode 100644 index 0000000..927fd0c --- /dev/null +++ b/models/bronze/streamline/bronze__streamline_eth_transactions.sql @@ -0,0 +1,12 @@ + +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_") [-1] %} +{{ streamline_external_table_query( + "eth_transactions", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", + partition_name = "_partition_by_block_id", + unique_key = "block_number" +) }} \ No newline at end of file diff --git a/models/bronze/streamline/bronze__streamline_tendermint_blocks.sql b/models/bronze/streamline/bronze__streamline_tendermint_blocks.sql new file mode 100644 index 0000000..e21f1f7 --- /dev/null +++ b/models/bronze/streamline/bronze__streamline_tendermint_blocks.sql @@ -0,0 +1,12 @@ + +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_") [-1] %} +{{ streamline_external_table_query( + "tendermint_blocks", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", + partition_name = "_partition_by_block_id", + unique_key = "block_number" +) }} \ No newline at end of file diff --git a/models/bronze/streamline/bronze__streamline_tendermint_transactions.sql b/models/bronze/streamline/bronze__streamline_tendermint_transactions.sql new file mode 100644 index 0000000..0a66bd1 --- /dev/null +++ b/models/bronze/streamline/bronze__streamline_tendermint_transactions.sql @@ -0,0 +1,12 @@ + +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_") [-1] %} +{{ streamline_external_table_query( + "tendermint_transactions", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", + partition_name = "_partition_by_block_id", + unique_key = "block_number" +) }} \ No newline at end of file diff --git a/models/bronze/streamline/bronze__streamline_tendermint_validators.sql b/models/bronze/streamline/bronze__streamline_tendermint_validators.sql new file mode 100644 index 0000000..b19ed78 --- /dev/null +++ b/models/bronze/streamline/bronze__streamline_tendermint_validators.sql @@ -0,0 +1,12 @@ + +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_") [-1] %} +{{ streamline_external_table_query( + "tendermint_validators", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", + partition_name = "_partition_by_block_id", + unique_key = "block_number" +) }} \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index cce853d..04155ed 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -7,7 +7,18 @@ sources: tables: - name: dim_date_hours - name: address_tags + - name: address_labels - name: dim_dates + - name: bronze_streamline + database: streamline + schema: | + {{ "EVMOS_DEV" if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else "EVMOS" }} + tables: + - name: eth_blocks + - name: eth_transactions + - name: tendermint_blocks + - name: tendermint_transactions + - name: tendermint_validators - name: address_labels - name: bronze database: evmos diff --git a/models/streamline/complete/streamline__complete_eth_blocks.sql b/models/streamline/complete/streamline__complete_eth_blocks.sql new file mode 100644 index 0000000..8b25bf3 --- /dev/null +++ b/models/streamline/complete/streamline__complete_eth_blocks.sql @@ -0,0 +1,30 @@ +-- depends_on: {{ ref('bronze__streamline_eth_blocks') }} +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)" +) }} + +SELECT + id, + block_number, + _inserted_timestamp +FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_eth_blocks') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) +{% else %} + {{ ref('bronze__streamline_FR_eth_blocks') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/complete/streamline__complete_eth_transactions.sql b/models/streamline/complete/streamline__complete_eth_transactions.sql new file mode 100644 index 0000000..95b14f4 --- /dev/null +++ b/models/streamline/complete/streamline__complete_eth_transactions.sql @@ -0,0 +1,30 @@ +-- depends_on: {{ ref('bronze__streamline_eth_transactions') }} +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)" +) }} + +SELECT + id, + block_number, + _inserted_timestamp +FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_eth_transactions') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) +{% else %} + {{ ref('bronze__streamline_FR_eth_transactions') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 \ No newline at end of file diff --git a/models/streamline/complete/streamline__complete_tendermint_blocks.sql b/models/streamline/complete/streamline__complete_tendermint_blocks.sql new file mode 100644 index 0000000..f7cc942 --- /dev/null +++ b/models/streamline/complete/streamline__complete_tendermint_blocks.sql @@ -0,0 +1,30 @@ +-- depends_on: {{ ref('bronze__streamline_tendermint_blocks') }} +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)" +) }} + +SELECT + id, + block_number, + _inserted_timestamp +FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_tendermint_blocks') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) +{% else %} + {{ ref('bronze__streamline_FR_tendermint_blocks') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 \ No newline at end of file diff --git a/models/streamline/complete/streamline__complete_tendermint_transactions.sql b/models/streamline/complete/streamline__complete_tendermint_transactions.sql new file mode 100644 index 0000000..02d4bde --- /dev/null +++ b/models/streamline/complete/streamline__complete_tendermint_transactions.sql @@ -0,0 +1,30 @@ +-- depends_on: {{ ref('bronze__streamline_tendermint_transactions') }} +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)" +) }} + +SELECT + id, + block_number, + _inserted_timestamp +FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_tendermint_transactions') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) +{% else %} + {{ ref('bronze__streamline_FR_tendermint_transactions') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 \ No newline at end of file diff --git a/models/streamline/complete/streamline__complete_tendermint_validators.sql b/models/streamline/complete/streamline__complete_tendermint_validators.sql new file mode 100644 index 0000000..b0697ba --- /dev/null +++ b/models/streamline/complete/streamline__complete_tendermint_validators.sql @@ -0,0 +1,30 @@ +-- depends_on: {{ ref('bronze__streamline_tendermint_validators') }} +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)" +) }} + +SELECT + id, + block_number, + _inserted_timestamp +FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_tendermint_validators') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) +{% else %} + {{ ref('bronze__streamline_FR_tendermint_validators') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 \ No newline at end of file diff --git a/models/streamline/realtime/streamline__eth_blocks_realtime.sql b/models/streamline/realtime/streamline__eth_blocks_realtime.sql new file mode 100644 index 0000000..6daf73f --- /dev/null +++ b/models/streamline/realtime/streamline__eth_blocks_realtime.sql @@ -0,0 +1,62 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_json_rpc(object_construct('url_route','eth_rpc', 'sql_source', '{{this.identifier}}', 'external_table', 'eth_blocks', 'method', 'eth_getBlockByNumber', 'producer_batch_size',1000, 'producer_limit_size', 1000000, 'worker_batch_size',100, 'producer_batch_chunks_size', 1000))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH last_3_days AS ({% if var('STREAMLINE_RUN_HISTORY') %} + + SELECT + 0 AS block_number + {% else %} + SELECT + MAX(block_number) - 100000 AS block_number --aprox 3 days + FROM + {{ ref("streamline__blocks") }} + {% endif %}), + tbl AS ( + SELECT + block_number, + block_number_hex + FROM + {{ ref("streamline__blocks") }} + WHERE + ( + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + ) + AND block_number IS NOT NULL + EXCEPT + SELECT + block_number, + REPLACE( + concat_ws('', '0x', to_char(block_number, 'XXXXXXXX')), + ' ', + '' + ) AS block_number_hex + FROM + {{ ref("streamline__complete_eth_blocks") }} + WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + ) +SELECT + block_number, + 'eth_getBlockByNumber' AS method, + CONCAT( + block_number_hex, + '_-_', + 'false' + ) AS params +FROM + tbl diff --git a/models/streamline/realtime/streamline__eth_transactions_realtime.sql b/models/streamline/realtime/streamline__eth_transactions_realtime.sql new file mode 100644 index 0000000..60843c2 --- /dev/null +++ b/models/streamline/realtime/streamline__eth_transactions_realtime.sql @@ -0,0 +1,62 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'eth_transactions', 'exploded_key','[\"result\", \"transactions\"]', 'method', 'eth_getBlockByNumber', 'producer_batch_size',1000, 'producer_limit_size', 1000000, 'worker_batch_size',100))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH last_3_days AS ({% if var('STREAMLINE_RUN_HISTORY') %} + + SELECT + 0 AS block_number + {% else %} + SELECT + MAX(block_number) - 100000 AS block_number --aprox 3 days + FROM + {{ ref("streamline__blocks") }} + {% endif %}), + tbl AS ( + SELECT + block_number, + block_number_hex + FROM + {{ ref("streamline__blocks") }} + WHERE + ( + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + ) + AND block_number IS NOT NULL + EXCEPT + SELECT + block_number, + REPLACE( + concat_ws('', '0x', to_char(block_number, 'XXXXXXXX')), + ' ', + '' + ) AS block_number_hex + FROM + {{ ref("streamline__complete_eth_transactions") }} + WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + ) +SELECT + block_number, + 'eth_getBlockByNumber' AS method, + CONCAT( + block_number_hex, + '_-_', + 'true' + ) AS params +FROM + tbl diff --git a/models/streamline/realtime/streamline__tendermint_blocks_realtime.sql b/models/streamline/realtime/streamline__tendermint_blocks_realtime.sql new file mode 100644 index 0000000..cf3b392 --- /dev/null +++ b/models/streamline/realtime/streamline__tendermint_blocks_realtime.sql @@ -0,0 +1,52 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_json_rpc(object_construct('url_route','tendermint_rpc', 'sql_source', '{{this.identifier}}', 'external_table', 'tendermint_blocks', 'method', 'eth_getBlockByNumber', 'producer_batch_size',1000, 'producer_limit_size', 1000000, 'worker_batch_size',100, 'producer_batch_chunks_size', 1000))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH last_3_days AS ({% if var('STREAMLINE_RUN_HISTORY') %} + + SELECT + 0 AS block_number + {% else %} + SELECT + MAX(block_number) - 100000 AS block_number --aprox 3 days + FROM + {{ ref("streamline__blocks") }} + {% endif %}), + tbl AS ( + SELECT + block_number + FROM + {{ ref("streamline__blocks") }} + WHERE + ( + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + ) + AND block_number IS NOT NULL + EXCEPT + SELECT + block_number + FROM + {{ ref("streamline__complete_tendermint_blocks") }} + WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + ) +SELECT + block_number, + 'block' AS method, + block_number :: STRING AS params +FROM + tbl diff --git a/models/streamline/realtime/streamline__tendermint_transactions_realtime.sql b/models/streamline/realtime/streamline__tendermint_transactions_realtime.sql new file mode 100644 index 0000000..174b520 --- /dev/null +++ b/models/streamline/realtime/streamline__tendermint_transactions_realtime.sql @@ -0,0 +1,50 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.bulk_get_tendermint_transactions(object_construct('sql_source', '{{this.identifier}}'))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH last_3_days AS ({% if var('STREAMLINE_RUN_HISTORY') %} + + SELECT + 0 AS block_number + {% else %} + SELECT + MAX(block_number) - 100000 AS block_number --aprox 3 days + FROM + {{ ref("streamline__blocks") }} + {% endif %}), + tbl AS ( + SELECT + block_number + FROM + {{ ref("streamline__blocks") }} + WHERE + ( + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + ) + AND block_number IS NOT NULL + EXCEPT + SELECT + block_number + FROM + {{ ref("streamline__complete_tendermint_transactions") }} + WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + ) +SELECT + block_number +FROM + tbl diff --git a/models/streamline/realtime/streamline__tendermint_validators_realtime.sql b/models/streamline/realtime/streamline__tendermint_validators_realtime.sql new file mode 100644 index 0000000..9f53896 --- /dev/null +++ b/models/streamline/realtime/streamline__tendermint_validators_realtime.sql @@ -0,0 +1,50 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.bulk_get_tendermint_validators(object_construct('sql_source', '{{this.identifier}}'))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH last_3_days AS ({% if var('STREAMLINE_RUN_HISTORY') %} + + SELECT + 0 AS block_number + {% else %} + SELECT + MAX(block_number) - 100000 AS block_number --aprox 3 days + FROM + {{ ref("streamline__blocks") }} + {% endif %}), + tbl AS ( + SELECT + block_number + FROM + {{ ref("streamline__blocks") }} + WHERE + ( + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + ) + AND block_number IS NOT NULL + EXCEPT + SELECT + block_number + FROM + {{ ref("streamline__complete_tendermint_validators") }} + WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + ) +SELECT + block_number +FROM + tbl diff --git a/models/streamline/streamline__blocks.sql b/models/streamline/streamline__blocks.sql new file mode 100644 index 0000000..d43cee8 --- /dev/null +++ b/models/streamline/streamline__blocks.sql @@ -0,0 +1,21 @@ +{{ config ( + materialized = "view", + tags = ['streamline_view'] +) }} + +{% if execute %} + {% set height = run_query('SELECT streamline.udf_get_chainhead()') %} + {% set block_height = height.columns [0].values() [0] %} +{% else %} + {% set block_height = 0 %} +{% endif %} + +SELECT + height AS block_number, + REPLACE( + concat_ws('', '0x', to_char(block_number, 'XXXXXXXX')), + ' ', + '' + ) AS block_number_hex +FROM + TABLE(streamline.udtf_get_base_table({{ block_height }})) From de363549ec1d19683ab14231fb93b9ac985644c7 Mon Sep 17 00:00:00 2001 From: xiuy001 <97179309+xiuy001@users.noreply.github.com> Date: Fri, 12 May 2023 13:56:30 -0400 Subject: [PATCH 07/12] Update vars GitHub actions (#4) * updated * updated * removed the branch * updated --- .github/workflows/dbt_run_streamline_realtime.yml | 2 +- macros/streamline/streamline_udfs.sql | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dbt_run_streamline_realtime.yml b/.github/workflows/dbt_run_streamline_realtime.yml index f7cf2bc..67414b1 100644 --- a/.github/workflows/dbt_run_streamline_realtime.yml +++ b/.github/workflows/dbt_run_streamline_realtime.yml @@ -26,7 +26,7 @@ jobs: run_dbt_jobs: runs-on: ubuntu-latest environment: - name: workflow_prod_backfill + name: workflow_prod steps: - uses: actions/checkout@v3 diff --git a/macros/streamline/streamline_udfs.sql b/macros/streamline/streamline_udfs.sql index 823d8b9..df13c22 100644 --- a/macros/streamline/streamline_udfs.sql +++ b/macros/streamline/streamline_udfs.sql @@ -7,8 +7,7 @@ {% endmacro %} {% macro create_udf_json_rpc() %} - CREATE - OR REPLACE EXTERNAL FUNCTION streamline.udf_json_rpc( + CREATE EXTERNAL FUNCTION IF NOT EXISTS streamline.udf_json_rpc( json OBJECT ) returns ARRAY api_integration = aws_evmos_api AS {% if target.name == "prod" %} 'https://n0reh6ugbf.execute-api.us-east-1.amazonaws.com/prod/bulk_get_json_rpc' @@ -18,8 +17,7 @@ {% endmacro %} {% macro create_udf_get_tendermint_transactions() %} - CREATE - OR REPLACE EXTERNAL FUNCTION streamline.bulk_get_tendermint_transactions( + CREATE EXTERNAL FUNCTION IF NOT EXISTS streamline.bulk_get_tendermint_transactions( json OBJECT ) returns ARRAY api_integration = aws_evmos_api AS {% if target.name == "prod" %} 'https://n0reh6ugbf.execute-api.us-east-1.amazonaws.com/prod/bulk_get_tendermint_transactions' @@ -29,8 +27,7 @@ {% endmacro %} {% macro create_udf_get_tendermint_validators() %} - CREATE - OR REPLACE EXTERNAL FUNCTION streamline.bulk_get_tendermint_validators( + CREATE EXTERNAL FUNCTION IF NOT EXISTS streamline.bulk_get_tendermint_validators( json OBJECT ) returns ARRAY api_integration = aws_evmos_api AS {% if target.name == "prod" %} 'https://n0reh6ugbf.execute-api.us-east-1.amazonaws.com/prod/bulk_get_tendermint_validators' From 643cbb1f8285f4840e6452c11d809e72bfdc4936 Mon Sep 17 00:00:00 2001 From: eric-laurello <102970824+eric-laurello@users.noreply.github.com> Date: Fri, 12 May 2023 14:00:18 -0400 Subject: [PATCH 08/12] update workflows to use vars (#5) --- .github/workflows/dbt_run_incremental.yml | 45 +++++++++-------------- .github/workflows/dbt_test.yml | 23 +++++++----- 2 files changed, 31 insertions(+), 37 deletions(-) diff --git a/.github/workflows/dbt_run_incremental.yml b/.github/workflows/dbt_run_incremental.yml index e6f108e..1c5e21d 100644 --- a/.github/workflows/dbt_run_incremental.yml +++ b/.github/workflows/dbt_run_incremental.yml @@ -7,35 +7,26 @@ on: - cron: '0 1,7,13,19 * * *' env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" - - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" - SCHEMA: "${{ secrets.SCHEMA }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + with: + dbt_command: > + dbt run -s ./models + environment: workflow_prod + warehouse: ${{ vars.WAREHOUSE }} + secrets: inherit - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v1 - with: - python-version: "3.7.x" - - - name: install dependencies - run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click - dbt deps - - name: Run DBT Jobs - run: | - dbt run -s ./models diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml index 93241a8..ec7ff03 100644 --- a/.github/workflows/dbt_test.yml +++ b/.github/workflows/dbt_test.yml @@ -7,17 +7,20 @@ on: - cron: '0 4 * * *' env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" - - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" - SCHEMA: "${{ secrets.SCHEMA }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" +concurrency: + group: ${{ github.workflow }} jobs: run_dbt_jobs: @@ -34,7 +37,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click dbt deps - name: Run DBT Jobs run: | From 109a73a49e2f2143e9dbe795e48801c1df2a1675 Mon Sep 17 00:00:00 2001 From: jhuhnke Date: Mon, 15 May 2023 10:08:34 -0500 Subject: [PATCH 09/12] remove referece --- models/core/core__fact_validators.sql | 3 +-- models/core/core__fact_validators.yml | 4 ---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/models/core/core__fact_validators.sql b/models/core/core__fact_validators.sql index 38a6f7e..559f7f5 100644 --- a/models/core/core__fact_validators.sql +++ b/models/core/core__fact_validators.sql @@ -17,7 +17,6 @@ SELECT max_rate, min_self_delegation, RANK, - raw_metadata, - unique_key + raw_metadata FROM {{ ref('silver__validators') }} diff --git a/models/core/core__fact_validators.yml b/models/core/core__fact_validators.yml index 76e391c..8c65e3d 100644 --- a/models/core/core__fact_validators.yml +++ b/models/core/core__fact_validators.yml @@ -2,10 +2,6 @@ version: 2 models: - name: core__fact_validators description: Information about all active and inactive set validators on Evmos. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - UNIQUE_KEY columns: - name: ADDRESS description: "{{ doc('address') }}" From a038e0d53b146258446177c1a1fb215db9a8431f Mon Sep 17 00:00:00 2001 From: xiuy001 <97179309+xiuy001@users.noreply.github.com> Date: Wed, 17 May 2023 16:28:31 -0400 Subject: [PATCH 10/12] Params tuning backfill (#6) * updated * updated --- models/streamline/realtime/streamline__eth_blocks_realtime.sql | 2 +- .../realtime/streamline__eth_transactions_realtime.sql | 2 +- .../realtime/streamline__tendermint_blocks_realtime.sql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/models/streamline/realtime/streamline__eth_blocks_realtime.sql b/models/streamline/realtime/streamline__eth_blocks_realtime.sql index 6daf73f..6506ca6 100644 --- a/models/streamline/realtime/streamline__eth_blocks_realtime.sql +++ b/models/streamline/realtime/streamline__eth_blocks_realtime.sql @@ -1,7 +1,7 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_json_rpc(object_construct('url_route','eth_rpc', 'sql_source', '{{this.identifier}}', 'external_table', 'eth_blocks', 'method', 'eth_getBlockByNumber', 'producer_batch_size',1000, 'producer_limit_size', 1000000, 'worker_batch_size',100, 'producer_batch_chunks_size', 1000))", + func = "{{this.schema}}.udf_json_rpc(object_construct('url_route','eth_rpc', 'sql_source', '{{this.identifier}}', 'external_table', 'eth_blocks', 'method', 'eth_getBlockByNumber', 'producer_batch_size',10000, 'producer_limit_size', 2000000, 'worker_batch_size',100))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} diff --git a/models/streamline/realtime/streamline__eth_transactions_realtime.sql b/models/streamline/realtime/streamline__eth_transactions_realtime.sql index 60843c2..72ff6ed 100644 --- a/models/streamline/realtime/streamline__eth_transactions_realtime.sql +++ b/models/streamline/realtime/streamline__eth_transactions_realtime.sql @@ -1,7 +1,7 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'eth_transactions', 'exploded_key','[\"result\", \"transactions\"]', 'method', 'eth_getBlockByNumber', 'producer_batch_size',1000, 'producer_limit_size', 1000000, 'worker_batch_size',100))", + func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'eth_transactions', 'exploded_key','[\"result\", \"transactions\"]', 'method', 'eth_getBlockByNumber', 'producer_batch_size',10000, 'producer_limit_size', 2000000, 'worker_batch_size',100))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} diff --git a/models/streamline/realtime/streamline__tendermint_blocks_realtime.sql b/models/streamline/realtime/streamline__tendermint_blocks_realtime.sql index cf3b392..00d96f1 100644 --- a/models/streamline/realtime/streamline__tendermint_blocks_realtime.sql +++ b/models/streamline/realtime/streamline__tendermint_blocks_realtime.sql @@ -1,7 +1,7 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_json_rpc(object_construct('url_route','tendermint_rpc', 'sql_source', '{{this.identifier}}', 'external_table', 'tendermint_blocks', 'method', 'eth_getBlockByNumber', 'producer_batch_size',1000, 'producer_limit_size', 1000000, 'worker_batch_size',100, 'producer_batch_chunks_size', 1000))", + func = "{{this.schema}}.udf_json_rpc(object_construct('url_route','tendermint_rpc', 'sql_source', '{{this.identifier}}', 'external_table', 'tendermint_blocks', 'method', 'eth_getBlockByNumber', 'producer_batch_size',10000, 'producer_limit_size', 2000000, 'worker_batch_size',100, 'producer_batch_chunks_size', 1000))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} From 5a311980c99f55965a2ab2bfb12e410401a5d46c Mon Sep 17 00:00:00 2001 From: jhuhnke Date: Mon, 22 May 2023 09:05:36 -0500 Subject: [PATCH 11/12] unneeded field in transactions table --- models/core/core__fact_transactions.sql | 3 +- models/silver/silver__transactions.sql | 95 ++++++++++++++++--------- 2 files changed, 61 insertions(+), 37 deletions(-) diff --git a/models/core/core__fact_transactions.sql b/models/core/core__fact_transactions.sql index 784dc4a..104068e 100644 --- a/models/core/core__fact_transactions.sql +++ b/models/core/core__fact_transactions.sql @@ -58,8 +58,7 @@ SELECT ) AS fee, gas_used, gas_wanted, - tx_code, - tx_type, + tx_code msgs FROM {{ ref('silver__transactions') }} diff --git a/models/silver/silver__transactions.sql b/models/silver/silver__transactions.sql index da974c7..e49a910 100644 --- a/models/silver/silver__transactions.sql +++ b/models/silver/silver__transactions.sql @@ -5,46 +5,71 @@ cluster_by = 'block_timestamp::DATE', ) }} -SELECT - b.block_id, - block_timestamp, - tx_id, - b.data :tx_result :codespace :: STRING AS codespace, - b.data :tx_result :gas_used :: NUMBER AS gas_used, - b.data :tx_result :gas_wanted :: NUMBER AS gas_wanted, - CASE - WHEN b.data :tx_result :code :: NUMBER = 0 THEN TRUE - ELSE FALSE - END AS tx_succeeded, - b.data :tx_result :code :: NUMBER AS tx_code, - b.data :tx_result :events AS msgs, - TRY_BASE64_DECODE_STRING( - DATA :tx_result :data - ) AS tx_type, - TRY_PARSE_JSON( - b.data :tx_result :log - ) AS tx_log, - b._inserted_timestamp -FROM - {{ ref('bronze__tendermint_transactions') }} - b - LEFT OUTER JOIN {{ ref('silver__blocks') }} - bb - ON b.block_id = bb.block_id +WITH base_transactions AS ( + + SELECT + block_id, + t.value :hash :: STRING AS tx_id, + t.value :tx_result :codespace AS codespace, + t.value :tx_result :gas_used :: NUMBER AS gas_used, + t.value :tx_result :gas_wanted :: NUMBER AS gas_wanted, + CASE + WHEN t.value :tx_result :code :: NUMBER = 0 THEN TRUE + ELSE FALSE + END AS tx_succeeded, + t.value :tx_result :code :: NUMBER AS tx_code, + t.value :tx_result :events AS msgs, + t.value :tx_result :log :: STRING AS tx_log, + _inserted_timestamp + FROM + {{ ref('bronze__tendermint_transactions') }}, + TABLE(FLATTEN(DATA :result :txs)) t -WHERE tx_id is not null {% if is_incremental() %} -AND - bb._inserted_timestamp :: DATE >= ( - SELECT - MAX(_inserted_timestamp) :: DATE - 2 - FROM - {{ this }} - ) - AND b._inserted_timestamp :: DATE >= ( +WHERE + _inserted_timestamp :: DATE >= ( SELECT MAX(_inserted_timestamp) :: DATE - 2 FROM {{ this }} ) {% endif %} +) +SELECT + t.block_id, + b.block_timestamp, + tx_id, + codespace, + gas_used, + gas_wanted, + tx_succeeded, + tx_code, + msgs, + tx_log, + t._inserted_timestamp, + concat_ws( + '-', + t.block_id, + tx_id + ) AS unique_key +FROM + base_transactions t + LEFT OUTER JOIN {{ ref('silver__blocks') }} + b + ON t.block_id = b.block_id + +{% if is_incremental() %} +WHERE + b._inserted_timestamp :: DATE >= ( + SELECT + MAX(_inserted_timestamp) :: DATE - 2 + FROM + {{ this }} + ) +{% endif %} + +qualify ROW_NUMBER() over ( + PARTITION BY tx_id + ORDER BY + t._inserted_timestamp DESC +) = 1 From 9d28b09de4975aaff6c7aab8f0d0fbe801599742 Mon Sep 17 00:00:00 2001 From: jhuhnke Date: Tue, 23 May 2023 15:40:19 -0500 Subject: [PATCH 12/12] resolve failing tests --- models/core/core__fact_transactions.yml | 8 -------- models/core/core__fact_transfers.yml | 4 ---- models/silver/silver__blocks.sql | 2 ++ 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/models/core/core__fact_transactions.yml b/models/core/core__fact_transactions.yml index 3e13d2d..4461e0e 100644 --- a/models/core/core__fact_transactions.yml +++ b/models/core/core__fact_transactions.yml @@ -44,14 +44,6 @@ models: description: "{{ doc('gas_wanted') }}" tests: - dbt_expectations.expect_column_to_exist - - name: TX_CODE - description: "{{ doc('tx_code') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: TX_TYPE - description: "The type of transaction" - tests: - - dbt_expectations.expect_column_to_exist - name: MSGS description: "The underlying json from the messages or events within the transactions" tests: diff --git a/models/core/core__fact_transfers.yml b/models/core/core__fact_transfers.yml index d04153a..a19e550 100644 --- a/models/core/core__fact_transfers.yml +++ b/models/core/core__fact_transfers.yml @@ -37,9 +37,5 @@ models: - dbt_expectations.expect_column_to_exist - name: RECEIVER description: "{{ doc('receiver') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: UNIQUE_KEY - description: "{{ doc('_unique_key') }}" tests: - dbt_expectations.expect_column_to_exist \ No newline at end of file diff --git a/models/silver/silver__blocks.sql b/models/silver/silver__blocks.sql index 8ef46e7..ef8856a 100644 --- a/models/silver/silver__blocks.sql +++ b/models/silver/silver__blocks.sql @@ -35,6 +35,8 @@ FROM WHERE data [0] :error IS NULL AND DATA :error IS NULL + AND DATA :result :block :header :chain_id :: STRING IS NOT NULL + AND DATA[0] :result :block :header :chain_id :: STRING IS NOT NULL {% if is_incremental() %} AND _inserted_timestamp :: DATE >= (